private void doIndexUpdate(List<Index> tblIndexes) throws HiveException { Driver driver = new Driver(this.conf); for (Index idx : tblIndexes) { StringBuilder sb = new StringBuilder(); sb.append("ALTER INDEX "); sb.append(idx.getIndexName()); sb.append(" ON "); sb.append(idx.getDbName()).append('.'); sb.append(idx.getOrigTableName()); sb.append(" REBUILD"); driver.compile(sb.toString(), false); tasks.addAll(driver.getPlan().getRootTasks()); inputs.addAll(driver.getPlan().getInputs()); } }
private void doIndexUpdate(Index index, Map<String, String> partSpec) throws HiveException { StringBuilder ps = new StringBuilder(); boolean first = true; ps.append("("); for (String key : partSpec.keySet()) { if (!first) { ps.append(", "); } else { first = false; } ps.append(key); ps.append("="); ps.append(partSpec.get(key)); } ps.append(")"); StringBuilder sb = new StringBuilder(); sb.append("ALTER INDEX "); sb.append(index.getIndexName()); sb.append(" ON "); sb.append(index.getDbName()).append('.'); sb.append(index.getOrigTableName()); sb.append(" PARTITION "); sb.append(ps.toString()); sb.append(" REBUILD"); Driver driver = new Driver(this.conf); driver.compile(sb.toString(), false); tasks.addAll(driver.getPlan().getRootTasks()); inputs.addAll(driver.getPlan().getInputs()); }
private boolean containsPartition(Index index, Map<String, String> partSpec) throws HiveException { String[] qualified = Utilities.getDbTableName(index.getDbName(), index.getIndexTableName()); Table indexTable = hive.getTable(qualified[0], qualified[1]); List<Partition> parts = hive.getPartitions(indexTable, partSpec); return (parts == null || parts.size() == 0); } }
String[] qualified = Utilities.getDbTableName(index.getDbName(), index.getIndexTableName()); Table idxTbl = hiveInstance.getTable(qualified[0], qualified[1]);
/** * check that every index table contains the given partition and is fresh */ private static boolean containsPartition(Hive hive, Partition part, List<Index> indexes) throws HiveException { HashMap<String, String> partSpec = part.getSpec(); if (partSpec.isEmpty()) { // empty specs come from non-partitioned tables return isIndexTableFresh(hive, indexes, part.getTable()); } for (Index index : indexes) { // index.getDbName() is used as a default database, which is database of target table, // if index.getIndexTableName() does not contain database name String[] qualified = Utilities.getDbTableName(index.getDbName(), index.getIndexTableName()); Table indexTable = hive.getTable(qualified[0], qualified[1]); // get partitions that match the spec Partition matchingPartition = hive.getPartition(indexTable, partSpec, false); if (matchingPartition == null) { LOG.info("Index table " + indexTable + "did not contain built partition that matched " + partSpec); return false; } else if (!isIndexPartitionFresh(hive, index, part)) { return false; } } return true; }
private List<Task<?>> getIndexBuilderMapRed(String[] names, String indexName, HashMap<String, String> partSpec) throws SemanticException { try { Index index = db.getIndex(names[0], names[1], indexName); Table indexTbl = null; String indexTableName = index.getIndexTableName(); if (indexTableName != null) { indexTbl = getTable(Utilities.getDbTableName(index.getDbName(), indexTableName)); } Table baseTbl = getTable(new String[] {index.getDbName(), index.getOrigTableName()}); String handlerCls = index.getIndexHandlerClass(); HiveIndexHandler handler = HiveUtils.getIndexHandler(conf, handlerCls); List<Partition> indexTblPartitions = null; List<Partition> baseTblPartitions = null; if (indexTbl != null) { indexTblPartitions = new ArrayList<Partition>(); baseTblPartitions = preparePartitions(baseTbl, partSpec, indexTbl, db, indexTblPartitions); } List<Task<?>> ret = handler.generateIndexBuildTaskList(baseTbl, index, indexTblPartitions, baseTblPartitions, indexTbl, getInputs(), getOutputs()); return ret; } catch (Exception e) { throw new SemanticException(e); } }
private void doIndexUpdate(List<Index> tblIndexes) throws HiveException { Driver driver = new Driver(this.conf); for (Index idx : tblIndexes) { StringBuilder sb = new StringBuilder(); sb.append("ALTER INDEX "); sb.append(idx.getIndexName()); sb.append(" ON "); sb.append(idx.getDbName()).append('.'); sb.append(idx.getOrigTableName()); sb.append(" REBUILD"); driver.compile(sb.toString(), false); tasks.addAll(driver.getPlan().getRootTasks()); inputs.addAll(driver.getPlan().getInputs()); } }
private void doIndexUpdate(Index index, Map<String, String> partSpec) throws HiveException { StringBuilder ps = new StringBuilder(); boolean first = true; ps.append("("); for (String key : partSpec.keySet()) { if (!first) { ps.append(", "); } else { first = false; } ps.append(key); ps.append("="); ps.append(partSpec.get(key)); } ps.append(")"); StringBuilder sb = new StringBuilder(); sb.append("ALTER INDEX "); sb.append(index.getIndexName()); sb.append(" ON "); sb.append(index.getDbName()).append('.'); sb.append(index.getOrigTableName()); sb.append(" PARTITION "); sb.append(ps.toString()); sb.append(" REBUILD"); Driver driver = new Driver(this.conf); driver.compile(sb.toString(), false); tasks.addAll(driver.getPlan().getRootTasks()); inputs.addAll(driver.getPlan().getInputs()); }
private boolean containsPartition(Index index, Map<String, String> partSpec) throws HiveException { String[] qualified = Utilities.getDbTableName(index.getDbName(), index.getIndexTableName()); Table indexTable = hive.getTable(qualified[0], qualified[1]); List<Partition> parts = hive.getPartitions(indexTable, partSpec); return (parts == null || parts.size() == 0); } }
public Object getFieldValue(_Fields field) { switch (field) { case INDEX_NAME: return getIndexName(); case INDEX_HANDLER_CLASS: return getIndexHandlerClass(); case DB_NAME: return getDbName(); case ORIG_TABLE_NAME: return getOrigTableName(); case CREATE_TIME: return Integer.valueOf(getCreateTime()); case LAST_ACCESS_TIME: return Integer.valueOf(getLastAccessTime()); case INDEX_TABLE_NAME: return getIndexTableName(); case SD: return getSd(); case PARAMETERS: return getParameters(); case DEFERRED_REBUILD: return Boolean.valueOf(isDeferredRebuild()); } throw new IllegalStateException(); }
public Object getFieldValue(_Fields field) { switch (field) { case INDEX_NAME: return getIndexName(); case INDEX_HANDLER_CLASS: return getIndexHandlerClass(); case DB_NAME: return getDbName(); case ORIG_TABLE_NAME: return getOrigTableName(); case CREATE_TIME: return Integer.valueOf(getCreateTime()); case LAST_ACCESS_TIME: return Integer.valueOf(getLastAccessTime()); case INDEX_TABLE_NAME: return getIndexTableName(); case SD: return getSd(); case PARAMETERS: return getParameters(); case DEFERRED_REBUILD: return Boolean.valueOf(isDeferredRebuild()); } throw new IllegalStateException(); }
public Object getFieldValue(_Fields field) { switch (field) { case INDEX_NAME: return getIndexName(); case INDEX_HANDLER_CLASS: return getIndexHandlerClass(); case DB_NAME: return getDbName(); case ORIG_TABLE_NAME: return getOrigTableName(); case CREATE_TIME: return new Integer(getCreateTime()); case LAST_ACCESS_TIME: return new Integer(getLastAccessTime()); case INDEX_TABLE_NAME: return getIndexTableName(); case SD: return getSd(); case PARAMETERS: return getParameters(); case DEFERRED_REBUILD: return new Boolean(isDeferredRebuild()); } throw new IllegalStateException(); }
@Override public Index add_index(final Index newIndex, final Table indexTable) throws InvalidObjectException, AlreadyExistsException, MetaException, TException { startFunction("add_index", ": db=" + newIndex.getDbName() + " tbl=" + newIndex.getOrigTableName() + " index=" + newIndex.getIndexName()); Index ret = null; try { ret = executeWithRetry(new Command<Index>() { @Override Index run(RawStore ms) throws Exception { return add_index_core(ms, newIndex, indexTable); } }); } catch (InvalidObjectException e) { throw e; } catch (AlreadyExistsException e) { throw e; } catch (MetaException e) { throw e; } catch (Exception e) { assert(e instanceof RuntimeException); throw (RuntimeException)e; } finally { endFunction("add_index"); } return ret; }
private List<Task<?>> getIndexBuilderMapRed(String[] names, String indexName, HashMap<String, String> partSpec) throws SemanticException { try { Index index = db.getIndex(names[0], names[1], indexName); Table indexTbl = null; String indexTableName = index.getIndexTableName(); if (indexTableName != null) { indexTbl = getTable(Utilities.getDbTableName(index.getDbName(), indexTableName)); } Table baseTbl = getTable(new String[] {index.getDbName(), index.getOrigTableName()}); String handlerCls = index.getIndexHandlerClass(); HiveIndexHandler handler = HiveUtils.getIndexHandler(conf, handlerCls); List<Partition> indexTblPartitions = null; List<Partition> baseTblPartitions = null; if (indexTbl != null) { indexTblPartitions = new ArrayList<Partition>(); baseTblPartitions = preparePartitions(baseTbl, partSpec, indexTbl, db, indexTblPartitions); } List<Task<?>> ret = handler.generateIndexBuildTaskList(baseTbl, index, indexTblPartitions, baseTblPartitions, indexTbl, getInputs(), getOutputs()); return ret; } catch (Exception e) { throw new SemanticException(e); } }
/** * check that every index table contains the given partition and is fresh */ private static boolean containsPartition(Hive hive, Partition part, List<Index> indexes) throws HiveException { HashMap<String, String> partSpec = part.getSpec(); if (partSpec.isEmpty()) { // empty specs come from non-partitioned tables return isIndexTableFresh(hive, indexes, part.getTable()); } for (Index index : indexes) { // index.getDbName() is used as a default database, which is database of target table, // if index.getIndexTableName() does not contain database name String[] qualified = Utilities.getDbTableName(index.getDbName(), index.getIndexTableName()); Table indexTable = hive.getTable(qualified[0], qualified[1]); // get partitions that match the spec Partition matchingPartition = hive.getPartition(indexTable, partSpec, false); if (matchingPartition == null) { LOG.info("Index table " + indexTable + "did not contain built partition that matched " + partSpec); return false; } else if (!isIndexPartitionFresh(hive, index, part)) { return false; } } return true; }
private MIndex convertToMIndex(Index index) throws InvalidObjectException, MetaException { StorageDescriptor sd = index.getSd(); if (sd == null) { throw new InvalidObjectException("Storage descriptor is not defined for index."); } MStorageDescriptor msd = this.convertToMStorageDescriptor(sd); MTable origTable = getMTable(index.getDbName(), index.getOrigTableName()); if (origTable == null) { throw new InvalidObjectException( "Original table does not exist for the given index."); } MTable indexTable = getMTable(index.getDbName(), index.getIndexTableName()); if (indexTable == null) { throw new InvalidObjectException( "Underlying index table does not exist for the given index."); } return new MIndex(index.getIndexName(), origTable, index.getCreateTime(), index.getLastAccessTime(), index.getParameters(), indexTable, msd, index.getIndexHandlerClass(), index.isDeferredRebuild()); }
private MIndex convertToMIndex(Index index) throws InvalidObjectException, MetaException { StorageDescriptor sd = index.getSd(); if (sd == null) { throw new InvalidObjectException("Storage descriptor is not defined for index."); } MStorageDescriptor msd = this.convertToMStorageDescriptor(sd); MTable origTable = getMTable(index.getDbName(), index.getOrigTableName()); if (origTable == null) { throw new InvalidObjectException( "Original table does not exist for the given index."); } MTable indexTable = getMTable(index.getDbName(), index.getIndexTableName()); if (indexTable == null) { throw new InvalidObjectException( "Underlying index table does not exist for the given index."); } return new MIndex(index.getIndexName(), origTable, index.getCreateTime(), index.getLastAccessTime(), index.getParameters(), indexTable, msd, index.getIndexHandlerClass(), index.isDeferredRebuild()); }
private MIndex convertToMIndex(Index index) throws InvalidObjectException, MetaException { StorageDescriptor sd = index.getSd(); if (sd == null) { throw new InvalidObjectException("Storage descriptor is not defined for index."); } MStorageDescriptor msd = this.convertToMStorageDescriptor(sd); MTable origTable = getMTable(index.getDbName(), index.getOrigTableName()); if (origTable == null) { throw new InvalidObjectException( "Original table does not exist for the given index."); } String[] qualified = MetaStoreUtils.getQualifiedName(index.getDbName(), index.getIndexTableName()); MTable indexTable = getMTable(qualified[0], qualified[1]); if (indexTable == null) { throw new InvalidObjectException( "Underlying index table does not exist for the given index."); } return new MIndex(HiveStringUtils.normalizeIdentifier(index.getIndexName()), origTable, index.getCreateTime(), index.getLastAccessTime(), index.getParameters(), indexTable, msd, index.getIndexHandlerClass(), index.isDeferredRebuild()); }
private MIndex convertToMIndex(Index index) throws InvalidObjectException, MetaException { StorageDescriptor sd = index.getSd(); if (sd == null) { throw new InvalidObjectException("Storage descriptor is not defined for index."); } MStorageDescriptor msd = this.convertToMStorageDescriptor(sd); MTable origTable = getMTable(index.getDbName(), index.getOrigTableName()); if (origTable == null) { throw new InvalidObjectException( "Original table does not exist for the given index."); } String[] qualified = MetaStoreUtils.getQualifiedName(index.getDbName(), index.getIndexTableName()); MTable indexTable = getMTable(qualified[0], qualified[1]); if (indexTable == null) { throw new InvalidObjectException( "Underlying index table does not exist for the given index."); } return new MIndex(HiveStringUtils.normalizeIdentifier(index.getIndexName()), origTable, index.getCreateTime(), index.getLastAccessTime(), index.getParameters(), indexTable, msd, index.getIndexHandlerClass(), index.isDeferredRebuild()); }
String[] qualified = MetaStoreUtils.getQualifiedName(index.getDbName(), idxTblName); Table tbl = get_table_core(qualified[0], qualified[1]); if (tbl.getSd() == null) {