private boolean containsPartition(Index index, Map<String, String> partSpec) throws HiveException { String[] qualified = Utilities.getDbTableName(index.getDbName(), index.getIndexTableName()); Table indexTable = hive.getTable(qualified[0], qualified[1]); List<Partition> parts = hive.getPartitions(indexTable, partSpec); return (parts == null || parts.size() == 0); } }
public static String getIndexInformation(Index index, boolean isOutputPadded) { StringBuilder indexInfo = new StringBuilder(DEFAULT_STRINGBUILDER_SIZE); List<String> indexColumns = new ArrayList<String>(); indexColumns.add(index.getIndexName()); indexColumns.add(index.getOrigTableName()); // index key names List<FieldSchema> indexKeys = index.getSd().getCols(); StringBuilder keyString = new StringBuilder(); boolean first = true; for (FieldSchema key : indexKeys) { if (!first) { keyString.append(", "); } keyString.append(key.getName()); first = false; } indexColumns.add(keyString.toString()); indexColumns.add(index.getIndexTableName()); // index type String indexHandlerClass = index.getIndexHandlerClass(); IndexType indexType = HiveIndex.getIndexTypeByClassName(indexHandlerClass); indexColumns.add(indexType.getName()); String comment = index.getParameters().get("comment"); indexColumns.add(comment == null ? null : HiveStringUtils.escapeJava(comment)); formatOutput(indexColumns.toArray(new String[0]), indexInfo, isOutputPadded); return indexInfo.toString(); }
try { String[] qualified = Utilities.getDbTableName(index.getDbName(), index.getIndexTableName()); Table idxTbl = hiveInstance.getTable(qualified[0], qualified[1]); for (FieldSchema idxTblCol : idxTbl.getCols()) {
if (indexPredicate != null) { iqs.add(new BitmapInnerQuery( index.getIndexTableName(), indexPredicate, "ind" + i++));
&& checkIfAllRewriteCriteriaIsMet(canApplyCtx)) { canApplyCtx.setAggFunction("_count_of_" + indexKeyName + ""); canApplyCtx.addTable(canApplyCtx.getBaseTableName(), index.getIndexTableName()); canApplyCtx.setIndexTableName(index.getIndexTableName()); tsOpToProcess.put(alias, canApplyCtx); return true;
/** * check that every index table contains the given partition and is fresh */ private static boolean containsPartition(Hive hive, Partition part, List<Index> indexes) throws HiveException { HashMap<String, String> partSpec = part.getSpec(); if (partSpec.isEmpty()) { // empty specs come from non-partitioned tables return isIndexTableFresh(hive, indexes, part.getTable()); } for (Index index : indexes) { // index.getDbName() is used as a default database, which is database of target table, // if index.getIndexTableName() does not contain database name String[] qualified = Utilities.getDbTableName(index.getDbName(), index.getIndexTableName()); Table indexTable = hive.getTable(qualified[0], qualified[1]); // get partitions that match the spec Partition matchingPartition = hive.getPartition(indexTable, partSpec, false); if (matchingPartition == null) { LOG.info("Index table " + indexTable + "did not contain built partition that matched " + partSpec); return false; } else if (!isIndexPartitionFresh(hive, index, part)) { return false; } } return true; }
qlCommand.append( "\"" + tmpFile + "\" "); // QL includes " around file name qlCommand.append("SELECT `_bucketname` , `_offsets` FROM "); qlCommand.append(HiveUtils.unparseIdentifier(index.getIndexTableName())); qlCommand.append(" WHERE ");
private List<Task<?>> getIndexBuilderMapRed(String[] names, String indexName, HashMap<String, String> partSpec) throws SemanticException { try { Index index = db.getIndex(names[0], names[1], indexName); Table indexTbl = null; String indexTableName = index.getIndexTableName(); if (indexTableName != null) { indexTbl = getTable(Utilities.getDbTableName(index.getDbName(), indexTableName)); } Table baseTbl = getTable(new String[] {index.getDbName(), index.getOrigTableName()}); String handlerCls = index.getIndexHandlerClass(); HiveIndexHandler handler = HiveUtils.getIndexHandler(conf, handlerCls); List<Partition> indexTblPartitions = null; List<Partition> baseTblPartitions = null; if (indexTbl != null) { indexTblPartitions = new ArrayList<Partition>(); baseTblPartitions = preparePartitions(baseTbl, partSpec, indexTbl, db, indexTblPartitions); } List<Task<?>> ret = handler.generateIndexBuildTaskList(baseTbl, index, indexTblPartitions, baseTblPartitions, indexTbl, getInputs(), getOutputs()); return ret; } catch (Exception e) { throw new SemanticException(e); } }
private boolean containsPartition(Index index, Map<String, String> partSpec) throws HiveException { String[] qualified = Utilities.getDbTableName(index.getDbName(), index.getIndexTableName()); Table indexTable = hive.getTable(qualified[0], qualified[1]); List<Partition> parts = hive.getPartitions(indexTable, partSpec); return (parts == null || parts.size() == 0); } }
indexColumns.add(index.getIndexTableName());
public Object getFieldValue(_Fields field) { switch (field) { case INDEX_NAME: return getIndexName(); case INDEX_HANDLER_CLASS: return getIndexHandlerClass(); case DB_NAME: return getDbName(); case ORIG_TABLE_NAME: return getOrigTableName(); case CREATE_TIME: return Integer.valueOf(getCreateTime()); case LAST_ACCESS_TIME: return Integer.valueOf(getLastAccessTime()); case INDEX_TABLE_NAME: return getIndexTableName(); case SD: return getSd(); case PARAMETERS: return getParameters(); case DEFERRED_REBUILD: return Boolean.valueOf(isDeferredRebuild()); } throw new IllegalStateException(); }
public Object getFieldValue(_Fields field) { switch (field) { case INDEX_NAME: return getIndexName(); case INDEX_HANDLER_CLASS: return getIndexHandlerClass(); case DB_NAME: return getDbName(); case ORIG_TABLE_NAME: return getOrigTableName(); case CREATE_TIME: return Integer.valueOf(getCreateTime()); case LAST_ACCESS_TIME: return Integer.valueOf(getLastAccessTime()); case INDEX_TABLE_NAME: return getIndexTableName(); case SD: return getSd(); case PARAMETERS: return getParameters(); case DEFERRED_REBUILD: return Boolean.valueOf(isDeferredRebuild()); } throw new IllegalStateException(); }
public Object getFieldValue(_Fields field) { switch (field) { case INDEX_NAME: return getIndexName(); case INDEX_HANDLER_CLASS: return getIndexHandlerClass(); case DB_NAME: return getDbName(); case ORIG_TABLE_NAME: return getOrigTableName(); case CREATE_TIME: return new Integer(getCreateTime()); case LAST_ACCESS_TIME: return new Integer(getLastAccessTime()); case INDEX_TABLE_NAME: return getIndexTableName(); case SD: return getSd(); case PARAMETERS: return getParameters(); case DEFERRED_REBUILD: return new Boolean(isDeferredRebuild()); } throw new IllegalStateException(); }
/** * check that every index table contains the given partition and is fresh */ private static boolean containsPartition(Hive hive, Partition part, List<Index> indexes) throws HiveException { HashMap<String, String> partSpec = part.getSpec(); if (partSpec.isEmpty()) { // empty specs come from non-partitioned tables return isIndexTableFresh(hive, indexes, part.getTable()); } for (Index index : indexes) { // index.getDbName() is used as a default database, which is database of target table, // if index.getIndexTableName() does not contain database name String[] qualified = Utilities.getDbTableName(index.getDbName(), index.getIndexTableName()); Table indexTable = hive.getTable(qualified[0], qualified[1]); // get partitions that match the spec Partition matchingPartition = hive.getPartition(indexTable, partSpec, false); if (matchingPartition == null) { LOG.info("Index table " + indexTable + "did not contain built partition that matched " + partSpec); return false; } else if (!isIndexPartitionFresh(hive, index, part)) { return false; } } return true; }
private List<Task<?>> getIndexBuilderMapRed(String baseTableName, String indexName, HashMap<String, String> partSpec) throws SemanticException { try { String dbName = db.getCurrentDatabase(); Index index = db.getIndex(dbName, baseTableName, indexName); Table indexTbl = db.getTable(dbName, index.getIndexTableName()); String baseTblName = index.getOrigTableName(); Table baseTbl = db.getTable(dbName, baseTblName); String handlerCls = index.getIndexHandlerClass(); HiveIndexHandler handler = HiveUtils.getIndexHandler(conf, handlerCls); List<Partition> indexTblPartitions = null; List<Partition> baseTblPartitions = null; if(indexTbl != null) { indexTblPartitions = new ArrayList<Partition>(); baseTblPartitions = preparePartitions(baseTbl, partSpec, indexTbl, db, indexTblPartitions); } List<Task<?>> ret = handler.generateIndexBuildTaskList(baseTbl, index, indexTblPartitions, baseTblPartitions, indexTbl, getInputs(), getOutputs()); return ret; } catch (Exception e) { throw new SemanticException(e); } }
private List<Task<?>> getIndexBuilderMapRed(String[] names, String indexName, HashMap<String, String> partSpec) throws SemanticException { try { Index index = db.getIndex(names[0], names[1], indexName); Table indexTbl = null; String indexTableName = index.getIndexTableName(); if (indexTableName != null) { indexTbl = getTable(Utilities.getDbTableName(index.getDbName(), indexTableName)); } Table baseTbl = getTable(new String[] {index.getDbName(), index.getOrigTableName()}); String handlerCls = index.getIndexHandlerClass(); HiveIndexHandler handler = HiveUtils.getIndexHandler(conf, handlerCls); List<Partition> indexTblPartitions = null; List<Partition> baseTblPartitions = null; if (indexTbl != null) { indexTblPartitions = new ArrayList<Partition>(); baseTblPartitions = preparePartitions(baseTbl, partSpec, indexTbl, db, indexTblPartitions); } List<Task<?>> ret = handler.generateIndexBuildTaskList(baseTbl, index, indexTblPartitions, baseTblPartitions, indexTbl, getInputs(), getOutputs()); return ret; } catch (Exception e) { throw new SemanticException(e); } }
private MIndex convertToMIndex(Index index) throws InvalidObjectException, MetaException { StorageDescriptor sd = index.getSd(); if (sd == null) { throw new InvalidObjectException("Storage descriptor is not defined for index."); } MStorageDescriptor msd = this.convertToMStorageDescriptor(sd); MTable origTable = getMTable(index.getDbName(), index.getOrigTableName()); if (origTable == null) { throw new InvalidObjectException( "Original table does not exist for the given index."); } MTable indexTable = getMTable(index.getDbName(), index.getIndexTableName()); if (indexTable == null) { throw new InvalidObjectException( "Underlying index table does not exist for the given index."); } return new MIndex(index.getIndexName(), origTable, index.getCreateTime(), index.getLastAccessTime(), index.getParameters(), indexTable, msd, index.getIndexHandlerClass(), index.isDeferredRebuild()); }
private MIndex convertToMIndex(Index index) throws InvalidObjectException, MetaException { StorageDescriptor sd = index.getSd(); if (sd == null) { throw new InvalidObjectException("Storage descriptor is not defined for index."); } MStorageDescriptor msd = this.convertToMStorageDescriptor(sd); MTable origTable = getMTable(index.getDbName(), index.getOrigTableName()); if (origTable == null) { throw new InvalidObjectException( "Original table does not exist for the given index."); } MTable indexTable = getMTable(index.getDbName(), index.getIndexTableName()); if (indexTable == null) { throw new InvalidObjectException( "Underlying index table does not exist for the given index."); } return new MIndex(index.getIndexName(), origTable, index.getCreateTime(), index.getLastAccessTime(), index.getParameters(), indexTable, msd, index.getIndexHandlerClass(), index.isDeferredRebuild()); }
private MIndex convertToMIndex(Index index) throws InvalidObjectException, MetaException { StorageDescriptor sd = index.getSd(); if (sd == null) { throw new InvalidObjectException("Storage descriptor is not defined for index."); } MStorageDescriptor msd = this.convertToMStorageDescriptor(sd); MTable origTable = getMTable(index.getDbName(), index.getOrigTableName()); if (origTable == null) { throw new InvalidObjectException( "Original table does not exist for the given index."); } String[] qualified = MetaStoreUtils.getQualifiedName(index.getDbName(), index.getIndexTableName()); MTable indexTable = getMTable(qualified[0], qualified[1]); if (indexTable == null) { throw new InvalidObjectException( "Underlying index table does not exist for the given index."); } return new MIndex(HiveStringUtils.normalizeIdentifier(index.getIndexName()), origTable, index.getCreateTime(), index.getLastAccessTime(), index.getParameters(), indexTable, msd, index.getIndexHandlerClass(), index.isDeferredRebuild()); }
private MIndex convertToMIndex(Index index) throws InvalidObjectException, MetaException { StorageDescriptor sd = index.getSd(); if (sd == null) { throw new InvalidObjectException("Storage descriptor is not defined for index."); } MStorageDescriptor msd = this.convertToMStorageDescriptor(sd); MTable origTable = getMTable(index.getDbName(), index.getOrigTableName()); if (origTable == null) { throw new InvalidObjectException( "Original table does not exist for the given index."); } String[] qualified = MetaStoreUtils.getQualifiedName(index.getDbName(), index.getIndexTableName()); MTable indexTable = getMTable(qualified[0], qualified[1]); if (indexTable == null) { throw new InvalidObjectException( "Underlying index table does not exist for the given index."); } return new MIndex(HiveStringUtils.normalizeIdentifier(index.getIndexName()), origTable, index.getCreateTime(), index.getLastAccessTime(), index.getParameters(), indexTable, msd, index.getIndexHandlerClass(), index.isDeferredRebuild()); }