LOG.info("checking index staleness..."); try { String indexTs = index.getParameters().get("base_timestamp"); if (indexTs == null) { return false;
/** * Check the index partitions on a partitioned table exist and are fresh */ private static boolean isIndexPartitionFresh(Hive hive, Index index, Partition part) throws HiveException { LOG.info("checking index staleness..."); try { String indexTs = index.getParameters().get(part.getSpec().toString()); if (indexTs == null) { return false; } FileSystem partFs = part.getDataLocation().getFileSystem(hive.getConf()); FileStatus[] parts = partFs.listStatus(part.getDataLocation(), FileUtils.HIDDEN_FILES_PATH_FILTER); for (FileStatus status : parts) { if (status.getModificationTime() > Long.parseLong(indexTs)) { LOG.info("Index is stale on partition '" + part.getName() + "'. Modified time (" + status.getModificationTime() + ") for '" + status.getPath() + "' is higher than index creation time (" + indexTs + ")."); return false; } } } catch (IOException e) { throw new HiveException("Failed to grab timestamp information from partition '" + part.getName() + "': " + e.getMessage(), e); } return true; }
public static String getIndexInformation(Index index, boolean isOutputPadded) { StringBuilder indexInfo = new StringBuilder(DEFAULT_STRINGBUILDER_SIZE); List<String> indexColumns = new ArrayList<String>(); indexColumns.add(index.getIndexName()); indexColumns.add(index.getOrigTableName()); // index key names List<FieldSchema> indexKeys = index.getSd().getCols(); StringBuilder keyString = new StringBuilder(); boolean first = true; for (FieldSchema key : indexKeys) { if (!first) { keyString.append(", "); } keyString.append(key.getName()); first = false; } indexColumns.add(keyString.toString()); indexColumns.add(index.getIndexTableName()); // index type String indexHandlerClass = index.getIndexHandlerClass(); IndexType indexType = HiveIndex.getIndexTypeByClassName(indexHandlerClass); indexColumns.add(indexType.getName()); String comment = index.getParameters().get("comment"); indexColumns.add(comment == null ? null : HiveStringUtils.escapeJava(comment)); formatOutput(indexColumns.toArray(new String[0]), indexInfo, isOutputPadded); return indexInfo.toString(); }
Map<String, String> paraList = index.getParameters(); if(paraList != null && paraList.containsKey("AGGREGATES")){ command.append(paraList.get("AGGREGATES") + " ");
@Override public void analyzeIndexDefinition(Table baseTable, Index index, Table indexTable) throws HiveException { StorageDescriptor storageDesc = index.getSd(); if (this.usesIndexTable() && indexTable != null) { StorageDescriptor indexTableSd = storageDesc.deepCopy(); List<FieldSchema> indexTblCols = indexTableSd.getCols(); FieldSchema bucketFileName = new FieldSchema("_bucketname", "string", ""); indexTblCols.add(bucketFileName); FieldSchema offSets = new FieldSchema("_offsets", "array<bigint>", ""); indexTblCols.add(offSets); Map<String, String> paraList = index.getParameters(); if(paraList != null && paraList.containsKey("AGGREGATES")){ String propValue = paraList.get("AGGREGATES"); if(propValue.contains(",")){ String[] aggFuncs = propValue.split(","); for (int i = 0; i < aggFuncs.length; i++) { createAggregationFunction(indexTblCols, aggFuncs[i]); } }else{ createAggregationFunction(indexTblCols, propValue); } } indexTable.setSd(indexTableSd); } }
idx.getParameters().putAll(alterIndex.getProps()); break; case UPDATETIMESTAMP: idx.getParameters().putAll(props); } catch (HiveException e) { throw new HiveException("ERROR: Failed to update index timestamps"); if (!updateModifiedParameters(idx.getParameters(), conf)) { return 1;
indexSd, new HashMap<String,String>(), deferredRebuild); if (indexComment != null) { indexDesc.getParameters().put("comment", indexComment); indexDesc.getParameters().putAll(idxProps);
indexColumns.add(indexType.getName()); indexColumns.add(index.getParameters().get("comment"));
public static String getAllColumnsInformation(Index index) { StringBuilder indexInfo = new StringBuilder(DEFAULT_STRINGBUILDER_SIZE); List<String> indexColumns = new ArrayList<String>(); indexColumns.add(index.getIndexName()); indexColumns.add(index.getOrigTableName()); // index key names List<FieldSchema> indexKeys = index.getSd().getCols(); StringBuilder keyString = new StringBuilder(); boolean first = true; for (FieldSchema key : indexKeys) { if (!first) { keyString.append(", "); } keyString.append(key.getName()); first = false; } indexColumns.add(keyString.toString()); indexColumns.add(index.getIndexTableName()); // index type String indexHandlerClass = index.getIndexHandlerClass(); IndexType indexType = HiveIndex.getIndexTypeByClassName(indexHandlerClass); indexColumns.add(indexType.getName()); indexColumns.add(index.getParameters().get("comment")); formatOutput(indexColumns.toArray(new String[0]), indexInfo); return indexInfo.toString(); }
LOG.info("checking index staleness..."); try { String indexTs = index.getParameters().get("base_timestamp"); if (indexTs == null) { return false;
/** * Check the index partitions on a partitioned table exist and are fresh */ private static boolean isIndexPartitionFresh(Hive hive, Index index, Partition part) throws HiveException { LOG.info("checking index staleness..."); try { String indexTs = index.getParameters().get(part.getSpec().toString()); if (indexTs == null) { return false; } FileSystem partFs = part.getDataLocation().getFileSystem(hive.getConf()); FileStatus[] parts = partFs.listStatus(part.getDataLocation(), FileUtils.HIDDEN_FILES_PATH_FILTER); for (FileStatus status : parts) { if (status.getModificationTime() > Long.parseLong(indexTs)) { LOG.info("Index is stale on partition '" + part.getName() + "'. Modified time (" + status.getModificationTime() + ") for '" + status.getPath() + "' is higher than index creation time (" + indexTs + ")."); return false; } } } catch (IOException e) { throw new HiveException("Failed to grab timestamp information from partition '" + part.getName() + "': " + e.getMessage(), e); } return true; }
@Override public void analyzeIndexDefinition(Table baseTable, Index index, Table indexTable) throws HiveException { StorageDescriptor storageDesc = index.getSd(); if (this.usesIndexTable() && indexTable != null) { StorageDescriptor indexTableSd = storageDesc.deepCopy(); List<FieldSchema> indexTblCols = indexTableSd.getCols(); FieldSchema bucketFileName = new FieldSchema("_bucketname", "string", ""); indexTblCols.add(bucketFileName); FieldSchema offSets = new FieldSchema("_offsets", "array<bigint>", ""); indexTblCols.add(offSets); Map<String, String> paraList = index.getParameters(); if(paraList != null && paraList.containsKey("AGGREGATES")){ String propValue = paraList.get("AGGREGATES"); if(propValue.contains(",")){ String[] aggFuncs = propValue.split(","); for (int i = 0; i < aggFuncs.length; i++) { createAggregationFunction(indexTblCols, aggFuncs[i]); } }else{ createAggregationFunction(indexTblCols, propValue); } } indexTable.setSd(indexTableSd); } }
public Object getFieldValue(_Fields field) { switch (field) { case INDEX_NAME: return getIndexName(); case INDEX_HANDLER_CLASS: return getIndexHandlerClass(); case DB_NAME: return getDbName(); case ORIG_TABLE_NAME: return getOrigTableName(); case CREATE_TIME: return Integer.valueOf(getCreateTime()); case LAST_ACCESS_TIME: return Integer.valueOf(getLastAccessTime()); case INDEX_TABLE_NAME: return getIndexTableName(); case SD: return getSd(); case PARAMETERS: return getParameters(); case DEFERRED_REBUILD: return Boolean.valueOf(isDeferredRebuild()); } throw new IllegalStateException(); }
public Object getFieldValue(_Fields field) { switch (field) { case INDEX_NAME: return getIndexName(); case INDEX_HANDLER_CLASS: return getIndexHandlerClass(); case DB_NAME: return getDbName(); case ORIG_TABLE_NAME: return getOrigTableName(); case CREATE_TIME: return Integer.valueOf(getCreateTime()); case LAST_ACCESS_TIME: return Integer.valueOf(getLastAccessTime()); case INDEX_TABLE_NAME: return getIndexTableName(); case SD: return getSd(); case PARAMETERS: return getParameters(); case DEFERRED_REBUILD: return Boolean.valueOf(isDeferredRebuild()); } throw new IllegalStateException(); }
public Object getFieldValue(_Fields field) { switch (field) { case INDEX_NAME: return getIndexName(); case INDEX_HANDLER_CLASS: return getIndexHandlerClass(); case DB_NAME: return getDbName(); case ORIG_TABLE_NAME: return getOrigTableName(); case CREATE_TIME: return new Integer(getCreateTime()); case LAST_ACCESS_TIME: return new Integer(getLastAccessTime()); case INDEX_TABLE_NAME: return getIndexTableName(); case SD: return getSd(); case PARAMETERS: return getParameters(); case DEFERRED_REBUILD: return new Boolean(isDeferredRebuild()); } throw new IllegalStateException(); }
private int alterIndex(Hive db, AlterIndexDesc alterIndex) throws HiveException { String dbName = alterIndex.getDbName(); String baseTableName = alterIndex.getBaseTableName(); String indexName = alterIndex.getIndexName(); Index idx = db.getIndex(dbName, baseTableName, indexName); if (alterIndex.getOp() == AlterIndexDesc.AlterIndexTypes.ADDPROPS) { idx.getParameters().putAll(alterIndex.getProps()); } else { console.printError("Unsupported Alter commnad"); return 1; } // set last modified by properties if (!updateModifiedParameters(idx.getParameters(), conf)) { return 1; } try { db.alterIndex(dbName, baseTableName, indexName, idx); } catch (InvalidOperationException e) { console.printError("Invalid alter operation: " + e.getMessage()); LOG.info("alter index: " + stringifyException(e)); return 1; } catch (HiveException e) { console.printError("Invalid alter operation: " + e.getMessage()); return 1; } return 0; }
private MIndex convertToMIndex(Index index) throws InvalidObjectException, MetaException { StorageDescriptor sd = index.getSd(); if (sd == null) { throw new InvalidObjectException("Storage descriptor is not defined for index."); } MStorageDescriptor msd = this.convertToMStorageDescriptor(sd); MTable origTable = getMTable(index.getDbName(), index.getOrigTableName()); if (origTable == null) { throw new InvalidObjectException( "Original table does not exist for the given index."); } MTable indexTable = getMTable(index.getDbName(), index.getIndexTableName()); if (indexTable == null) { throw new InvalidObjectException( "Underlying index table does not exist for the given index."); } return new MIndex(index.getIndexName(), origTable, index.getCreateTime(), index.getLastAccessTime(), index.getParameters(), indexTable, msd, index.getIndexHandlerClass(), index.isDeferredRebuild()); }
private MIndex convertToMIndex(Index index) throws InvalidObjectException, MetaException { StorageDescriptor sd = index.getSd(); if (sd == null) { throw new InvalidObjectException("Storage descriptor is not defined for index."); } MStorageDescriptor msd = this.convertToMStorageDescriptor(sd); MTable origTable = getMTable(index.getDbName(), index.getOrigTableName()); if (origTable == null) { throw new InvalidObjectException( "Original table does not exist for the given index."); } MTable indexTable = getMTable(index.getDbName(), index.getIndexTableName()); if (indexTable == null) { throw new InvalidObjectException( "Underlying index table does not exist for the given index."); } return new MIndex(index.getIndexName(), origTable, index.getCreateTime(), index.getLastAccessTime(), index.getParameters(), indexTable, msd, index.getIndexHandlerClass(), index.isDeferredRebuild()); }
private MIndex convertToMIndex(Index index) throws InvalidObjectException, MetaException { StorageDescriptor sd = index.getSd(); if (sd == null) { throw new InvalidObjectException("Storage descriptor is not defined for index."); } MStorageDescriptor msd = this.convertToMStorageDescriptor(sd); MTable origTable = getMTable(index.getDbName(), index.getOrigTableName()); if (origTable == null) { throw new InvalidObjectException( "Original table does not exist for the given index."); } String[] qualified = MetaStoreUtils.getQualifiedName(index.getDbName(), index.getIndexTableName()); MTable indexTable = getMTable(qualified[0], qualified[1]); if (indexTable == null) { throw new InvalidObjectException( "Underlying index table does not exist for the given index."); } return new MIndex(HiveStringUtils.normalizeIdentifier(index.getIndexName()), origTable, index.getCreateTime(), index.getLastAccessTime(), index.getParameters(), indexTable, msd, index.getIndexHandlerClass(), index.isDeferredRebuild()); }
private MIndex convertToMIndex(Index index) throws InvalidObjectException, MetaException { StorageDescriptor sd = index.getSd(); if (sd == null) { throw new InvalidObjectException("Storage descriptor is not defined for index."); } MStorageDescriptor msd = this.convertToMStorageDescriptor(sd); MTable origTable = getMTable(index.getDbName(), index.getOrigTableName()); if (origTable == null) { throw new InvalidObjectException( "Original table does not exist for the given index."); } String[] qualified = MetaStoreUtils.getQualifiedName(index.getDbName(), index.getIndexTableName()); MTable indexTable = getMTable(qualified[0], qualified[1]); if (indexTable == null) { throw new InvalidObjectException( "Underlying index table does not exist for the given index."); } return new MIndex(HiveStringUtils.normalizeIdentifier(index.getIndexName()), origTable, index.getCreateTime(), index.getLastAccessTime(), index.getParameters(), indexTable, msd, index.getIndexHandlerClass(), index.isDeferredRebuild()); }