Index indexDesc = new Index(indexName, indexHandlerClass, tdname, ttname, time, time, itname, indexSd, new HashMap<String,String>(), deferredRebuild); if (indexComment != null) { indexDesc.getParameters().put("comment", indexComment); indexDesc.getParameters().putAll(idxProps);
public static String getIndexInformation(Index index, boolean isOutputPadded) { StringBuilder indexInfo = new StringBuilder(DEFAULT_STRINGBUILDER_SIZE); List<String> indexColumns = new ArrayList<String>(); indexColumns.add(index.getIndexName()); indexColumns.add(index.getOrigTableName()); // index key names List<FieldSchema> indexKeys = index.getSd().getCols(); StringBuilder keyString = new StringBuilder(); boolean first = true; for (FieldSchema key : indexKeys) { if (!first) { keyString.append(", "); } keyString.append(key.getName()); first = false; } indexColumns.add(keyString.toString()); indexColumns.add(index.getIndexTableName()); // index type String indexHandlerClass = index.getIndexHandlerClass(); IndexType indexType = HiveIndex.getIndexTypeByClassName(indexHandlerClass); indexColumns.add(indexType.getName()); String comment = index.getParameters().get("comment"); indexColumns.add(comment == null ? null : HiveStringUtils.escapeJava(comment)); formatOutput(indexColumns.toArray(new String[0]), indexInfo, isOutputPadded); return indexInfo.toString(); }
private void doIndexUpdate(List<Index> tblIndexes) throws HiveException { Driver driver = new Driver(this.conf); for (Index idx : tblIndexes) { StringBuilder sb = new StringBuilder(); sb.append("ALTER INDEX "); sb.append(idx.getIndexName()); sb.append(" ON "); sb.append(idx.getDbName()).append('.'); sb.append(idx.getOrigTableName()); sb.append(" REBUILD"); driver.compile(sb.toString(), false); tasks.addAll(driver.getPlan().getRootTasks()); inputs.addAll(driver.getPlan().getInputs()); } }
private boolean containsPartition(Index index, Map<String, String> partSpec) throws HiveException { String[] qualified = Utilities.getDbTableName(index.getDbName(), index.getIndexTableName()); Table indexTable = hive.getTable(qualified[0], qualified[1]); List<Partition> parts = hive.getPartitions(indexTable, partSpec); return (parts == null || parts.size() == 0); } }
public Object getFieldValue(_Fields field) { switch (field) { case INDEX_NAME: return getIndexName(); case INDEX_HANDLER_CLASS: return getIndexHandlerClass(); case DB_NAME: return getDbName(); case ORIG_TABLE_NAME: return getOrigTableName(); case CREATE_TIME: return Integer.valueOf(getCreateTime()); case LAST_ACCESS_TIME: return Integer.valueOf(getLastAccessTime()); case INDEX_TABLE_NAME: return getIndexTableName(); case SD: return getSd(); case PARAMETERS: return getParameters(); case DEFERRED_REBUILD: return Boolean.valueOf(isDeferredRebuild()); } throw new IllegalStateException(); }
Index old_index = null; try { old_index = get_index_by_name(index.getDbName(), index .getOrigTableName(), index.getIndexName()); } catch (Exception e) { throw new AlreadyExistsException("Index already exists:" + index); Table origTbl = ms.getTable(index.getDbName(), index.getOrigTableName()); if (origTbl == null) { throw new InvalidObjectException( if (indexTbl != null) { try { indexTbl = ms.getTable(index.getDbName(), index.getIndexTableName()); } catch (Exception e) { index.setCreateTime((int) time); index.putToParameters(hive_metastoreConstants.DDL_TIME, Long.toString(time)); if (indexTableCreated) { try { this.drop_table(index.getDbName(), index.getIndexTableName(), false); } catch (Exception e) {
private List<Task<?>> getIndexBuilderMapRed(String[] names, String indexName, HashMap<String, String> partSpec) throws SemanticException { try { Index index = db.getIndex(names[0], names[1], indexName); Table indexTbl = null; String indexTableName = index.getIndexTableName(); if (indexTableName != null) { indexTbl = getTable(Utilities.getDbTableName(index.getDbName(), indexTableName)); } Table baseTbl = getTable(new String[] {index.getDbName(), index.getOrigTableName()}); String handlerCls = index.getIndexHandlerClass(); HiveIndexHandler handler = HiveUtils.getIndexHandler(conf, handlerCls); List<Partition> indexTblPartitions = null; List<Partition> baseTblPartitions = null; if (indexTbl != null) { indexTblPartitions = new ArrayList<Partition>(); baseTblPartitions = preparePartitions(baseTbl, partSpec, indexTbl, db, indexTblPartitions); } List<Task<?>> ret = handler.generateIndexBuildTaskList(baseTbl, index, indexTblPartitions, baseTblPartitions, indexTbl, getInputs(), getOutputs()); return ret; } catch (Exception e) { throw new SemanticException(e); } }
private List<Task<?>> getIndexBuilderMapRed(String baseTableName, String indexName, HashMap<String, String> partSpec) throws SemanticException { try { String dbName = db.getCurrentDatabase(); Index index = db.getIndex(dbName, baseTableName, indexName); Table indexTbl = db.getTable(dbName, index.getIndexTableName()); String baseTblName = index.getOrigTableName(); Table baseTbl = db.getTable(dbName, baseTblName); String handlerCls = index.getIndexHandlerClass(); HiveIndexHandler handler = HiveUtils.getIndexHandler(conf, handlerCls); List<Partition> indexTblPartitions = null; List<Partition> baseTblPartitions = null; if(indexTbl != null) { indexTblPartitions = new ArrayList<Partition>(); baseTblPartitions = preparePartitions(baseTbl, partSpec, indexTbl, db, indexTblPartitions); } List<Task<?>> ret = handler.generateIndexBuildTaskList(baseTbl, index, indexTblPartitions, baseTblPartitions, indexTbl, getInputs(), getOutputs()); return ret; } catch (Exception e) { throw new SemanticException(e); } }
Index index = indexTables.get(idxCtr); StorageDescriptor sd = index.getSd(); List<FieldSchema> idxColList = sd.getCols(); assert idxColList.size()==1; String[] qualified = Utilities.getDbTableName(index.getDbName(), index.getIndexTableName()); Table idxTbl = hiveInstance.getTable(qualified[0], qualified[1]); for (FieldSchema idxTblCol : idxTbl.getCols()) {
@Override public void analyzeIndexDefinition(Table baseTable, Index index, Table indexTable) throws HiveException { StorageDescriptor storageDesc = index.getSd(); if (this.usesIndexTable() && indexTable != null) { StorageDescriptor indexTableSd = storageDesc.deepCopy(); List<FieldSchema> indexTblCols = indexTableSd.getCols(); FieldSchema bucketFileName = new FieldSchema("_bucketname", "string", ""); indexTblCols.add(bucketFileName); FieldSchema offSets = new FieldSchema("_offsets", "array<bigint>", ""); indexTblCols.add(offSets); Map<String, String> paraList = index.getParameters(); if(paraList != null && paraList.containsKey("AGGREGATES")){ String propValue = paraList.get("AGGREGATES"); if(propValue.contains(",")){ String[] aggFuncs = propValue.split(","); for (int i = 0; i < aggFuncs.length; i++) { createAggregationFunction(indexTblCols, aggFuncs[i]); } }else{ createAggregationFunction(indexTblCols, propValue); } } indexTable.setSd(indexTableSd); } }
protected Task<?> getIndexBuilderMapRedTask(Set<ReadEntity> inputs, Set<WriteEntity> outputs, Index index, boolean partitioned, PartitionDesc indexTblPartDesc, String indexTableName, PartitionDesc baseTablePartDesc, String baseTableName, String dbName) throws HiveException { return getIndexBuilderMapRedTask(inputs, outputs, index.getSd().getCols(), partitioned, indexTblPartDesc, indexTableName, baseTablePartDesc, baseTableName, dbName); }
public void alter_index(final String dbname, final String base_table_name, final String index_name, final Index newIndex) throws InvalidOperationException, MetaException { startFunction("alter_index", ": db=" + dbname + " base_tbl=" + base_table_name + " idx=" + index_name + " newidx=" + newIndex.getIndexName()); newIndex.putToParameters(Constants.DDL_TIME, Long.toString(System .currentTimeMillis() / 1000)); try { executeWithRetry(new Command<Boolean>() { @Override Boolean run(RawStore ms) throws Exception { ms.alterIndex(dbname, base_table_name, index_name, newIndex); return Boolean.TRUE; } }); } catch (MetaException e) { throw e; } catch (InvalidOperationException e) { throw e; } catch (Exception e) { assert(e instanceof RuntimeException); throw (RuntimeException)e; } finally { endFunction("alter_index"); } return; }
private Index deepCopy(Index index) { Index copy = null; if (index != null) { copy = new Index(index); } return copy; }
if (indexPredicate != null) { iqs.add(new BitmapInnerQuery( index.getIndexTableName(), indexPredicate, "ind" + i++));
LOG.info("checking index staleness..."); try { String indexTs = index.getParameters().get("base_timestamp"); if (indexTs == null) { return false;
/** * Get a list of indexes on a table that match given types. */ public static List<Index> getIndexes(Table baseTableMetaData, List<String> matchIndexTypes) throws SemanticException { List<Index> matchingIndexes = new ArrayList<Index>(); List<Index> indexesOnTable; try { indexesOnTable = getAllIndexes(baseTableMetaData, (short) -1); // get all indexes } catch (HiveException e) { throw new SemanticException("Error accessing metastore", e); } for (Index index : indexesOnTable) { String indexType = index.getIndexHandlerClass(); if (matchIndexTypes.contains(indexType)) { matchingIndexes.add(index); } } return matchingIndexes; }
while(indexes != null && indexes.size()>0) { for (Index idx : indexes) { this.drop_index_by_name(dbname, name, idx.getIndexName(), true);
if (!index.isDeferredRebuild()) { throw new HCatException(ErrorType.ERROR_NOT_SUPPORTED, "Store into a table with an automatic index from Pig/Mapreduce is not supported");
public Object getFieldValue(_Fields field) { switch (field) { case INDEX_NAME: return getIndexName(); case INDEX_HANDLER_CLASS: return getIndexHandlerClass(); case DB_NAME: return getDbName(); case ORIG_TABLE_NAME: return getOrigTableName(); case CREATE_TIME: return Integer.valueOf(getCreateTime()); case LAST_ACCESS_TIME: return Integer.valueOf(getLastAccessTime()); case INDEX_TABLE_NAME: return getIndexTableName(); case SD: return getSd(); case PARAMETERS: return getParameters(); case DEFERRED_REBUILD: return Boolean.valueOf(isDeferredRebuild()); } throw new IllegalStateException(); }
Index old_index = null; try { old_index = get_index_by_name(index.getDbName(), index .getOrigTableName(), index.getIndexName()); } catch (Exception e) { throw new AlreadyExistsException("Index already exists:" + index); Table origTbl = ms.getTable(index.getDbName(), index.getOrigTableName()); if (origTbl == null) { throw new InvalidObjectException( if (indexTbl != null) { try { indexTbl = ms.getTable(index.getDbName(), index.getIndexTableName()); } catch (Exception e) { index.setCreateTime((int) time); index.putToParameters(Constants.DDL_TIME, Long.toString(time)); if (indexTableCreated) { try { this.drop_table(index.getDbName(), index.getIndexTableName(), false); } catch (Exception e) {