Refine search
public static Set<Partition> getConfirmedPartitionsForScan(TableScanOperator tableScanOp) { Set<Partition> confirmedPartns = new HashSet<Partition>(); TableSpec tblSpec = tableScanOp.getConf().getTableMetadata().getTableSpec(); if (tblSpec.specType == TableSpec.SpecType.STATIC_PARTITION) { // static partition if (tblSpec.partHandle != null) { confirmedPartns.add(tblSpec.partHandle); } else { // partial partition spec has null partHandle confirmedPartns.addAll(tblSpec.partitions); } } else if (tblSpec.specType == TableSpec.SpecType.DYNAMIC_PARTITION) { // dynamic partition confirmedPartns.addAll(tblSpec.partitions); } return confirmedPartns; }
private static ArrayListMultimap<String, TableScanOperator> splitTableScanOpsByTable( ParseContext pctx) { ArrayListMultimap<String, TableScanOperator> tableNameToOps = ArrayListMultimap.create(); // Sort by operator ID so we get deterministic results Map<String, TableScanOperator> sortedTopOps = new TreeMap<>(pctx.getTopOps()); for (Entry<String, TableScanOperator> e : sortedTopOps.entrySet()) { TableScanOperator tsOp = e.getValue(); tableNameToOps.put( tsOp.getConf().getTableMetadata().getDbName() + "." + tsOp.getConf().getTableMetadata().getTableName(), tsOp); } return tableNameToOps; }
public static Set<Partition> getConfirmedPartitionsForScan(TableScanOperator tableScanOp) { Set<Partition> confirmedPartns = new HashSet<Partition>(); TableSpec tblSpec = tableScanOp.getConf().getTableMetadata().getTableSpec(); if (tblSpec.specType == TableSpec.SpecType.STATIC_PARTITION) { // static partition if (tblSpec.partHandle != null) { confirmedPartns.add(tblSpec.partHandle); } else { // partial partition spec has null partHandle confirmedPartns.addAll(tblSpec.partitions); } } else if (tblSpec.specType == TableSpec.SpecType.DYNAMIC_PARTITION) { // dynamic partition confirmedPartns.addAll(tblSpec.partitions); } return confirmedPartns; }
private static boolean hasExternalTableAncestor(Operator op, StringBuilder sb) { boolean result = false; Operator ancestor = OperatorUtils.findSingleOperatorUpstream(op, TableScanOperator.class); if (ancestor != null) { TableScanOperator ts = (TableScanOperator) ancestor; if (MetaStoreUtils.isExternalTable(ts.getConf().getTableMetadata().getTTable())) { sb.append(ts.getConf().getTableMetadata().getFullyQualifiedName()); return true; } } return result; } }
public static List<String> getPartitionColumns(TableScanOperator tableScanOp) { TableSpec tblSpec = tableScanOp.getConf().getTableMetadata().getTableSpec(); if (tblSpec.tableHandle.isPartitioned()) { return new ArrayList<String>(tblSpec.getPartSpec().keySet()); } return Collections.emptyList(); }
/** * Get a list of indexes which can be used for rewrite. * @return * @throws SemanticException */ private Map<Table, List<Index>> getIndexesForRewrite() throws SemanticException{ List<String> supportedIndexes = new ArrayList<String>(); supportedIndexes.add(AggregateIndexHandler.class.getName()); // query the metastore to know what columns we have indexed Collection<TableScanOperator> topTables = parseContext.getTopOps().values(); Map<Table, List<Index>> indexes = new HashMap<Table, List<Index>>(); for (TableScanOperator op : topTables) { TableScanOperator tsOP = op; List<Index> tblIndexes = IndexUtils.getIndexes(tsOP.getConf().getTableMetadata(), supportedIndexes); if (tblIndexes.size() > 0) { indexes.put(tsOP.getConf().getTableMetadata(), tblIndexes); } } return indexes; }
public static List<String> getPartitionColumns(TableScanOperator tableScanOp) { TableSpec tblSpec = tableScanOp.getConf().getTableMetadata().getTableSpec(); if (tblSpec.tableHandle.isPartitioned()) { return new ArrayList<String>(tblSpec.getPartSpec().keySet()); } return Collections.emptyList(); }
private Set<String> getTablesUsed(ParseContext parseCtx) throws SemanticException { Set<String> tablesUsed = new HashSet<>(); for (TableScanOperator topOp : parseCtx.getTopOps().values()) { Table table = topOp.getConf().getTableMetadata(); if (!table.isMaterializedTable() && !table.isView()) { // Add to signature tablesUsed.add(table.getFullyQualifiedName()); } } return tablesUsed; }
/** * Get the partition list for the TS operator that satisfies the partition pruner * condition. */ public static PrunedPartitionList prune(TableScanOperator ts, ParseContext parseCtx, String alias) throws SemanticException { return prune(ts.getConf().getTableMetadata(), parseCtx.getOpToPartPruner().get(ts), parseCtx.getConf(), alias, parseCtx.getPrunedPartitions()); }
private static List<Entry<String, Long>> rankTablesByAccumulatedSize(ParseContext pctx) { Map<String, Long> tableToTotalSize = new HashMap<>(); for (Entry<String, TableScanOperator> e : pctx.getTopOps().entrySet()) { TableScanOperator tsOp = e.getValue(); String tableName = tsOp.getConf().getTableMetadata().getDbName() + "." + tsOp.getConf().getTableMetadata().getTableName(); long tableSize = tsOp.getStatistics() != null ? tsOp.getStatistics().getDataSize() : 0L; Long totalSize = tableToTotalSize.get(tableName); if (totalSize != null) { tableToTotalSize.put(tableName, StatsUtils.safeAdd(totalSize, tableSize)); } else { tableToTotalSize.put(tableName, tableSize); } } List<Entry<String, Long>> sortedTables = new ArrayList<>(tableToTotalSize.entrySet()); Collections.sort(sortedTables, Collections.reverseOrder( new Comparator<Map.Entry<String, Long>>() { @Override public int compare(Map.Entry<String, Long> o1, Map.Entry<String, Long> o2) { return (o1.getValue()).compareTo(o2.getValue()); } })); return sortedTables; }
private void processAlias(MapWork work, Set<TableScanOperator> tableScans) { Set<String> aliases = new HashSet<>(); for (TableScanOperator tso : tableScans) { // use LinkedHashMap<String, Operator<? extends OperatorDesc>> // getAliasToWork() should not apply this for non-native table if (tso.getConf().getTableMetadata().getStorageHandler() != null) { continue; } String alias = getAliasForTableScanOperator(work, tso); aliases.add(alias); tso.getConf().setIsMetadataOnly(true); } // group path alias according to work Map<Path, ArrayList<String>> candidates = new HashMap<>(); for (Path path : work.getPaths()) { ArrayList<String> aliasesAffected = work.getPathToAliases().get(path); if (CollectionUtils.isNotEmpty(aliasesAffected)) { candidates.put(path, aliasesAffected); } } for (Entry<Path, ArrayList<String>> entry : candidates.entrySet()) { processAlias(work, entry.getKey(), entry.getValue(), aliases); } }
public static void addInputsForView(ParseContext parseCtx) throws HiveException { Set<ReadEntity> inputs = parseCtx.getSemanticInputs(); for (Map.Entry<String, TableScanOperator> entry : parseCtx.getTopOps().entrySet()) { String alias = entry.getKey(); TableScanOperator topOp = entry.getValue(); ReadEntity parentViewInfo = getParentViewInfo(alias, parseCtx.getViewAliasToInput()); // Adds tables only for create view (PPD filter can be appended by outer query) Table table = topOp.getConf().getTableMetadata(); PlanUtils.addInput(inputs, new ReadEntity(table, parentViewInfo)); } }
/** * Get the partition list for the TS operator that satisfies the partition pruner * condition. */ public static PrunedPartitionList prune(TableScanOperator ts, ParseContext parseCtx, String alias) throws SemanticException { return prune(ts.getConf().getTableMetadata(), parseCtx.getOpToPartPruner().get(ts), parseCtx.getConf(), alias, parseCtx.getPrunedPartitions()); }
TableScanOperator tsOp = (TableScanOperator) nd; WalkerCtx context = (WalkerCtx)procCtx; TableScanDesc tableScanDesc = tsOp.getConf(); Table table = tsOp.getConf().getTableMetadata().getTTable(); deserializerClassName = tableScanDesc.getTableMetadata().getSd().getSerdeInfo().getSerializationLib(); Deserializer deserializer = ReflectionUtil.newInstance( context.conf.getClassByName(deserializerClassName)
public static void addInputsForView(ParseContext parseCtx) throws HiveException { Set<ReadEntity> inputs = parseCtx.getSemanticInputs(); for (Map.Entry<String, TableScanOperator> entry : parseCtx.getTopOps().entrySet()) { String alias = entry.getKey(); TableScanOperator topOp = entry.getValue(); ReadEntity parentViewInfo = getParentViewInfo(alias, parseCtx.getViewAliasToInput()); // Adds tables only for create view (PPD filter can be appended by outer query) Table table = topOp.getConf().getTableMetadata(); PlanUtils.addInput(inputs, new ReadEntity(table, parentViewInfo)); } }
private void processAlias(MapWork work, HashSet<TableScanOperator> tableScans) { ArrayList<String> aliases = new ArrayList<String>(); for (TableScanOperator tso : tableScans) { // use LinkedHashMap<String, Operator<? extends OperatorDesc>> // getAliasToWork() // should not apply this for non-native table if (tso.getConf().getTableMetadata().getStorageHandler() != null) { continue; } String alias = getAliasForTableScanOperator(work, tso); aliases.add(alias); tso.getConf().setIsMetadataOnly(true); } // group path alias according to work LinkedHashMap<Path, ArrayList<String>> candidates = new LinkedHashMap<>(); for (Path path : work.getPaths()) { ArrayList<String> aliasesAffected = work.getPathToAliases().get(path); if (aliasesAffected != null && aliasesAffected.size() > 0) { candidates.put(path, aliasesAffected); } } for (Entry<Path, ArrayList<String>> entry : candidates.entrySet()) { processAlias(work, entry.getKey(), entry.getValue(), aliases); } }
if (conf.getBoolVar(HiveConf.ConfVars.HIVE_DISABLE_UNSAFE_EXTERNALTABLE_OPERATIONS)) { if (MetaStoreUtils.isExternalTable(ts.getConf().getTableMetadata().getTTable())) { LOG.debug("Disabling semijoin optimzation on {} since it is an external table.", ts.getConf().getTableMetadata().getFullyQualifiedName()); disableSemiJoin = true; } else { if (MetaStoreUtils.isExternalTable(joinKeyTs.getConf().getTableMetadata().getTTable())) { LOG.debug("Join key {} is from {} which is an external table. Disabling semijoin optimization.", columnOrigin.col, joinKeyTs.getConf().getTableMetadata().getFullyQualifiedName()); disableSemiJoin = true;
for (Operator<? extends OperatorDesc> op : pctx.getTopOps().values()) { if (op instanceof TableScanOperator) { List<Index> tblIndexes = IndexUtils.getIndexes(((TableScanOperator) op).getConf() .getTableMetadata(), supportedIndexes); if (tblIndexes.size() > 0) { indexes.put((TableScanOperator) op, tblIndexes);
public static List<Path> getInputPathsForPartialScan(TableScanOperator tableScanOp, Appendable aggregationKey) throws SemanticException { List<Path> inputPaths = new ArrayList<Path>(); switch (tableScanOp.getConf().getTableMetadata().getTableSpec().specType) { case TABLE_ONLY: inputPaths.add(tableScanOp.getConf().getTableMetadata() .getTableSpec().tableHandle.getPath()); break; case STATIC_PARTITION: Partition part = tableScanOp.getConf().getTableMetadata() .getTableSpec().partHandle; try { aggregationKey.append(Warehouse.makePartPath(part.getSpec())); } catch (MetaException e) { throw new SemanticException(ErrorMsg.ANALYZE_TABLE_PARTIALSCAN_AGGKEY.getMsg( part.getDataLocation().toString() + e.getMessage())); } catch (IOException e) { throw new RuntimeException(e); } inputPaths.add(part.getDataLocation()); break; default: assert false; } return inputPaths; }
public MapWork createMapWork(GenTezProcContext context, Operator<?> root, TezWork tezWork, PrunedPartitionList partitions) throws SemanticException { assert root.getParentOperators().isEmpty(); MapWork mapWork = new MapWork(Utilities.MAPNAME + context.nextSequenceNumber()); LOG.debug("Adding map work (" + mapWork.getName() + ") for " + root); // map work starts with table scan operators assert root instanceof TableScanOperator; TableScanOperator ts = (TableScanOperator) root; String alias = ts.getConf().getAlias(); setupMapWork(mapWork, context, partitions, ts, alias); if (ts.getConf().getTableMetadata() != null && ts.getConf().getTableMetadata().isDummyTable()) { mapWork.setDummyTableScan(true); } if (ts.getConf().getNumBuckets() > 0) { mapWork.setIncludedBuckets(ts.getConf().getIncludedBuckets()); } // add new item to the tez work tezWork.add(mapWork); return mapWork; }