/** * Create a clone of the parse context */ public ParseContext getParseContext(ParseContext pCtx, List<Task<? extends Serializable>> rootTasks) { ParseContext clone = new ParseContext(queryState, pCtx.getOpToPartPruner(), pCtx.getOpToPartList(), pCtx.getTopOps(), pCtx.getJoinOps(), pCtx.getSmbMapJoinOps(), pCtx.getLoadTableWork(), pCtx.getLoadFileWork(), pCtx.getColumnStatsAutoGatherContexts(), pCtx.getContext(), pCtx.getIdToTableNameMap(), pCtx.getDestTableId(), pCtx.getUCtx(), pCtx.getListMapJoinOpsNoReducer(), pCtx.getPrunedPartitions(), pCtx.getTabNameToTabObject(), pCtx.getOpToSamplePruner(), pCtx.getGlobalLimitCtx(), pCtx.getNameToSplitSample(), pCtx.getSemanticInputs(), rootTasks, pCtx.getOpToPartToSkewedPruner(), pCtx.getViewAliasToInput(), pCtx.getReduceSinkOperatorsAddedByEnforceBucketingSorting(), pCtx.getAnalyzeRewrite(), pCtx.getCreateTable(), pCtx.getCreateViewDesc(), pCtx.getMaterializedViewUpdateDesc(), pCtx.getQueryProperties(), pCtx.getViewProjectToTableSchema(), pCtx.getAcidSinks()); clone.setFetchTask(pCtx.getFetchTask()); clone.setLineageInfo(pCtx.getLineageInfo()); clone.setMapJoinOps(pCtx.getMapJoinOps()); clone.setRsToRuntimeValuesInfoMap(pCtx.getRsToRuntimeValuesInfoMap()); clone.setRsToSemiJoinBranchInfo(pCtx.getRsToSemiJoinBranchInfo()); clone.setColExprToGBMap(pCtx.getColExprToGBMap()); clone.setSemiJoinHints(pCtx.getSemiJoinHints()); return clone; }
@Override public ParseContext transform(ParseContext pctx) throws SemanticException { // 0. We check the conditions to apply this transformation, // if we do not meet them we bail out final boolean cboEnabled = HiveConf.getBoolVar(pctx.getConf(), HiveConf.ConfVars.HIVE_CBO_ENABLED); final boolean returnPathEnabled = HiveConf.getBoolVar(pctx.getConf(), HiveConf.ConfVars.HIVE_CBO_RETPATH_HIVEOP); final boolean cboSucceeded = pctx.getContext().isCboSucceeded(); if(cboEnabled && returnPathEnabled && cboSucceeded) { return pctx; } // 1. We apply the transformation Map<Rule, NodeProcessor> opRules = new LinkedHashMap<Rule, NodeProcessor>(); opRules.put(new RuleRegExp("R1", "(" + SelectOperator.getOperatorName() + "%)"), new ProjectRemover()); GraphWalker ogw = new DefaultGraphWalker(new DefaultRuleDispatcher(null, opRules, null)); ArrayList<Node> topNodes = new ArrayList<Node>(); topNodes.addAll(pctx.getTopOps().values()); ogw.startWalking(topNodes, null); return pctx; }
public void initParseCtx(ParseContext pctx) { opToPartPruner = pctx.getOpToPartPruner(); opToPartList = pctx.getOpToPartList(); opToSamplePruner = pctx.getOpToSamplePruner(); topOps = pctx.getTopOps(); loadTableWork = pctx.getLoadTableWork(); loadFileWork = pctx.getLoadFileWork(); ctx = pctx.getContext(); destTableId = pctx.getDestTableId(); idToTableNameMap = pctx.getIdToTableNameMap(); uCtx = pctx.getUCtx(); listMapJoinOpsNoReducer = pctx.getListMapJoinOpsNoReducer(); prunedPartitions = pctx.getPrunedPartitions(); tabNameToTabObject = pctx.getTabNameToTabObject(); fetchTask = pctx.getFetchTask(); setLineageInfo(pctx.getLineageInfo()); }
/** * Get the partition list for the TS operator that satisfies the partition pruner * condition. */ public static PrunedPartitionList prune(TableScanOperator ts, ParseContext parseCtx, String alias) throws SemanticException { return prune(ts.getConf().getTableMetadata(), parseCtx.getOpToPartPruner().get(ts), parseCtx.getConf(), alias, parseCtx.getPrunedPartitions()); }
/** * Create a clone of the parse context */ public ParseContext getParseContext(ParseContext pCtx, List<Task<? extends Serializable>> rootTasks) { ParseContext clone = new ParseContext(conf, pCtx.getOpToPartPruner(), pCtx.getOpToPartList(), pCtx.getTopOps(), pCtx.getJoinOps(), pCtx.getSmbMapJoinOps(), pCtx.getLoadTableWork(), pCtx.getLoadFileWork(), pCtx.getContext(), pCtx.getIdToTableNameMap(), pCtx.getDestTableId(), pCtx.getUCtx(), pCtx.getListMapJoinOpsNoReducer(), pCtx.getPrunedPartitions(), pCtx.getOpToSamplePruner(), pCtx.getGlobalLimitCtx(), pCtx.getNameToSplitSample(), pCtx.getSemanticInputs(), rootTasks, pCtx.getOpToPartToSkewedPruner(), pCtx.getViewAliasToInput(), pCtx.getReduceSinkOperatorsAddedByEnforceBucketingSorting(), pCtx.getAnalyzeRewrite(), pCtx.getCreateTable(), pCtx.getQueryProperties()); clone.setFetchTask(pCtx.getFetchTask()); clone.setLineageInfo(pCtx.getLineageInfo()); clone.setMapJoinOps(pCtx.getMapJoinOps()); return clone; } }
Context ctx = pCtx.getContext(); GlobalLimitCtx globalLimitCtx = pCtx.getGlobalLimitCtx(); List<Task<MoveWork>> mvTask = new ArrayList<>(); List<LoadTableDesc> loadTableWork = pCtx.getLoadTableWork(); List<LoadFileDesc> loadFileWork = pCtx.getLoadFileWork(); boolean isCStats = pCtx.getQueryProperties().isAnalyzeRewrite(); int outerQueryLimit = pCtx.getQueryProperties().getOuterQueryLimit(); if (pCtx.getFetchTask() != null) { if (pCtx.getFetchTask().getTblDesc() == null) { return; pCtx.getFetchTask().getWork().setHiveServerQuery(SessionState.get().isHiveServerQuery()); TableDesc resultTab = pCtx.getFetchTask().getTblDesc(); if (pCtx.getQueryProperties().isQuery() && !isCStats) { if ((!loadTableWork.isEmpty()) || (loadFileWork.size() != 1)) { throw new SemanticException(ErrorMsg.INVALID_LOAD_TABLE_FILE_WORK.getMsg()); TableDesc resultTab = pCtx.getFetchTableDesc(); if (resultTab == null) { resFileFormat = HiveConf.getVar(conf, HiveConf.ConfVars.HIVEQUERYRESULTFILEFORMAT); boolean isHiveServerQuery = SessionState.get().isHiveServerQuery(); fetch.setHiveServerQuery(isHiveServerQuery); fetch.setSource(pCtx.getFetchSource()); fetch.setSink(pCtx.getFetchSink());
public void init(ParseContext pctx) { opToPartPruner = pctx.getOpToPartPruner(); opToPartList = pctx.getOpToPartList(); opToSamplePruner = pctx.getOpToSamplePruner(); topOps = pctx.getTopOps(); topSelOps = pctx.getTopSelOps(); opParseCtx = pctx.getOpParseCtx(); loadTableWork = pctx.getLoadTableWork(); loadFileWork = pctx.getLoadFileWork(); joinContext = pctx.getJoinContext(); ctx = pctx.getContext(); destTableId = pctx.getDestTableId(); idToTableNameMap = pctx.getIdToTableNameMap(); uCtx = pctx.getUCtx(); listMapJoinOpsNoReducer = pctx.getListMapJoinOpsNoReducer(); qb = pctx.getQB(); groupOpToInputTables = pctx.getGroupOpToInputTables(); prunedPartitions = pctx.getPrunedPartitions(); setLineageInfo(pctx.getLineageInfo()); }
@Override public ParseContext transform(ParseContext pctx) throws SemanticException { if (pctx.getFetchTask() != null || !pctx.getQueryProperties().isQuery() || pctx.getQueryProperties().isAnalyzeRewrite() || pctx.getQueryProperties().isCTAS() || pctx.getLoadFileWork().size() > 1 || !pctx.getLoadTableWork().isEmpty()) { return pctx; } String GBY = GroupByOperator.getOperatorName() + "%"; String RS = ReduceSinkOperator.getOperatorName() + "%"; String SEL = SelectOperator.getOperatorName() + "%"; String FS = FileSinkOperator.getOperatorName() + "%"; Map<Rule, NodeProcessor> opRules = new LinkedHashMap<Rule, NodeProcessor>(); opRules.put(new RuleRegExp("R1", GBY + RS + GBY + SEL + FS), new SingleGBYProcessor(pctx)); opRules.put(new RuleRegExp("R2", GBY + RS + GBY + FS), new SingleGBYProcessor(pctx)); Dispatcher disp = new DefaultRuleDispatcher(null, opRules, null); GraphWalker ogw = new DefaultGraphWalker(disp); ArrayList<Node> topNodes = new ArrayList<Node>(); topNodes.addAll(pctx.getTopOps().values()); ogw.startWalking(topNodes, null); return pctx; }
private boolean isTableSorted(ParseContext pctx, MapJoinOperator op, QBJoinTree joinTree, String alias, int pos) throws SemanticException { Map<String, Operator<? extends Serializable>> topOps = this.pGraphContext .getTopOps(); Map<TableScanOperator, Table> topToTable = this.pGraphContext .getTopToTable(); TableScanOperator tso = (TableScanOperator) topOps.get(alias); if (tso == null) { PrunedPartitionList prunedParts = null; try { prunedParts = pGraphContext.getOpToPartList().get(tso); if (prunedParts == null) { prunedParts = PartitionPruner.prune(tbl, pGraphContext .getOpToPartPruner().get(tso), pGraphContext.getConf(), alias, pGraphContext.getPrunedPartitions()); pGraphContext.getOpToPartList().put(tso, prunedParts);
@Override public ParseContext transform(ParseContext pctx) throws SemanticException { Context ctx = pctx.getContext(); Map<String, TableScanOperator> topOps = pctx.getTopOps(); GlobalLimitCtx globalLimitCtx = pctx.getGlobalLimitCtx(); Map<String, SplitSample> nameToSplitSample = pctx.getNameToSplitSample(); PrunedPartitionList partsList = pctx.getPrunedPartitions(alias, ts);
if (!pctx.getQueryProperties().isCTAS() && !pctx.getQueryProperties().isMaterializedView() && pctx.getQueryProperties().isQuery() && pctx.getCreateTable() == null && pctx.getCreateViewDesc() == null && (pctx.getLoadTableWork() == null || pctx.getLoadTableWork().isEmpty())) { LOG.debug("Not evaluating lineage"); return pctx; Index index = pctx.getQueryState().getLineageState().getIndex(); if (index == null) { index = new Index(); topNodes.addAll(pctx.getTopOps().values()); ogw.startWalking(topNodes, null);
@Override public ParseContext transform(ParseContext pctx) throws SemanticException { LOG.info("TablePropertyEnrichmentOptimizer::transform()."); Map<Rule, NodeProcessor> opRules = Maps.newLinkedHashMap(); opRules.put(new RuleRegExp("R1", TableScanOperator.getOperatorName() + "%"), new Processor()); WalkerCtx context = new WalkerCtx(pctx.getConf()); Dispatcher disp = new DefaultRuleDispatcher(null, opRules, context); List<Node> topNodes = Lists.newArrayList(); topNodes.addAll(pctx.getTopOps().values()); GraphWalker walker = new PreOrderWalker(disp); walker.startWalking(topNodes, null); LOG.info("TablePropertyEnrichmentOptimizer::transform() complete!"); return pctx; } }
ParseContext pCtx = new ParseContext(queryState, opToPartPruner, opToPartList, topOps, new HashSet<JoinOperator>(joinContext.keySet()), new HashSet<SMBMapJoinOperator>(smbMapJoinContext.keySet()), pCtx.setSemiJoinHints(parseSemiJoinHint(getQB().getParseInfo().getHintList())); pCtx.setDisableMapJoin(disableMapJoinWithHint(getQB().getParseInfo().getHintList())); LOG.debug("Before logical optimization\n" + Operator.toString(pCtx.getTopOps().values())); optm.initialize(conf); pCtx = optm.optimize(); if (pCtx.getColumnAccessInfo() != null) { setColumnAccessInfo(pCtx.getColumnAccessInfo()); LOG.debug("After logical optimization\n" + Operator.toString(pCtx.getTopOps().values())); compiler.init(queryState, console, db); compiler.compile(pCtx, rootTasks, inputs, outputs); fetchTask = pCtx.getFetchTask();
public SyntheticContext(ParseContext pCtx) { parseContext = pCtx; extended = parseContext.getConf().getBoolVar(ConfVars.TEZ_DYNAMIC_PARTITION_PRUNING_EXTENDED); }
this.pGraphContext.getListMapJoinOpsNoReducer().indexOf(mapJoinOp); if (indexInListMapJoinNoReducer >= 0 ) { this.pGraphContext.getListMapJoinOpsNoReducer().remove(indexInListMapJoinNoReducer); this.pGraphContext.getListMapJoinOpsNoReducer().add(indexInListMapJoinNoReducer, smbJop); pGraphContext.getSmbMapJoinOps().add(smbJop); pGraphContext.getMapJoinOps().remove(mapJoinOp);
partsList = parseCtx.getOpToPartList().get((TableScanOperator)topOp); if (partsList == null) { partsList = PartitionPruner.prune(parseCtx.getTopToTable().get(topOp), parseCtx.getOpToPartPruner().get(topOp), opProcCtx.getConf(), alias_id, parseCtx.getPrunedPartitions()); parseCtx.getOpToPartList().put((TableScanOperator)topOp, partsList); .getTopToTable().get(topOp)), null); sampleDesc sampleDescr = parseCtx.getOpToSamplePruner().get(topOp);
ParseContext pCtx = new ParseContext(queryState, opToPartPruner, opToPartList, topOps, new HashSet<JoinOperator>(joinContext.keySet()), new HashSet<SMBMapJoinOperator>(smbMapJoinContext.keySet()), LOG.debug("Before logical optimization\n" + Operator.toString(pCtx.getTopOps().values())); optm.initialize(conf); pCtx = optm.optimize(); if (pCtx.getColumnAccessInfo() != null) { setColumnAccessInfo(pCtx.getColumnAccessInfo()); FetchTask origFetchTask = pCtx.getFetchTask(); if (LOG.isDebugEnabled()) { LOG.debug("After logical optimization\n" + Operator.toString(pCtx.getTopOps().values())); compiler.init(queryState, console, db); compiler.compile(pCtx, rootTasks, inputs, outputs); fetchTask = pCtx.getFetchTask();
public static void setOrAnnotateStats(Set<Operator<? extends OperatorDesc>> ops, ParseContext pctx) throws SemanticException { for (Operator<? extends OperatorDesc> op : ops) { if (pctx.getContext().getExplainAnalyze() == AnalyzeState.RUNNING) { setRuntimeStatsDir(op, pctx); } else if (pctx.getContext().getExplainAnalyze() == AnalyzeState.ANALYZING) { annotateRuntimeStats(op, pctx); } else { throw new SemanticException("Unexpected stats in AnnotateWithRunTimeStatistics."); } } }
private static void setRuntimeStatsDir(Operator<? extends OperatorDesc> op, ParseContext pctx) throws SemanticException { try { OperatorDesc conf = op.getConf(); if (conf != null) { LOG.info("setRuntimeStatsDir for " + op.getOperatorId()); String path = new Path(pctx.getContext().getExplainConfig().getExplainRootPath(), op.getOperatorId()).toString(); StatsPublisher statsPublisher = new FSStatsPublisher(); StatsCollectionContext runtimeStatsContext = new StatsCollectionContext(pctx.getConf()); runtimeStatsContext.setStatsTmpDir(path); if (!statsPublisher.init(runtimeStatsContext)) { LOG.error("StatsPublishing error: StatsPublisher is not initialized."); throw new HiveException(ErrorMsg.STATSPUBLISHER_NOT_OBTAINED.getErrorCodedMsg()); } conf.setRuntimeStatsTmpDir(path); } else { LOG.debug("skip setRuntimeStatsDir for " + op.getOperatorId() + " because OperatorDesc is null"); } } catch (HiveException e) { throw new SemanticException(e); } }
@Override public Object process(Node nd, Stack<Node> stack, NodeProcessorCtx procCtx, Object... nodeOutputs) throws SemanticException { TableScanOperator tsop = (TableScanOperator) nd; AnnotateStatsProcCtx aspCtx = (AnnotateStatsProcCtx) procCtx; PrunedPartitionList partList = aspCtx.getParseContext().getPrunedPartitions(tsop); ColumnStatsList colStatsCached = aspCtx.getParseContext().getColStatsCached(partList); Table table = tsop.getConf().getTableMetadata(); try { // gather statistics for the first time and the attach it to table scan operator Statistics stats = StatsUtils.collectStatistics(aspCtx.getConf(), partList, colStatsCached, table, tsop); stats = applyRuntimeStats(aspCtx.getParseContext().getContext(), stats, tsop); tsop.setStatistics(stats); if (LOG.isDebugEnabled()) { LOG.debug("[0] STATS-" + tsop.toString() + " (" + table.getTableName() + "): " + stats.extendedToString()); } } catch (HiveException e) { LOG.debug("Failed to retrieve stats ", e); throw new SemanticException(e); } return null; }