private ExprNodeDesc generateInClauses(ExprNodeDesc predicate) throws SemanticException { Map<Rule, NodeProcessor> exprRules = new LinkedHashMap<Rule, NodeProcessor>(); exprRules.put(new TypeRule(ExprNodeGenericFuncDesc.class), new StructInExprProcessor()); // The dispatcher fires the processor corresponding to the closest matching // rule and passes the context along Dispatcher disp = new DefaultRuleDispatcher(null, exprRules, null); GraphWalker egw = new PreOrderOnceWalker(disp); List<Node> startNodes = new ArrayList<Node>(); startNodes.add(predicate); HashMap<Node, Object> outputMap = new HashMap<Node, Object>(); egw.startWalking(startNodes, outputMap); return (ExprNodeDesc) outputMap.get(predicate); } }
private ExprNodeDesc generateInClause(ExprNodeDesc predicate) throws SemanticException { Map<Rule, NodeProcessor> exprRules = new LinkedHashMap<Rule, NodeProcessor>(); exprRules.put(new TypeRule(ExprNodeGenericFuncDesc.class), new OrExprProcessor()); // The dispatcher fires the processor corresponding to the closest matching // rule and passes the context along Dispatcher disp = new DefaultRuleDispatcher(null, exprRules, null); GraphWalker egw = new PreOrderOnceWalker(disp); List<Node> startNodes = new ArrayList<Node>(); startNodes.add(predicate); HashMap<Node, Object> outputMap = new HashMap<Node, Object>(); egw.startWalking(startNodes, outputMap); return (ExprNodeDesc) outputMap.get(predicate); } }
private ExprNodeDesc generateInClauses(ExprNodeDesc predicate) throws SemanticException { Map<Rule, NodeProcessor> exprRules = new LinkedHashMap<Rule, NodeProcessor>(); exprRules.put(new TypeRule(ExprNodeGenericFuncDesc.class), new StructInExprProcessor()); // The dispatcher fires the processor corresponding to the closest matching // rule and passes the context along Dispatcher disp = new DefaultRuleDispatcher(null, exprRules, null); GraphWalker egw = new PreOrderOnceWalker(disp); List<Node> startNodes = new ArrayList<Node>(); startNodes.add(predicate); HashMap<Node, Object> outputMap = new HashMap<Node, Object>(); egw.startWalking(startNodes, outputMap); return (ExprNodeDesc) outputMap.get(predicate); } }
private ExprNodeDesc generateInClause(ExprNodeDesc predicate) throws SemanticException { Map<Rule, NodeProcessor> exprRules = new LinkedHashMap<Rule, NodeProcessor>(); exprRules.put(new TypeRule(ExprNodeGenericFuncDesc.class), new OrExprProcessor()); // The dispatcher fires the processor corresponding to the closest matching // rule and passes the context along Dispatcher disp = new DefaultRuleDispatcher(null, exprRules, null); GraphWalker egw = new PreOrderOnceWalker(disp); List<Node> startNodes = new ArrayList<Node>(); startNodes.add(predicate); HashMap<Node, Object> outputMap = new HashMap<Node, Object>(); egw.startWalking(startNodes, outputMap); return (ExprNodeDesc) outputMap.get(predicate); } }
private void markOperatorsWithUnstableRuntimeStats(OptimizeTezProcContext procCtx) throws SemanticException { Map<Rule, NodeProcessor> opRules = new LinkedHashMap<Rule, NodeProcessor>(); opRules.put( new RuleRegExp("R1", ReduceSinkOperator.getOperatorName() + "%"), new MarkRuntimeStatsAsIncorrect()); opRules.put( new RuleRegExp("R2", AppMasterEventOperator.getOperatorName() + "%"), new MarkRuntimeStatsAsIncorrect()); Dispatcher disp = new DefaultRuleDispatcher(null, opRules, procCtx); List<Node> topNodes = new ArrayList<Node>(); topNodes.addAll(procCtx.parseContext.getTopOps().values()); GraphWalker ogw = new PreOrderOnceWalker(disp); ogw.startWalking(topNodes, null); }
private void removeSemiJoinIfNoStats(OptimizeTezProcContext procCtx) throws SemanticException { Map<Rule, NodeProcessor> opRules = new LinkedHashMap<Rule, NodeProcessor>(); opRules.put( new RuleRegExp("R1", GroupByOperator.getOperatorName() + "%" + ReduceSinkOperator.getOperatorName() + "%" + GroupByOperator.getOperatorName() + "%" + ReduceSinkOperator.getOperatorName() + "%"), new SemiJoinRemovalProc(true, false)); SemiJoinRemovalContext ctx = new SemiJoinRemovalContext(procCtx.parseContext); Dispatcher disp = new DefaultRuleDispatcher(null, opRules, ctx); List<Node> topNodes = new ArrayList<Node>(); topNodes.addAll(procCtx.parseContext.getTopOps().values()); GraphWalker ogw = new PreOrderOnceWalker(disp); ogw.startWalking(topNodes, null); }
GraphWalker ogw = new PreOrderOnceWalker(disp);
private void vectorizeMapWork(MapWork mapWork, VectorTaskColumnInfo vectorTaskColumnInfo, boolean isTezOrSpark) throws SemanticException { LOG.info("Vectorizing MapWork..."); mapWork.setVectorMode(true); Map<Rule, NodeProcessor> opRules = new LinkedHashMap<Rule, NodeProcessor>(); MapWorkVectorizationNodeProcessor vnp = new MapWorkVectorizationNodeProcessor(mapWork, isTezOrSpark, vectorTaskColumnInfo); addMapWorkRules(opRules, vnp); Dispatcher disp = new DefaultRuleDispatcher(vnp, opRules, null); GraphWalker ogw = new PreOrderOnceWalker(disp); // iterator the mapper operator tree ArrayList<Node> topNodes = new ArrayList<Node>(); topNodes.addAll(mapWork.getAliasToWork().values()); HashMap<Node, Object> nodeOutput = new HashMap<Node, Object>(); ogw.startWalking(topNodes, nodeOutput); vectorTaskColumnInfo.setScratchTypeNameArray(vnp.getVectorScratchColumnTypeNames()); vectorTaskColumnInfo.transferToBaseWork(mapWork); if (LOG.isDebugEnabled()) { debugDisplayAllMaps(mapWork); } return; }
List<Node> topNodes = new ArrayList<Node>(); topNodes.addAll(procCtx.parseContext.getTopOps().values()); GraphWalker ogw = new PreOrderOnceWalker(disp); ogw.startWalking(topNodes, null);
GraphWalker ogw = new PreOrderOnceWalker(disp);
private void removeSemiJoinIfNoStats(OptimizeTezProcContext procCtx) throws SemanticException { if(!procCtx.conf.getBoolVar(ConfVars.TEZ_DYNAMIC_SEMIJOIN_REDUCTION)) { // Not needed without semi-join reduction return; } Map<Rule, NodeProcessor> opRules = new LinkedHashMap<Rule, NodeProcessor>(); opRules.put( new RuleRegExp("R1", GroupByOperator.getOperatorName() + "%" + ReduceSinkOperator.getOperatorName() + "%" + GroupByOperator.getOperatorName() + "%" + ReduceSinkOperator.getOperatorName() + "%"), new SemiJoinRemovalIfNoStatsProc()); Dispatcher disp = new DefaultRuleDispatcher(null, opRules, procCtx); List<Node> topNodes = new ArrayList<Node>(); topNodes.addAll(procCtx.parseContext.getTopOps().values()); GraphWalker ogw = new PreOrderOnceWalker(disp); ogw.startWalking(topNodes, null); }
@Override public ParseContext transform(ParseContext pctx) throws SemanticException { boolean enabled = false; String queryEngine = pctx.getConf().getVar(ConfVars.HIVE_EXECUTION_ENGINE); if (queryEngine.equals("tez") && pctx.getConf().getBoolVar(ConfVars.TEZ_DYNAMIC_PARTITION_PRUNING)) { enabled = true; } else if ((queryEngine.equals("spark") && pctx.getConf().isSparkDPPAny())) { enabled = true; } if (!enabled) { return pctx; } Map<Rule, NodeProcessor> opRules = new LinkedHashMap<Rule, NodeProcessor>(); opRules.put(new RuleRegExp("R1", "(" + TableScanOperator.getOperatorName() + "%" + ".*" + ReduceSinkOperator.getOperatorName() + "%" + JoinOperator.getOperatorName() + "%)"), new JoinSynthetic()); // The dispatcher fires the processor corresponding to the closest matching // rule and passes the context along SyntheticContext context = new SyntheticContext(pctx); Dispatcher disp = new DefaultRuleDispatcher(null, opRules, context); GraphWalker ogw = new PreOrderOnceWalker(disp); // Create a list of top op nodes List<Node> topNodes = new ArrayList<Node>(); topNodes.addAll(pctx.getTopOps().values()); ogw.startWalking(topNodes, null); return pctx; }
List<Node> topNodes = new ArrayList<Node>(); topNodes.addAll(procCtx.parseContext.getTopOps().values()); GraphWalker ogw = new PreOrderOnceWalker(disp); ogw.startWalking(topNodes, null);
List<Node> topNodes = new ArrayList<Node>(); topNodes.addAll(procCtx.parseContext.getTopOps().values()); GraphWalker ogw = new PreOrderOnceWalker(disp); ogw.startWalking(topNodes, null);
@Override public ParseContext transform(ParseContext pctx) throws SemanticException { boolean enabled = false; String queryEngine = pctx.getConf().getVar(ConfVars.HIVE_EXECUTION_ENGINE); if (queryEngine.equals("tez") && pctx.getConf().getBoolVar(ConfVars.TEZ_DYNAMIC_PARTITION_PRUNING)) { enabled = true; } else if ((queryEngine.equals("spark") && pctx.getConf().getBoolVar(ConfVars.SPARK_DYNAMIC_PARTITION_PRUNING))) { enabled = true; } if (!enabled) { return pctx; } Map<Rule, NodeProcessor> opRules = new LinkedHashMap<Rule, NodeProcessor>(); opRules.put(new RuleRegExp("R1", "(" + TableScanOperator.getOperatorName() + "%" + ".*" + ReduceSinkOperator.getOperatorName() + "%" + JoinOperator.getOperatorName() + "%)"), new JoinSynthetic()); // The dispatcher fires the processor corresponding to the closest matching // rule and passes the context along SyntheticContext context = new SyntheticContext(pctx); Dispatcher disp = new DefaultRuleDispatcher(null, opRules, context); GraphWalker ogw = new PreOrderOnceWalker(disp); // Create a list of top op nodes List<Node> topNodes = new ArrayList<Node>(); topNodes.addAll(pctx.getTopOps().values()); ogw.startWalking(topNodes, null); return pctx; }
GraphWalker ogw = new PreOrderOnceWalker(disp);
List<Node> topNodes = new ArrayList<Node>(); topNodes.addAll(procCtx.parseContext.getTopOps().values()); GraphWalker ogw = new PreOrderOnceWalker(disp); ogw.startWalking(topNodes, null);