/** * The operator name for this operator type. This is used to construct the * rule for an operator * * @return the operator name **/ @Override public String getName() { return TableScanOperator.getOperatorName(); }
/** * The operator name for this operator type. This is used to construct the * rule for an operator * * @return the operator name **/ @Override public String getName() { return TableScanOperator.getOperatorName(); }
private void addMapWorkRules(Map<Rule, NodeProcessor> opRules, NodeProcessor np) { opRules.put(new RuleRegExp("R1", TableScanOperator.getOperatorName() + ".*" + FileSinkOperator.getOperatorName()), np); opRules.put(new RuleRegExp("R2", TableScanOperator.getOperatorName() + ".*" + ReduceSinkOperator.getOperatorName()), np); }
private void addMapWorkRules(Map<Rule, NodeProcessor> opRules, NodeProcessor np) { opRules.put(new RuleRegExp("R1", TableScanOperator.getOperatorName() + ".*" + FileSinkOperator.getOperatorName()), np); opRules.put(new RuleRegExp("R2", TableScanOperator.getOperatorName() + ".*" + ReduceSinkOperator.getOperatorName()), np); }
@Override public ParseContext transform(ParseContext pctx) throws SemanticException { LOG.info("TablePropertyEnrichmentOptimizer::transform()."); Map<Rule, NodeProcessor> opRules = Maps.newLinkedHashMap(); opRules.put(new RuleRegExp("R1", TableScanOperator.getOperatorName() + "%"), new Processor()); WalkerCtx context = new WalkerCtx(pctx.getConf()); Dispatcher disp = new DefaultRuleDispatcher(null, opRules, context); List<Node> topNodes = Lists.newArrayList(); topNodes.addAll(pctx.getTopOps().values()); GraphWalker walker = new PreOrderWalker(disp); walker.startWalking(topNodes, null); LOG.info("TablePropertyEnrichmentOptimizer::transform() complete!"); return pctx; } }
/** * Walk operator tree for pruner generation. * * @param pctx * @param opWalkerCtx * @param filterProc * @param defaultProc * @throws SemanticException */ public static void walkOperatorTree(ParseContext pctx, NodeProcessorCtx opWalkerCtx, NodeProcessor filterProc, NodeProcessor defaultProc) throws SemanticException { Map<Rule, NodeProcessor> opRules = new LinkedHashMap<Rule, NodeProcessor>(); // Build regular expression for operator rule. // "(TS%FIL%)|(TS%FIL%FIL%)" String tsOprName = TableScanOperator.getOperatorName(); String filtOprName = FilterOperator.getOperatorName(); opRules.put(new RuleExactMatch("R1", new String[] {tsOprName, filtOprName, filtOprName}), filterProc); opRules.put(new RuleExactMatch("R2", new String[] {tsOprName, filtOprName}), filterProc); // The dispatcher fires the processor corresponding to the closest matching // rule and passes the context along Dispatcher disp = new DefaultRuleDispatcher(defaultProc, opRules, opWalkerCtx); GraphWalker ogw = new DefaultGraphWalker(disp); // Create a list of topop nodes ArrayList<Node> topNodes = new ArrayList<Node>(); topNodes.addAll(pctx.getTopOps().values()); ogw.startWalking(topNodes, null); }
@Override public PhysicalContext resolve(PhysicalContext pctx) throws SemanticException { Map<Rule, NodeProcessor> opRules = new LinkedHashMap<Rule, NodeProcessor>(); opRules.put(new RuleRegExp("R1", TableScanOperator.getOperatorName() + "%"), new TableScanProcessor()); opRules.put(new RuleRegExp("R2", GroupByOperator.getOperatorName() + "%.*" + FileSinkOperator.getOperatorName() + "%"), new FileSinkProcessor()); Dispatcher disp = new NullScanTaskDispatcher(pctx, opRules); GraphWalker ogw = new DefaultGraphWalker(disp); ArrayList<Node> topNodes = new ArrayList<Node>(); topNodes.addAll(pctx.getRootTasks()); ogw.startWalking(topNodes, null); return pctx; } }
/** * Transform the query tree. * * @param pctx the current parse context */ @Override public ParseContext transform(ParseContext pctx) throws SemanticException { // Make sure semijoin is not enabled. If it is, then do not remove the dynamic partition pruning predicates. if (!pctx.getConf().getBoolVar(HiveConf.ConfVars.TEZ_DYNAMIC_SEMIJOIN_REDUCTION)) { Map<Rule, NodeProcessor> opRules = new LinkedHashMap<Rule, NodeProcessor>(); opRules.put(new RuleRegExp("R1", TableScanOperator.getOperatorName() + "%" + FilterOperator.getOperatorName() + "%"), new FilterTransformer()); Dispatcher disp = new DefaultRuleDispatcher(null, opRules, null); GraphWalker ogw = new DefaultGraphWalker(disp); List<Node> topNodes = new ArrayList<Node>(); topNodes.addAll(pctx.getTopOps().values()); ogw.startWalking(topNodes, null); } return pctx; }
@Override public ParseContext transform(ParseContext pctx) throws SemanticException { // create a the context for walking operators SamplePrunerCtx samplePrunerCtx = new SamplePrunerCtx(pctx .getOpToSamplePruner()); Map<Rule, NodeProcessor> opRules = new LinkedHashMap<Rule, NodeProcessor>(); opRules.put(new RuleRegExp("R1", "(" + TableScanOperator.getOperatorName() + "%" + FilterOperator.getOperatorName() + "%" + FilterOperator.getOperatorName() + "%|" + TableScanOperator.getOperatorName() + "%" + FilterOperator.getOperatorName() + "%)"), getFilterProc()); // The dispatcher fires the processor corresponding to the closest matching // rule and passes the context along Dispatcher disp = new DefaultRuleDispatcher(getDefaultProc(), opRules, samplePrunerCtx); GraphWalker ogw = new DefaultGraphWalker(disp); // Create a list of topop nodes ArrayList<Node> topNodes = new ArrayList<Node>(); topNodes.addAll(pctx.getTopOps().values()); ogw.startWalking(topNodes, null); return pctx; }
@Override public PhysicalContext resolve(PhysicalContext pctx) throws SemanticException { Map<Rule, NodeProcessor> opRules = new LinkedHashMap<Rule, NodeProcessor>(); opRules.put(new RuleRegExp("R1", TableScanOperator.getOperatorName() + "%"), new TableScanProcessor()); opRules.put(new RuleRegExp("R2", GroupByOperator.getOperatorName() + "%.*" + FileSinkOperator.getOperatorName() + "%"), new FileSinkProcessor()); Dispatcher disp = new NullScanTaskDispatcher(pctx, opRules); GraphWalker ogw = new DefaultGraphWalker(disp); ArrayList<Node> topNodes = new ArrayList<Node>(); topNodes.addAll(pctx.getRootTasks()); ogw.startWalking(topNodes, null); return pctx; } }
@Override public PhysicalContext resolve(PhysicalContext pctx) throws SemanticException { Map<Rule, NodeProcessor> opRules = new LinkedHashMap<Rule, NodeProcessor>(); opRules.put(new RuleRegExp("R1", TableScanOperator.getOperatorName() + "%.*" + FilterOperator.getOperatorName() + "%"), new WhereFalseProcessor()); Dispatcher disp = new NullScanTaskDispatcher(pctx, opRules); GraphWalker ogw = new DefaultGraphWalker(disp); ArrayList<Node> topNodes = new ArrayList<Node>(); topNodes.addAll(pctx.getRootTasks()); ogw.startWalking(topNodes, null); opRules.clear(); opRules.put(new RuleRegExp("R1", TableScanOperator.getOperatorName()+ "%"),new TSMarker()); opRules.put(new RuleRegExp("R2", LimitOperator.getOperatorName()+ "%"), new Limit0Processor()); disp = new NullScanTaskDispatcher(pctx, opRules); ogw = new DefaultGraphWalker(disp); topNodes = new ArrayList<Node>(); topNodes.addAll(pctx.getRootTasks()); ogw.startWalking(topNodes, null); return pctx; }
/** * Transform the query tree. * * @param pctx the current parse context */ @Override public ParseContext transform(ParseContext pctx) throws SemanticException { // Make sure semijoin is not enabled. If it is, then do not remove the dynamic partition pruning predicates. if (!pctx.getConf().getBoolVar(HiveConf.ConfVars.TEZ_DYNAMIC_SEMIJOIN_REDUCTION)) { Map<Rule, NodeProcessor> opRules = new LinkedHashMap<Rule, NodeProcessor>(); opRules.put(new RuleRegExp("R1", TableScanOperator.getOperatorName() + "%" + FilterOperator.getOperatorName() + "%"), new FilterTransformer()); Dispatcher disp = new DefaultRuleDispatcher(null, opRules, null); GraphWalker ogw = new DefaultGraphWalker(disp); List<Node> topNodes = new ArrayList<Node>(); topNodes.addAll(pctx.getTopOps().values()); ogw.startWalking(topNodes, null); } return pctx; }
@Override public PhysicalContext resolve(PhysicalContext pctx) throws SemanticException { Map<Rule, NodeProcessor> opRules = new LinkedHashMap<>(); opRules.put( new RuleRegExp("R1", TableScanOperator.getOperatorName() + "%.*" + FilterOperator.getOperatorName() + "%"), new WhereFalseProcessor()); Dispatcher disp = new NullScanTaskDispatcher(pctx, opRules); GraphWalker ogw = new DefaultGraphWalker(disp); List<Node> topNodes = new ArrayList<>(pctx.getRootTasks()); ogw.startWalking(topNodes, null); opRules.clear(); opRules.put(new RuleRegExp("R1", TableScanOperator.getOperatorName() + "%"), new TSMarker()); opRules.put(new RuleRegExp("R2", LimitOperator.getOperatorName() + "%"), new Limit0Processor()); disp = new NullScanTaskDispatcher(pctx, opRules); ogw = new DefaultGraphWalker(disp); topNodes = new ArrayList<>(pctx.getRootTasks()); ogw.startWalking(topNodes, null); return pctx; }
@Override public ParseContext transform(ParseContext pctx) throws SemanticException { // create a the context for walking operators SamplePrunerCtx samplePrunerCtx = new SamplePrunerCtx(pctx .getOpToSamplePruner()); Map<Rule, NodeProcessor> opRules = new LinkedHashMap<Rule, NodeProcessor>(); opRules.put(new RuleRegExp("R1", "(" + TableScanOperator.getOperatorName() + "%" + FilterOperator.getOperatorName() + "%" + FilterOperator.getOperatorName() + "%|" + TableScanOperator.getOperatorName() + "%" + FilterOperator.getOperatorName() + "%)"), getFilterProc()); // The dispatcher fires the processor corresponding to the closest matching // rule and passes the context along Dispatcher disp = new DefaultRuleDispatcher(getDefaultProc(), opRules, samplePrunerCtx); GraphWalker ogw = new DefaultGraphWalker(disp); // Create a list of topop nodes ArrayList<Node> topNodes = new ArrayList<Node>(); topNodes.addAll(pctx.getTopOps().values()); ogw.startWalking(topNodes, null); return pctx; }
@Override public ParseContext transform(ParseContext pctx) throws SemanticException { // create a the context for walking operators List<PcrOpWalkerCtx.OpToDeleteInfo> opToRemove = new ArrayList<PcrOpWalkerCtx.OpToDeleteInfo>(); PcrOpWalkerCtx opWalkerCtx = new PcrOpWalkerCtx(pctx, opToRemove); Map<Rule, NodeProcessor> opRules = new LinkedHashMap<Rule, NodeProcessor>(); opRules.put(new RuleRegExp("R1", "(" + TableScanOperator.getOperatorName() + "%" + FilterOperator.getOperatorName() + "%)|(" + TableScanOperator.getOperatorName() + "%" + FilterOperator.getOperatorName() + "%" + FilterOperator.getOperatorName() + "%)"), PcrOpProcFactory.getFilterProc()); // The dispatcher fires the processor corresponding to the closest matching // rule and passes the context along Dispatcher disp = new DefaultRuleDispatcher(PcrOpProcFactory.getDefaultProc(), opRules, opWalkerCtx); GraphWalker ogw = new DefaultGraphWalker(disp); // Create a list of topop nodes ArrayList<Node> topNodes = new ArrayList<Node>(); topNodes.addAll(pctx.getTopOps().values()); ogw.startWalking(topNodes, null); for (PcrOpWalkerCtx.OpToDeleteInfo entry : opToRemove) { entry.getParent().removeChildAndAdoptItsChildren(entry.getOperator()); } return pctx; }
@Override public ParseContext transform(ParseContext pctx) throws SemanticException { // 0. We check the conditions to apply this transformation, // if we do not meet them we bail out final boolean cboEnabled = HiveConf.getBoolVar(pctx.getConf(), HiveConf.ConfVars.HIVE_CBO_ENABLED); final boolean returnPathEnabled = HiveConf.getBoolVar(pctx.getConf(), HiveConf.ConfVars.HIVE_CBO_RETPATH_HIVEOP); final boolean cboSucceeded = pctx.getContext().isCboSucceeded(); if(!(cboEnabled && returnPathEnabled && cboSucceeded)) { return pctx; } // 1. Initialize aux data structures this.pctx = pctx; this.aliasToOpInfo = new HashMap<String, Operator<? extends OperatorDesc>>(); // 2. Trigger transformation Map<Rule, NodeProcessor> opRules = new LinkedHashMap<Rule, NodeProcessor>(); opRules.put(new RuleRegExp("R1", JoinOperator.getOperatorName() + "%"), new JoinAnnotate()); opRules.put(new RuleRegExp("R2", TableScanOperator.getOperatorName() + "%"), new TableScanAnnotate()); Dispatcher disp = new DefaultRuleDispatcher(null, opRules, null); GraphWalker ogw = new ForwardWalker(disp); List<Node> topNodes = new ArrayList<Node>(); topNodes.addAll(pctx.getTopOps().values()); ogw.startWalking(topNodes, null); return pctx; }
@Override public ParseContext transform(ParseContext pctx) throws SemanticException { boolean enabled = false; String queryEngine = pctx.getConf().getVar(ConfVars.HIVE_EXECUTION_ENGINE); if (queryEngine.equals("tez") && pctx.getConf().getBoolVar(ConfVars.TEZ_DYNAMIC_PARTITION_PRUNING)) { enabled = true; } else if ((queryEngine.equals("spark") && pctx.getConf().isSparkDPPAny())) { enabled = true; } if (!enabled) { return pctx; } Map<Rule, NodeProcessor> opRules = new LinkedHashMap<Rule, NodeProcessor>(); opRules.put(new RuleRegExp("R1", "(" + TableScanOperator.getOperatorName() + "%" + ".*" + ReduceSinkOperator.getOperatorName() + "%" + JoinOperator.getOperatorName() + "%)"), new JoinSynthetic()); // The dispatcher fires the processor corresponding to the closest matching // rule and passes the context along SyntheticContext context = new SyntheticContext(pctx); Dispatcher disp = new DefaultRuleDispatcher(null, opRules, context); GraphWalker ogw = new PreOrderOnceWalker(disp); // Create a list of top op nodes List<Node> topNodes = new ArrayList<Node>(); topNodes.addAll(pctx.getTopOps().values()); ogw.startWalking(topNodes, null); return pctx; }
@Override public ParseContext transform(ParseContext pctx) throws SemanticException { // 0. We check the conditions to apply this transformation, // if we do not meet them we bail out final boolean cboEnabled = HiveConf.getBoolVar(pctx.getConf(), HiveConf.ConfVars.HIVE_CBO_ENABLED); final boolean returnPathEnabled = HiveConf.getBoolVar(pctx.getConf(), HiveConf.ConfVars.HIVE_CBO_RETPATH_HIVEOP); final boolean cboSucceeded = pctx.getContext().isCboSucceeded(); if(!(cboEnabled && returnPathEnabled && cboSucceeded)) { return pctx; } // 1. Initialize aux data structures this.pctx = pctx; this.aliasToOpInfo = new HashMap<String, Operator<? extends OperatorDesc>>(); // 2. Trigger transformation Map<Rule, NodeProcessor> opRules = new LinkedHashMap<Rule, NodeProcessor>(); opRules.put(new RuleRegExp("R1", JoinOperator.getOperatorName() + "%"), new JoinAnnotate()); opRules.put(new RuleRegExp("R2", TableScanOperator.getOperatorName() + "%"), new TableScanAnnotate()); Dispatcher disp = new DefaultRuleDispatcher(null, opRules, null); GraphWalker ogw = new ForwardWalker(disp); List<Node> topNodes = new ArrayList<Node>(); topNodes.addAll(pctx.getTopOps().values()); ogw.startWalking(topNodes, null); return pctx; }
@Override public ParseContext transform(ParseContext pctx) throws SemanticException { if (pctx.getFetchTask() != null || !pctx.getQueryProperties().isQuery() || pctx.getQueryProperties().isAnalyzeRewrite() || pctx.getQueryProperties().isCTAS() || pctx.getLoadFileWork().size() > 1 || !pctx.getLoadTableWork().isEmpty() // If getNameToSplitSample is not empty, at least one of the source // tables is being sampled and we can not optimize. || !pctx.getNameToSplitSample().isEmpty()) { return pctx; } String TS = TableScanOperator.getOperatorName() + "%"; String GBY = GroupByOperator.getOperatorName() + "%"; String RS = ReduceSinkOperator.getOperatorName() + "%"; String SEL = SelectOperator.getOperatorName() + "%"; String FS = FileSinkOperator.getOperatorName() + "%"; Map<Rule, NodeProcessor> opRules = new LinkedHashMap<Rule, NodeProcessor>(); opRules.put(new RuleRegExp("R1", TS + SEL + GBY + RS + GBY + SEL + FS), new MetaDataProcessor(pctx)); opRules.put(new RuleRegExp("R2", TS + SEL + GBY + RS + GBY + FS), new MetaDataProcessor(pctx)); NodeProcessorCtx soProcCtx = new StatsOptimizerProcContext(); Dispatcher disp = new DefaultRuleDispatcher(null, opRules, soProcCtx); GraphWalker ogw = new DefaultGraphWalker(disp); ArrayList<Node> topNodes = new ArrayList<Node>(); topNodes.addAll(pctx.getTopOps().values()); ogw.startWalking(topNodes, null); return pctx; }
@Test public void testPatternWithoutWildCardChar() { String patternStr = ReduceSinkOperator.getOperatorName() + "%" + SelectOperator.getOperatorName() + "%" + FileSinkOperator.getOperatorName() + "%"; RuleRegExp rule1 = new RuleRegExp("R1", patternStr); assertEquals(rule1.rulePatternIsValidWithoutWildCardChar(), true); assertEquals(rule1.rulePatternIsValidWithWildCardChar(), false); // positive test Stack<Node> ns1 = new Stack<Node>(); ns1.push(new TestNode(ReduceSinkOperator.getOperatorName())); ns1.push(new TestNode(SelectOperator.getOperatorName())); ns1.push(new TestNode(FileSinkOperator.getOperatorName())); try { assertEquals(rule1.cost(ns1), patternStr.length()); } catch (SemanticException e) { fail(e.getMessage()); } // negative test Stack<Node> ns2 = new Stack<Node>(); ns2.push(new TestNode(ReduceSinkOperator.getOperatorName())); ns1.push(new TestNode(TableScanOperator.getOperatorName())); ns1.push(new TestNode(FileSinkOperator.getOperatorName())); try { assertEquals(rule1.cost(ns2), -1); } catch (SemanticException e) { fail(e.getMessage()); } }