private void addCollectFileSinkUrisRules(Map<Rule, NodeProcessor> opRules, NodeProcessor np) { opRules.put(new RuleRegExp("R1", FileSinkOperator.getOperatorName() + ".*"), np); }
/** * This function returns the cost of the rule for the specified stack. Lower * the cost, the better the rule is matched * * @param stack * Node stack encountered so far * @return cost of the function * @throws SemanticException */ @Override public int cost(Stack<Node> stack) throws SemanticException { if (rulePatternIsValidWithoutWildCardChar()) { return costPatternWithoutWildCardChar(stack); } if (rulePatternIsValidWithWildCardChar()) { return costPatternWithWildCardChar(stack); } if (rulePatternIsValidWithORWildCardChar()) { return costPatternWithORWildCardChar(stack); } // If we reached here, either : // 1. patternWithWildCardChar and patternWithoutWildCardChar are both nulls. // 2. patternWithWildCardChar and patternWithoutWildCardChar are both not nulls. // This is an internal error and we should not let this happen, so throw an exception. throw new SemanticException ( "Rule pattern is invalid for " + getName() + " : patternWithWildCardChar = " + patternWithWildCardChar + " patternWithoutWildCardChar = " + patternWithoutWildCardChar); }
/** * The rule specified by the regular expression. Note that, the regular * expression is specified in terms of Node name. For eg: TS.*RS -> means * TableScan Node followed by anything any number of times followed by * ReduceSink * * @param ruleName * name of the rule * @param regExp * regular expression for the rule **/ public RuleRegExp(String ruleName, String regExp) { this.ruleName = ruleName; if (patternHasWildCardChar(regExp)) { if (patternHasOnlyWildCardChar(regExp, '|')) { this.patternWithWildCardChar = null; this.patternWithoutWildCardChar = null; this.patternORWildChar = regExp.split("\\|"); } else { this.patternWithWildCardChar = Pattern.compile(regExp); this.patternWithoutWildCardChar = null; this.patternORWildChar = null; } } else { this.patternWithWildCardChar = null; this.patternWithoutWildCardChar = regExp; this.patternORWildChar = null; } }
@Test public void testPatternWithWildCardChar() { RuleRegExp rule1 = new RuleRegExp("R1", "(" + TableScanOperator.getOperatorName() + "%" + FilterOperator.getOperatorName() + "%)|(" + TableScanOperator.getOperatorName() + "%" + FileSinkOperator.getOperatorName() + "%)"); assertEquals(rule1.rulePatternIsValidWithoutWildCardChar(), false); assertEquals(rule1.rulePatternIsValidWithWildCardChar(), true); ns2.push(new TestNode(FileSinkOperator.getOperatorName())); try { assertNotEquals(rule1.cost(ns1), -1); assertNotEquals(rule1.cost(ns2), -1); } catch (SemanticException e) { fail(e.getMessage()); ns3.push(new TestNode(FileSinkOperator.getOperatorName())); try { assertEquals(rule1.cost(ns3), -1); } catch (SemanticException e) { fail(e.getMessage());
@Test public void testPatternWithoutWildCardChar() { String patternStr = ReduceSinkOperator.getOperatorName() + "%" + SelectOperator.getOperatorName() + "%" + FileSinkOperator.getOperatorName() + "%"; RuleRegExp rule1 = new RuleRegExp("R1", patternStr); assertEquals(rule1.rulePatternIsValidWithoutWildCardChar(), true); assertEquals(rule1.rulePatternIsValidWithWildCardChar(), false); // positive test Stack<Node> ns1 = new Stack<Node>(); ns1.push(new TestNode(ReduceSinkOperator.getOperatorName())); ns1.push(new TestNode(SelectOperator.getOperatorName())); ns1.push(new TestNode(FileSinkOperator.getOperatorName())); try { assertEquals(rule1.cost(ns1), patternStr.length()); } catch (SemanticException e) { fail(e.getMessage()); } // negative test Stack<Node> ns2 = new Stack<Node>(); ns2.push(new TestNode(ReduceSinkOperator.getOperatorName())); ns1.push(new TestNode(TableScanOperator.getOperatorName())); ns1.push(new TestNode(FileSinkOperator.getOperatorName())); try { assertEquals(rule1.cost(ns2), -1); } catch (SemanticException e) { fail(e.getMessage()); } }
private void addReduceWorkRules(Map<Rule, NodeProcessor> opRules, NodeProcessor np) { opRules.put(new RuleRegExp("R1", GroupByOperator.getOperatorName() + ".*"), np); opRules.put(new RuleRegExp("R2", SelectOperator.getOperatorName() + ".*"), np); }
/** * This function returns the cost of the rule for the specified stack. Lower * the cost, the better the rule is matched * * @param stack * Node stack encountered so far * @return cost of the function * @throws SemanticException */ @Override public int cost(Stack<Node> stack) throws SemanticException { if (rulePatternIsValidWithoutWildCardChar()) { return costPatternWithoutWildCardChar(stack); } if (rulePatternIsValidWithWildCardChar()) { return costPatternWithWildCardChar(stack); } if (rulePatternIsValidWithORWildCardChar()) { return costPatternWithORWildCardChar(stack); } // If we reached here, either : // 1. patternWithWildCardChar and patternWithoutWildCardChar are both nulls. // 2. patternWithWildCardChar and patternWithoutWildCardChar are both not nulls. // This is an internal error and we should not let this happen, so throw an exception. throw new SemanticException ( "Rule pattern is invalid for " + getName() + " : patternWithWildCardChar = " + patternWithWildCardChar + " patternWithoutWildCardChar = " + patternWithoutWildCardChar); }
/** * The rule specified by the regular expression. Note that, the regular * expression is specified in terms of Node name. For eg: TS.*RS -> means * TableScan Node followed by anything any number of times followed by * ReduceSink * * @param ruleName * name of the rule * @param regExp * regular expression for the rule **/ public RuleRegExp(String ruleName, String regExp) { this.ruleName = ruleName; if (patternHasWildCardChar(regExp)) { if (patternHasOnlyWildCardChar(regExp, '|')) { this.patternWithWildCardChar = null; this.patternWithoutWildCardChar = null; this.patternORWildChar = regExp.split("\\|"); } else { this.patternWithWildCardChar = Pattern.compile(regExp); this.patternWithoutWildCardChar = null; this.patternORWildChar = null; } } else { this.patternWithWildCardChar = null; this.patternWithoutWildCardChar = regExp; this.patternORWildChar = null; } }
private void addMapWorkRules(Map<Rule, NodeProcessor> opRules, NodeProcessor np) { opRules.put(new RuleRegExp("R1", TableScanOperator.getOperatorName() + ".*" + FileSinkOperator.getOperatorName()), np); opRules.put(new RuleRegExp("R2", TableScanOperator.getOperatorName() + ".*" + ReduceSinkOperator.getOperatorName()), np); }
private void addMapWorkRules(Map<Rule, NodeProcessor> opRules, NodeProcessor np) { opRules.put(new RuleRegExp("R1", TableScanOperator.getOperatorName() + ".*" + FileSinkOperator.getOperatorName()), np); opRules.put(new RuleRegExp("R2", TableScanOperator.getOperatorName() + ".*" + ReduceSinkOperator.getOperatorName()), np); }
private static void collectDynamicValuePredicates(ExprNodeDesc pred, NodeProcessorCtx ctx) throws SemanticException { // create a walker which walks the tree in a DFS manner while maintaining // the operator stack. The dispatcher // generates the plan from the operator tree Map<Rule, NodeProcessor> exprRules = new LinkedHashMap<Rule, NodeProcessor>(); exprRules.put(new RuleRegExp("R1", ExprNodeDynamicValueDesc.class.getName() + "%"), new DynamicValuePredicateProc()); Dispatcher disp = new DefaultRuleDispatcher(null, exprRules, ctx); GraphWalker egw = new DefaultGraphWalker(disp); List<Node> startNodes = new ArrayList<Node>(); startNodes.add(pred); egw.startWalking(startNodes, null); }
public static Map<Node, Object> collectDynamicPruningConditions(ExprNodeDesc pred, NodeProcessorCtx ctx) throws SemanticException { // create a walker which walks the tree in a DFS manner while maintaining // the operator stack. The dispatcher // generates the plan from the operator tree Map<Rule, NodeProcessor> exprRules = new LinkedHashMap<Rule, NodeProcessor>(); exprRules.put(new RuleRegExp("R1", ExprNodeDynamicListDesc.class.getName() + "%"), new DynamicPartitionPrunerProc()); // The dispatcher fires the processor corresponding to the closest matching // rule and passes the context along Dispatcher disp = new DefaultRuleDispatcher(null, exprRules, ctx); GraphWalker egw = new DefaultGraphWalker(disp); List<Node> startNodes = new ArrayList<Node>(); startNodes.add(pred); HashMap<Node, Object> outputMap = new HashMap<Node, Object>(); egw.startWalking(startNodes, outputMap); return outputMap; } }
private static void collectDynamicValuePredicates(ExprNodeDesc pred, NodeProcessorCtx ctx) throws SemanticException { // create a walker which walks the tree in a DFS manner while maintaining // the operator stack. The dispatcher // generates the plan from the operator tree Map<Rule, NodeProcessor> exprRules = new LinkedHashMap<Rule, NodeProcessor>(); exprRules.put(new RuleRegExp("R1", ExprNodeDynamicValueDesc.class.getName() + "%"), new DynamicValuePredicateProc()); Dispatcher disp = new DefaultRuleDispatcher(null, exprRules, ctx); GraphWalker egw = new DefaultGraphWalker(disp); List<Node> startNodes = new ArrayList<Node>(); startNodes.add(pred); egw.startWalking(startNodes, null); } }
@Override public ParseContext transform(ParseContext pctx) throws SemanticException { // 1. Trigger transformation Map<Rule, NodeProcessor> opRules = new LinkedHashMap<Rule, NodeProcessor>(); opRules.put(new RuleRegExp("R1", FilterOperator.getOperatorName() + "%"), new FilterTransformer()); Dispatcher disp = new DefaultRuleDispatcher(null, opRules, null); GraphWalker ogw = new ForwardWalker(disp); List<Node> topNodes = new ArrayList<Node>(); topNodes.addAll(pctx.getTopOps().values()); ogw.startWalking(topNodes, null); return pctx; }
@Override public ParseContext transform(ParseContext pctx) throws SemanticException { // 1. Trigger transformation Map<Rule, NodeProcessor> opRules = new LinkedHashMap<Rule, NodeProcessor>(); opRules.put(new RuleRegExp("R1", FilterOperator.getOperatorName() + "%"), new StructInTransformer()); Dispatcher disp = new DefaultRuleDispatcher(null, opRules, null); GraphWalker ogw = new ForwardWalker(disp); List<Node> topNodes = new ArrayList<Node>(); topNodes.addAll(pctx.getTopOps().values()); ogw.startWalking(topNodes, null); return pctx; }
private Map<Node, Object> collectDynamicPruningConditions(ExprNodeDesc pred, NodeProcessorCtx ctx) throws SemanticException { // create a walker which walks the tree in a DFS manner while maintaining // the operator stack. The dispatcher // generates the plan from the operator tree Map<Rule, NodeProcessor> exprRules = new LinkedHashMap<Rule, NodeProcessor>(); exprRules.put(new RuleRegExp("R1", ExprNodeDynamicListDesc.class.getName() + "%"), new DynamicPartitionPrunerProc()); // The dispatcher fires the processor corresponding to the closest matching // rule and passes the context along Dispatcher disp = new DefaultRuleDispatcher(null, exprRules, ctx); GraphWalker egw = new DefaultGraphWalker(disp); List<Node> startNodes = new ArrayList<Node>(); startNodes.add(pred); HashMap<Node, Object> outputMap = new HashMap<Node, Object>(); egw.startWalking(startNodes, outputMap); return outputMap; }
@Override public ParseContext transform(ParseContext pctx) throws SemanticException { // 1. We apply the transformation String SEL = SelectOperator.getOperatorName(); String FIL = FilterOperator.getOperatorName(); Map<Rule, NodeProcessor> opRules = new LinkedHashMap<Rule, NodeProcessor>(); opRules.put(new RuleRegExp("R1", SEL + "%" + SEL + "%"), new SelectDedup(pctx)); opRules.put(new RuleRegExp("R2", FIL + "%" + FIL + "%"), new FilterDedup()); Dispatcher disp = new DefaultRuleDispatcher(null, opRules, null); GraphWalker ogw = new DefaultGraphWalker(disp); List<Node> topNodes = new ArrayList<Node>(); topNodes.addAll(pctx.getTopOps().values()); ogw.startWalking(topNodes, null); return pctx; }
@Override public ParseContext transform(ParseContext pCtx) throws SemanticException { // create a walker which walks the tree in a DFS manner while maintaining the // operator stack. The dispatcher generates the plan from the operator tree Map<Rule, NodeProcessor> opRules = new LinkedHashMap<Rule, NodeProcessor>(); String FS = FileSinkOperator.getOperatorName() + "%"; opRules.put(new RuleRegExp("Sorted Dynamic Partition", FS), getSortDynPartProc(pCtx)); Dispatcher disp = new DefaultRuleDispatcher(null, opRules, null); GraphWalker ogw = new DefaultGraphWalker(disp); ArrayList<Node> topNodes = new ArrayList<Node>(); topNodes.addAll(pCtx.getTopOps().values()); ogw.startWalking(topNodes, null); return pCtx; }
@Override public ParseContext transform(ParseContext pCtx) throws SemanticException { // create a walker which walks the tree in a DFS manner while maintaining the // operator stack. The dispatcher generates the plan from the operator tree Map<Rule, NodeProcessor> opRules = new LinkedHashMap<Rule, NodeProcessor>(); String FS = FileSinkOperator.getOperatorName() + "%"; opRules.put(new RuleRegExp("Sorted Dynamic Partition Time Granularity", FS), getSortDynPartProc(pCtx)); Dispatcher disp = new DefaultRuleDispatcher(null, opRules, null); GraphWalker ogw = new DefaultGraphWalker(disp); ArrayList<Node> topNodes = new ArrayList<Node>(); topNodes.addAll(pCtx.getTopOps().values()); ogw.startWalking(topNodes, null); return pCtx; }
@Override public ParseContext transform(ParseContext pctx) throws SemanticException { // 1. Trigger transformation Map<Rule, NodeProcessor> opRules = new LinkedHashMap<Rule, NodeProcessor>(); opRules.put(new RuleRegExp("R1", FilterOperator.getOperatorName() + "%"), new StructInTransformer()); Dispatcher disp = new DefaultRuleDispatcher(null, opRules, null); GraphWalker ogw = new ForwardWalker(disp); List<Node> topNodes = new ArrayList<Node>(); topNodes.addAll(pctx.getTopOps().values()); ogw.startWalking(topNodes, null); return pctx; }