public Operator<?> getBranchingOp() { Operator<?> branchingOp = this; while (branchingOp != null) { if (branchingOp.getNumChild() > 1) { break; } else { branchingOp = branchingOp.getParentOperators().get(0); } } return branchingOp; }
private List<Operator> getRootOps(PlanMapper pmL) { List<Operator> ops = pmL.getAll(Operator.class); for (Iterator<Operator> iterator = ops.iterator(); iterator.hasNext();) { Operator o = iterator.next(); if (o.getNumChild() != 0) { iterator.remove(); } } return ops; }
private static void getAllOps(List<Operator> builder, Set<Operator> visited, Operator<?> op) { boolean added = visited.add(op); builder.add(op); if (!added) return; if (op.getNumChild() > 0) { List<Operator<?>> children = op.getChildOperators(); for (int i = 0; i < children.size(); i++) { getAllOps(builder, visited, children.get(i)); } } }
private static void getAllOps(List<Operator> builder, Set<Operator> visited, Operator<?> op) { boolean added = visited.add(op); builder.add(op); if (!added) return; if (op.getNumChild() > 0) { List<Operator<?>> children = op.getChildOperators(); for (int i = 0; i < children.size(); i++) { getAllOps(builder, visited, children.get(i)); } } }
/** * Returns a set containing all leaf operators from the operator tree in this work. * @return a set containing all leaf operators in this operator tree. */ public Set<Operator<? extends OperatorDesc>> getAllLeafOperators() { Set<Operator<?>> returnSet = new LinkedHashSet<Operator<?>>(); Set<Operator<?>> opSet = getAllRootOperators(); Stack<Operator<?>> opStack = new Stack<Operator<?>>(); // add all children opStack.addAll(opSet); while (!opStack.empty()) { Operator<?> op = opStack.pop(); if (op.getNumChild() == 0) { returnSet.add(op); } if (op.getChildOperators() != null) { opStack.addAll(op.getChildOperators()); } } return returnSet; }
private void removeSemiJoinEdges(Operator<?> op, OptimizeTezProcContext procCtx, Map<ReduceSinkOperator, TableScanOperator> sjToRemove) throws SemanticException { if(op instanceof ReduceSinkOperator && op.getNumChild() == 0) { Map<ReduceSinkOperator, SemiJoinBranchInfo> sjMap = procCtx.parseContext.getRsToSemiJoinBranchInfo(); if(sjMap.get(op) != null) { sjToRemove.put((ReduceSinkOperator)op, sjMap.get(op).getTsOp()); } } for(Operator<?> child:op.getChildOperators()) { removeSemiJoinEdges(child, procCtx, sjToRemove); } }
static boolean toString(StringBuilder builder, Set<String> visited, Operator<?> op, int start) { String name = op.toString(); boolean added = visited.add(name); if (start > 0) { builder.append("-"); start++; } builder.append(name); start += name.length(); if (added) { if (op.getNumChild() > 0) { List<Operator<?>> children = op.getChildOperators(); for (int i = 0; i < children.size(); i++) { if (i > 0) { builder.append('\n'); for (int j = 0; j < start; j++) { builder.append(' '); } } toString(builder, visited, children.get(i), start); } } return true; } return false; }
private static ExprWalkerInfo getChildWalkerInfo(Operator<?> current, OpWalkerInfo owi) throws SemanticException { if (current.getNumChild() == 0) { return null; } if (current.getNumChild() > 1) { // ppd for multi-insert query is not yet implemented // we assume that nothing can is pushed beyond this operator List<Operator<? extends OperatorDesc>> children = Lists.newArrayList(current.getChildOperators()); for (Operator<?> child : children) { ExprWalkerInfo childInfo = owi.getPrunedPreds(child); createFilter(child, childInfo, owi); } return null; } return owi.getPrunedPreds(current.getChildOperators().get(0)); }
public static void removeOperator(Operator<?> target, ParseContext context) { assert target.getNumParent() == 1 && target.getNumChild() == 1; removeOperator(target, target.getChildOperators().get(0), target.getParentOperators().get(0), context); }
private void removeOpRecursive(Operator<?> operator) { List<Operator<?>> parentOperators = new ArrayList<Operator<?>>(); for (Operator<?> op : operator.getParentOperators()) { parentOperators.add(op); } for (Operator<?> parentOperator : parentOperators) { Preconditions.checkArgument(parentOperator.getChildOperators().contains(operator), "AssertionError: parent of " + operator.getName() + " doesn't have it as child."); parentOperator.removeChild(operator); if (parentOperator.getNumChild() == 0) { removeOpRecursive(parentOperator); } } }
private static void removeBranch(Operator<?> currentOp, Set<Operator<?>> branchesOps, Set<Operator<?>> discardableOps) { if (currentOp.getNumChild() > 1) { for (Operator<?> childOp : currentOp.getChildOperators()) { if (!branchesOps.contains(childOp) && !discardableOps.contains(childOp)) { return; } } } branchesOps.add(currentOp); if (currentOp.getParentOperators() != null) { for (Operator<?> parentOp : currentOp.getParentOperators()) { removeBranch(parentOp, branchesOps, discardableOps); } } }
public static void removeOperator(Operator<?> target, ParseContext context) { assert target.getNumParent() == 1 && target.getNumChild() == 1; removeOperator(target, target.getChildOperators().get(0), target.getParentOperators().get(0), context); }
private Set<String> getQualifiedAliases(Operator<?> operator, OpWalkerInfo owi) { if (operator.getNumChild() != 1) { return null; } Operator<?> child = operator.getChildOperators().get(0); if (!(child instanceof JoinOperator || child instanceof LateralViewJoinOperator)) { return null; } if (operator instanceof ReduceSinkOperator && ((ReduceSinkOperator)operator).getInputAliases() != null) { String[] aliases = ((ReduceSinkOperator)operator).getInputAliases(); return new HashSet<String>(Arrays.asList(aliases)); } Set<String> includes = operator.getSchema().getTableNames(); if (includes.size() == 1 && includes.contains("")) { // Reduce sink of group by operator return null; } return includes; }
private Set<String> getQualifiedAliases(Operator<?> operator, OpWalkerInfo owi) { if (operator.getNumChild() != 1) { return null; } Operator<?> child = operator.getChildOperators().get(0); if (!(child instanceof JoinOperator || child instanceof LateralViewJoinOperator)) { return null; } if (operator instanceof ReduceSinkOperator && ((ReduceSinkOperator)operator).getInputAliases() != null) { String[] aliases = ((ReduceSinkOperator)operator).getInputAliases(); return new HashSet<String>(Arrays.asList(aliases)); } Set<String> includes = operator.getSchema().getTableNames(); if (includes.size() == 1 && includes.contains("")) { // Reduce sink of group by operator return null; } return includes; }
private void removeOpRecursive(Operator<?> operator) { List<Operator<?>> parentOperators = new ArrayList<Operator<?>>(); for (Operator<?> op : operator.getParentOperators()) { parentOperators.add(op); } for (Operator<?> parentOperator : parentOperators) { Preconditions.checkArgument(parentOperator.getChildOperators().contains(operator), "AssertionError: parent of " + operator.getName() + " doesn't have it as child."); parentOperator.removeChild(operator); if (parentOperator.getNumChild() == 0) { removeOpRecursive(parentOperator); } } }
private static ExprWalkerInfo getChildWalkerInfo(Operator<?> current, OpWalkerInfo owi) { if (current.getNumChild() == 0) { return null; } if (current.getNumChild() > 1) { // ppd for multi-insert query is not yet implemented // we assume that nothing can is pushed beyond this operator List<Operator<? extends OperatorDesc>> children = Lists.newArrayList(current.getChildOperators()); for (Operator<?> child : children) { ExprWalkerInfo childInfo = owi.getPrunedPreds(child); createFilter(child, childInfo, owi); } return null; } return owi.getPrunedPreds(current.getChildOperators().get(0)); }
/** * Remove operator from the tree, disconnecting it from its * parents and children. */ public static void removeOperator(Operator<?> op) { if (op.getNumParent() != 0) { List<Operator<? extends OperatorDesc>> allParent = Lists.newArrayList(op.getParentOperators()); for (Operator<?> parentOp : allParent) { parentOp.removeChild(op); } } if (op.getNumChild() != 0) { List<Operator<? extends OperatorDesc>> allChildren = Lists.newArrayList(op.getChildOperators()); for (Operator<?> childOp : allChildren) { childOp.removeParent(op); } } }
private boolean isConvertible(FetchData fetch, Operator<?> operator, Set<Operator<?>> traversed) { if (operator instanceof ReduceSinkOperator || operator instanceof CommonJoinOperator || operator instanceof ScriptOperator) { return false; } if (operator instanceof FilterOperator) { fetch.setFiltered(true); } if (!traversed.add(operator)) { return true; } if (operator.getNumChild() == 0) { if (operator instanceof FileSinkOperator) { fetch.fileSink = operator; return true; } return false; } for (Operator<?> child : operator.getChildOperators()) { if (!traversed.containsAll(child.getParentOperators())){ continue; } if (!isConvertible(fetch, child, traversed)) { return false; } } return true; }
@Override public Object process(Node nd, Stack<Node> stack, NodeProcessorCtx procCtx, Object... nodeOutputs) throws SemanticException { super.process(nd, stack, procCtx, nodeOutputs); Operator<?> operator = (Operator<?>) nd; OpWalkerInfo owi = (OpWalkerInfo) procCtx; if (operator.getNumChild() == 1 && operator.getChildOperators().get(0) instanceof JoinOperator) { if (HiveConf.getBoolVar(owi.getParseContext().getConf(), HiveConf.ConfVars.HIVEPPDRECOGNIZETRANSITIVITY)) { JoinOperator child = (JoinOperator) operator.getChildOperators().get(0); int targetPos = child.getParentOperators().indexOf(operator); applyFilterTransitivity(child, targetPos, owi); } } return null; }
public Object process(Node nd, Stack<Node> stack, NodeProcessorCtx procCtx, Object... nodeOutputs) throws SemanticException { super.process(nd, stack, procCtx, nodeOutputs); Operator<?> operator = (Operator<?>) nd; OpWalkerInfo owi = (OpWalkerInfo) procCtx; if (operator.getNumChild() == 1 && operator.getChildOperators().get(0) instanceof JoinOperator) { if (HiveConf.getBoolVar(owi.getParseContext().getConf(), HiveConf.ConfVars.HIVEPPDRECOGNIZETRANSITIVITY)) { JoinOperator child = (JoinOperator) operator.getChildOperators().get(0); int targetPos = child.getParentOperators().indexOf(operator); applyFilterTransitivity(child, targetPos, owi); } } return null; }