/** * Returns true if des is a descendant of ans (ancestor) */ private boolean isDescendant(Node ans, Node des) { if (ans.getChildren() == null) { return false; } for (Node c : ans.getChildren()) { if (c == des) { return true; } if (isDescendant(c, des)) { return true; } } return false; }
/** * Returns dispatch result */ public <T> T dispatchAndReturn(Node nd, Stack<Node> ndStack) throws SemanticException { Object[] nodeOutputs = null; if (nd.getChildren() != null) { nodeOutputs = new Object[nd.getChildren().size()]; int i = 0; for (Node child : nd.getChildren()) { nodeOutputs[i++] = retMap.get(child); } } Object retVal = dispatcher.dispatch(nd, ndStack, nodeOutputs); retMap.put(nd, retVal); return (T) retVal; }
/** * Returns true if des is a descendant of ans (ancestor) */ private boolean isDescendant(Node ans, Node des) { if (ans.getChildren() == null) { return false; } for (Node c : ans.getChildren()) { if (c == des) { return true; } if (isDescendant(c, des)) { return true; } } return false; }
/** * Returns dispatch result */ public <T> T dispatchAndReturn(Node nd, Stack<Node> ndStack) throws SemanticException { Object[] nodeOutputs = null; if (nd.getChildren() != null) { nodeOutputs = new Object[nd.getChildren().size()]; int i = 0; for (Node child : nd.getChildren()) { nodeOutputs[i++] = retMap.get(child); } } Object retVal = dispatcher.dispatch(nd, ndStack, nodeOutputs); retMap.put(nd, retVal); return (T) retVal; }
private static boolean isNullOpPresentInAllBranches(TableScanOperator ts, Node causeOfNullNode) { Queue<Node> middleNodes = new ArrayDeque<>(); middleNodes.add(ts); while (!middleNodes.isEmpty()) { Node curNode = middleNodes.remove(); List<? extends Node> curChd = curNode.getChildren(); for (Node chd: curChd) { List<? extends Node> children = chd.getChildren(); if (CollectionUtils.isEmpty(children) || chd == causeOfNullNode) { // If there is an end node that not the limit0/wherefalse.. if (chd != causeOfNullNode) { return false; } } else { middleNodes.add(chd); } } } return true; }
/** * Dispatch the current operator. * * @param nd * node being walked * @param ndStack * stack of nodes encountered * @throws SemanticException */ public void dispatch(Node nd, Stack<Node> ndStack,TaskGraphWalkerContext walkerCtx) throws SemanticException { Object[] nodeOutputs = null; if (nd.getChildren() != null) { nodeOutputs = new Object[nd.getChildren().size()+1]; nodeOutputs[0] = walkerCtx; int i = 1; for (Node child : nd.getChildren()) { nodeOutputs[i++] = retMap.get(child); } }else{ nodeOutputs = new Object[1]; nodeOutputs[0] = walkerCtx; } Object retVal = dispatcher.dispatch(nd, ndStack, nodeOutputs); retMap.put(nd, retVal); }
private static <T> List<Collection<? extends Node>> iterate(List<Collection<? extends Node>> listNodes, Class<T> clazz, Function<T> function, Set<Node> visited) { List<Collection<? extends Node>> childListNodes = new ArrayList<>(); for (Collection<? extends Node> nodes : listNodes) { for (Node node : nodes) { if (!visited.add(node)) { continue; } if (clazz.isInstance(node)) { function.apply(clazz.cast(node)); } if (node.getChildren() != null) { childListNodes.add(node.getChildren()); } } } return childListNodes; }
private static <T> void iterate(Node node, Class<T> clazz, Function<T> function, Set<Node> visited) { if (!visited.add(node)) { return; } if (clazz.isInstance(node)) { function.apply(clazz.cast(node)); } if (node.getChildren() != null) { for (Node child : node.getChildren()) { iterate(child, clazz, function, visited); } } }
/** * Walk the current operator and its descendants. * * @param nd * current operator in the graph * @throws SemanticException */ @Override public void walk(Node nd) throws SemanticException { opStack.push(nd); dispatch(nd, opStack); // The rule has been applied, we bail out if (retMap.get(nd) != null) { opStack.pop(); return; } // move all the children to the front of queue if (nd.getChildren() != null) { for (Node n : nd.getChildren()) { walk(n); } } opStack.pop(); }
@Override protected void walk(Node nd) throws SemanticException { List<Node> parents = ((Operator) nd).getParentOperators(); if ((parents == null) || getDispatchedList().containsAll(parents)) { opStack.push(nd); // all children are done or no need to walk the children dispatch(nd, opStack); opStack.pop(); } else { toWalk.removeAll(parents); toWalk.add(0, nd); toWalk.addAll(0, parents); return; } // move all the children to the front of queue List<? extends Node> children = nd.getChildren(); if (children != null) { toWalk.removeAll(children); toWalk.addAll(children); } } }
/** * Walk the given operator. * * @param nd * operator being walked */ @Override protected void walk(Node nd) throws SemanticException { List<? extends Node> children = nd.getChildren(); // maintain the stack of operators encountered opStack.push(nd); Boolean result = dispatchAndReturn(nd, opStack); // kids of reduce sink operator or mapjoin operators merged into root task // need not be traversed again if (children == null || result == Boolean.FALSE) { opStack.pop(); return; } // move all the children to the front of queue for (Node ch : children) { walk(ch); } // done with this operator opStack.pop(); } }
/** * Walk the given operator. * * @param nd * operator being walked */ @Override protected void walk(Node nd) throws SemanticException { List<? extends Node> children = nd.getChildren(); // maintain the stack of operators encountered opStack.push(nd); Boolean skip = dispatchAndReturn(nd, opStack); if (skip == null || !skip) { // move all the children to the front of queue for (Node ch : children) { walk(ch); } } // done with this operator opStack.pop(); } }
/** * Walk the given operator. * * @param nd * operator being walked */ @Override protected void walk(Node nd) throws SemanticException { List<? extends Node> children = nd.getChildren(); // maintain the stack of operators encountered opStack.push(nd); Boolean skip = dispatchAndReturn(nd, opStack); if (skip == null || !skip) { // move all the children to the front of queue for (Node ch : children) { walk(ch); } } // done with this operator opStack.pop(); } }
/** * Walk the given operator. * * @param nd operator being walked */ @Override protected void walk(Node nd) throws SemanticException { List<? extends Node> children = nd.getChildren(); // maintain the stack of operators encountered opStack.push(nd); Boolean skip = dispatchAndReturn(nd, opStack); // save some positional state Operator<? extends OperatorDesc> currentRoot = ctx.currentRootOperator; Operator<? extends OperatorDesc> parentOfRoot = ctx.parentOfRoot; BaseWork preceedingWork = ctx.preceedingWork; if (skip == null || !skip) { // move all the children to the front of queue for (Node ch : children) { // and restore the state before walking each child ctx.currentRootOperator = currentRoot; ctx.parentOfRoot = parentOfRoot; ctx.preceedingWork = preceedingWork; walk(ch); } } // done with this operator opStack.pop(); }
/** * walk the current operator and its descendants. * * @param nd * current operator in the graph * @throws SemanticException */ @Override protected void walk(Node nd) throws SemanticException { if (opStack.empty() || nd != opStack.peek()) { opStack.push(nd); } if (allParentsDispatched(nd)) { // all children are done or no need to walk the children if (!getDispatchedList().contains(nd)) { toWalk.addAll(nd.getChildren()); dispatch(nd, opStack); } opStack.pop(); return; } // add children, self to the front of the queue in that order toWalk.add(0, nd); addAllParents(nd); } }
@Override public Object process(Node nd, Stack<Node> stack, NodeProcessorCtx procCtx, Object... nodeOutputs) throws SemanticException { LOG.info("Processing for " + nd.getName() + "(" + ((Operator) nd).getIdentifier() + ")"); OpWalkerInfo owi = (OpWalkerInfo) procCtx; // The lateral view forward operator has 2 children, a SELECT(*) and // a SELECT(cols) (for the UDTF operator) The child at index 0 is the // SELECT(*) because that's the way that the DAG was constructed. We // only want to get the predicates from the SELECT(*). ExprWalkerInfo childPreds = owi .getPrunedPreds((Operator<? extends OperatorDesc>) nd.getChildren() .get(0)); owi.putPrunedPreds((Operator<? extends OperatorDesc>) nd, childPreds); return null; }
/** * Walk the current operator and its descendants. * * @param nd * current operator in the graph * @throws SemanticException */ @Override protected void walk(Node nd) throws SemanticException { opStack.push(nd); dispatch(nd, opStack); // move all the children to the front of queue if (nd.getChildren() != null) { for (Node n : nd.getChildren()) { walk(n); } } else if (nd instanceof ConditionalTask) { for (Task n : ((ConditionalTask) nd).getListTasks()) { if (n.getParentTasks() == null || n.getParentTasks().isEmpty()) { walk(n); } } } opStack.pop(); } }
protected ExprWalkerInfo mergeChildrenPred(Node nd, OpWalkerInfo owi, Set<String> excludedAliases, boolean ignoreAliases) throws SemanticException { if (nd.getChildren() == null) { return null; } Operator<? extends OperatorDesc> op = (Operator<? extends OperatorDesc>)nd; ExprWalkerInfo ewi = new ExprWalkerInfo(); for (Operator<? extends OperatorDesc> child : op.getChildOperators()) { ExprWalkerInfo childPreds = owi.getPrunedPreds(child); if (childPreds == null) { continue; } for (Entry<String, List<ExprNodeDesc>> e : childPreds .getFinalCandidates().entrySet()) { if (ignoreAliases || excludedAliases == null || !excludedAliases.contains(e.getKey()) || e.getKey() == null) { ewi.addPushDowns(e.getKey(), e.getValue()); logExpr(nd, ewi); } } } return ewi; } }
/** * Walk the current operator and its descendants. * * @param nd * current operator in the graph * @throws SemanticException */ @Override protected void walk(Node nd) throws SemanticException { opStack.push(nd); dispatch(nd, opStack); // move all the children to the front of queue if (nd.getChildren() != null) { for (Node n : nd.getChildren()) { walk(n); } } else if (nd instanceof ConditionalTask) { for (Task n : ((ConditionalTask) nd).getListTasks()) { if (n.getParentTasks() == null || n.getParentTasks().isEmpty()) { walk(n); } } } opStack.pop(); } }
@Override public Object process(Node nd, Stack<Node> stack, NodeProcessorCtx procCtx, Object... nodeOutputs) throws SemanticException { LOG.info("Processing for " + nd.getName() + "(" + ((Operator) nd).getIdentifier() + ")"); OpWalkerInfo owi = (OpWalkerInfo) procCtx; // The lateral view forward operator has 2 children, a SELECT(*) and // a SELECT(cols) (for the UDTF operator) The child at index 0 is the // SELECT(*) because that's the way that the DAG was constructed. We // only want to get the predicates from the SELECT(*). ExprWalkerInfo childPreds = owi .getPrunedPreds((Operator<? extends OperatorDesc>) nd.getChildren() .get(0)); owi.putPrunedPreds((Operator<? extends OperatorDesc>) nd, childPreds); return null; }