Refine search
/** * Return true if contain branch otherwise return false */ public static boolean isInBranch(SparkPartitionPruningSinkOperator op) { Operator<?> curr = op; while (curr.getChildOperators().size() <= 1) { if (curr.getParentOperators() == null || curr.getParentOperators().isEmpty()) { return false; } curr = curr.getParentOperators().get(0); } return true; }
/*** * Given group by operator on reduce side, this tries to get to the group by on map side (partial/merge). * @param reduceSideGbOp Make sure this is group by side reducer * @return map side gb if any, else null */ public static GroupByOperator findMapSideGb(final GroupByOperator reduceSideGbOp) { Operator<? extends OperatorDesc> parentOp = reduceSideGbOp; while(parentOp.getParentOperators() != null && parentOp.getParentOperators().size() > 0) { if(parentOp.getParentOperators().size() > 1) { return null; } parentOp = parentOp.getParentOperators().get(0); if(parentOp instanceof GroupByOperator) { return (GroupByOperator)parentOp; } } return null; } }
protected void getListTopOps( Operator<? extends OperatorDesc> op, List<TableScanOperator> topOps) { if ((op.getParentOperators() == null) || (op.getParentOperators().isEmpty())) { return; } for (Operator<? extends OperatorDesc> parentOp : op.getParentOperators()) { if (parentOp instanceof TableScanOperator) { topOps.add((TableScanOperator) parentOp); } else if (parentOp instanceof CommonJoinOperator) { topOps.add(null); } else { getListTopOps(parentOp, topOps); } } }
private void insertAnalyzePipeline(String command, boolean rewritten) throws SemanticException { // 1. Based on the statement, generate the selectOperator Operator<?> selOp = null; try { selOp = genSelOp(command, rewritten, origCtx); } catch (IOException | ParseException e) { throw new SemanticException(e); } // 2. attach this SEL to the operator right before FS op.getChildOperators().add(selOp); selOp.getParentOperators().clear(); selOp.getParentOperators().add(op); // 3. address the colExp, colList, etc for the SEL try { replaceSelectOperatorProcess((SelectOperator)selOp, op); } catch (HiveException e) { throw new SemanticException(e); } }
public static ListSinkOperator replaceFSwithLS(Operator<?> fileSink, String nullFormat) { ListSinkDesc desc = new ListSinkDesc(nullFormat); ListSinkOperator sink = (ListSinkOperator) OperatorFactory.get( fileSink.getCompilationOpContext(), desc); sink.setParentOperators(new ArrayList<Operator<? extends OperatorDesc>>()); Operator<? extends OperatorDesc> parent = fileSink.getParentOperators().get(0); sink.getParentOperators().add(parent); parent.replaceChild(fileSink, sink); fileSink.setParentOperators(null); return sink; } }
if (parents.size() != 1 || parents.get(0) instanceof LateralViewForwardOperator) { Operator<? extends OperatorDesc> parent = parents.get(0); if (parent instanceof ReduceSinkOperator && Iterators.any(sel.getChildOperators().iterator(), Predicates.instanceOf(ReduceSinkOperator.class))) { return null; if ((curParent instanceof FilterOperator) && curParent.getParentOperators() != null) { ancestorList.addAll(curParent.getParentOperators());
public Operator<?> getBranchingOp() { Operator<?> branchingOp = this; while (branchingOp != null) { if (branchingOp.getNumChild() > 1) { break; } else { branchingOp = branchingOp.getParentOperators().get(0); } } return branchingOp; }
@Override protected void walk(Node nd) throws SemanticException { List<Node> parents = ((Operator) nd).getParentOperators(); if ((parents == null) || getDispatchedList().containsAll(parents)) { opStack.push(nd); // all children are done or no need to walk the children dispatch(nd, opStack); opStack.pop(); } else { toWalk.removeAll(parents); toWalk.add(0, nd); toWalk.addAll(0, parents); return; } // move all the children to the front of queue List<? extends Node> children = nd.getChildren(); if (children != null) { toWalk.removeAll(children); toWalk.addAll(children); } } }
public void removeChild(Operator<? extends OperatorDesc> child) { int childIndex = childOperators.indexOf(child); assert childIndex != -1; if (childOperators.size() == 1) { setChildOperators(null); } else { childOperators.remove(childIndex); } int parentIndex = child.getParentOperators().indexOf(this); assert parentIndex != -1; if (child.getParentOperators().size() == 1) { child.setParentOperators(null); } else { child.getParentOperators().remove(parentIndex); } }
public static void connectOperators( Operator<? extends OperatorDesc> operator, Operator<? extends OperatorDesc> childOperator) throws HiveException { List<Operator<? extends OperatorDesc>> newParentOperators = newOperatorList(); newParentOperators.addAll(childOperator.getParentOperators()); newParentOperators.add(operator); childOperator.setParentOperators(newParentOperators); List<Operator<? extends OperatorDesc>> newChildOperators = newOperatorList(); newChildOperators.addAll(operator.getChildOperators()); newChildOperators.add(childOperator); operator.setChildOperators(newChildOperators); }
/** * Remove a child and add all of the child's children to the location of the child * * @param child If this operator is not the only parent of the child. There can be unpredictable result. * @throws SemanticException */ public void removeChildAndAdoptItsChildren( Operator<? extends OperatorDesc> child) throws SemanticException { int childIndex = childOperators.indexOf(child); if (childIndex == -1) { throw new SemanticException( "Exception when trying to remove partition predicates: fail to find child from parent"); } childOperators.remove(childIndex); if (child.getChildOperators() != null && child.getChildOperators().size() > 0) { childOperators.addAll(childIndex, child.getChildOperators()); } for (Operator<? extends OperatorDesc> gc : child.getChildOperators()) { List<Operator<? extends OperatorDesc>> parents = gc.getParentOperators(); int index = parents.indexOf(child); if (index == -1) { throw new SemanticException( "Exception when trying to remove partition predicates: fail to find parent from child"); } parents.set(index, this); } }
private static void gatherReduceSinkOpsByInput(Multimap<Operator<?>, ReduceSinkOperator> parentToRsOps, Set<Operator<?>> visited, Set<Operator<?>> ops) { for (Operator<?> op : ops) { // If the RS has other RS siblings, we will add it to be considered in next iteration if (op instanceof ReduceSinkOperator && !visited.contains(op)) { Operator<?> parent = op.getParentOperators().get(0); Set<ReduceSinkOperator> s = new LinkedHashSet<>(); for (Operator<?> c : parent.getChildOperators()) { if (c instanceof ReduceSinkOperator) { s.add((ReduceSinkOperator) c); visited.add(c); } } if (s.size() > 1) { parentToRsOps.putAll(parent, s); } } } }
private static void removeOperator(Operator<? extends OperatorDesc> operator) { // since removeParent/removeChild updates the childOperators and parentOperators list in place // we need to make a copy of list to iterator over them List<Operator<? extends OperatorDesc>> children = new ArrayList<>(operator.getChildOperators()); List<Operator<? extends OperatorDesc>> parents = new ArrayList<>(operator.getParentOperators()); for (Operator<? extends OperatorDesc> parent : parents) { parent.getChildOperators().addAll(children); parent.removeChild(operator); } for (Operator<? extends OperatorDesc> child : children) { child.getParentOperators().addAll(parents); child.removeParent(operator); } }
private static void removeBranch(Operator<?> currentOp, Set<Operator<?>> branchesOps, Set<Operator<?>> discardableOps) { if (currentOp.getNumChild() > 1) { for (Operator<?> childOp : currentOp.getChildOperators()) { if (!branchesOps.contains(childOp) && !discardableOps.contains(childOp)) { return; } } } branchesOps.add(currentOp); if (currentOp.getParentOperators() != null) { for (Operator<?> parentOp : currentOp.getParentOperators()) { removeBranch(parentOp, branchesOps, discardableOps); } } }
/** * Remove the branch that contains the specified operator. Do nothing if there's no branching, * i.e. all the upstream operators have only one child. */ public static void removeBranch(SparkPartitionPruningSinkOperator op) { Operator<?> child = op; Operator<?> curr = op; while (curr.getChildOperators().size() <= 1) { child = curr; if (curr.getParentOperators() == null || curr.getParentOperators().isEmpty()) { return; } curr = curr.getParentOperators().get(0); } curr.removeChild(child); }
/** * Get the table scan. */ private TableScanOperator getTableScanOperator( ParseContext parseContext, Operator<? extends OperatorDesc> op, List<TableScanOperator> tableScanOpsForJoin) { while (true) { if (op instanceof TableScanOperator) { TableScanOperator tsOp = (TableScanOperator)op; if (tableScanOpsForJoin.contains(tsOp)) { return tsOp; } } if ((op.getParentOperators() == null) || (op.getParentOperators().isEmpty()) || (op.getParentOperators().size() > 1)) { return null; } op = op.getParentOperators().get(0); } }
protected void getListTopOps( Operator<? extends OperatorDesc> op, List<TableScanOperator> topOps) { if ((op.getParentOperators() == null) || (op.getParentOperators().isEmpty())) { return; } for (Operator<? extends OperatorDesc> parentOp : op.getParentOperators()) { if (parentOp instanceof TableScanOperator) { topOps.add((TableScanOperator) parentOp); } else if (parentOp instanceof CommonJoinOperator) { topOps.add(null); } else { getListTopOps(parentOp, topOps); } } }
private void removeOpRecursive(Operator<?> operator) { List<Operator<?>> parentOperators = new ArrayList<Operator<?>>(); for (Operator<?> op : operator.getParentOperators()) { parentOperators.add(op); } for (Operator<?> parentOperator : parentOperators) { Preconditions.checkArgument(parentOperator.getChildOperators().contains(operator), "AssertionError: parent of " + operator.getName() + " doesn't have it as child."); parentOperator.removeChild(operator); if (parentOperator.getNumChild() == 0) { removeOpRecursive(parentOperator); } } }
public static ListSinkOperator replaceFSwithLS(Operator<?> fileSink, String nullFormat) { ListSinkDesc desc = new ListSinkDesc(nullFormat); ListSinkOperator sink = (ListSinkOperator) OperatorFactory.get( fileSink.getCompilationOpContext(), desc); sink.setParentOperators(new ArrayList<Operator<? extends OperatorDesc>>()); Operator<? extends OperatorDesc> parent = fileSink.getParentOperators().get(0); sink.getParentOperators().add(parent); parent.replaceChild(fileSink, sink); fileSink.setParentOperators(null); return sink; } }
private static boolean isStrictEqualityNeeded(ReduceSinkOperator cRS, ReduceSinkOperator pRS) { Operator<? extends OperatorDesc> parent = cRS.getParentOperators().get(0); while (parent != pRS) { assert parent.getNumParent() == 1; if (parent instanceof PTFOperator) { return true; } parent = parent.getParentOperators().get(0); } return false; }