public static <T> Set<T> findOperators(Collection<Operator<?>> starts, Class<T> clazz) { Set<T> found = new HashSet<T>(); for (Operator<?> start : starts) { if (start == null) { continue; } findOperators(start, clazz, found); } return found; }
public static <T> Set<T> findOperatorsUpstream(Collection<Operator<?>> starts, Class<T> clazz) { Set<T> found = new HashSet<T>(); for (Operator<?> start : starts) { findOperatorsUpstream(start, clazz, found); } return found; }
public static <T> T findSingleOperatorUpstreamJoinAccounted(Operator<?> start, Class<T> clazz) { Set<T> found = findOperatorsUpstreamJoinAccounted(start, clazz, new HashSet<T>()); return found.size() >= 1 ? found.iterator().next(): null; }
public static void setChildrenCollector(List<Operator<? extends OperatorDesc>> childOperators, Map<String, OutputCollector> outMap) { if (childOperators == null) { return; } for (Operator<? extends OperatorDesc> op : childOperators) { if (op.getIsReduceSink()) { String outputName = op.getReduceOutputName(); if (outMap.containsKey(outputName)) { LOG.info("Setting output collector: " + op + " --> " + outputName); op.setOutputCollector(outMap.get(outputName)); } } else { setChildrenCollector(op.getChildOperators(), outMap); } } }
l4j.info("Memory available for operators set to {}", LlapUtil.humanReadableByteCount(memoryAvailableToTask)); OperatorUtils.setMemoryAvailable(reducer.getChildOperators(), memoryAvailableToTask); OperatorUtils.setChildrenCollector(children, outMap);
public static Set<Operator<?>> findTopOps(Operator<?> startOp, final Class<?> clazz) { final Set<Operator<?>> operators = new LinkedHashSet<Operator<?>>(); OperatorUtils.iterateParents(startOp, new NodeUtils.Function<Operator<?>>() { @Override public void apply(Operator<?> argument) { if (argument.getNumParent() == 0 && (clazz == null || clazz.isInstance(argument))) { operators.add(argument); } } }); return operators; }
private static boolean hasExternalTableAncestor(Operator op, StringBuilder sb) { boolean result = false; Operator ancestor = OperatorUtils.findSingleOperatorUpstream(op, TableScanOperator.class); if (ancestor != null) { TableScanOperator ts = (TableScanOperator) ancestor; if (MetaStoreUtils.isExternalTable(ts.getConf().getTableMetadata().getTTable())) { sb.append(ts.getConf().getTableMetadata().getFullyQualifiedName()); return true; } } return result; } }
Set<FileSinkOperator> fsOps = OperatorUtils.findOperators( ts, FileSinkOperator.class); fsOpsAll.addAll(fsOps); Set<ReduceSinkOperator> rsOps = OperatorUtils.findOperatorsUpstream(parentOfFS, ReduceSinkOperator.class); if (rsOps.isEmpty()) { continue;
String mapJoinAlias = mapJoinAliasToWorkEntry.getKey(); TableScanOperator mapJoinTaskTableScanOperator = OperatorUtils.findSingleOperator( mapJoinAliasToWorkEntry.getValue(), TableScanOperator.class); if (mapJoinTaskTableScanOperator == null) { OperatorUtils.findSingleOperator( mapJoinTaskTableScanOperator, FileSinkOperator.class); if (mapJoinTaskFileSinkOperator == null) { OperatorUtils.findSingleOperator( childMapWork.getAliasToWork().get(childMRAlias.toString()), TableScanOperator.class); if (childMRTaskTableScanOperator == null) {
Set<TableScanOperator> tsOps = OperatorUtils.findOperatorsUpstream(parent, TableScanOperator.class); boolean found = false; LOG.debug("Found semijoin optimization from the big table side of a map join, which will cause a task cycle. " + "Removing semijoin " + OperatorUtils.getOpNamePretty(rs) + " - " + OperatorUtils.getOpNamePretty(semiJoinMap.get(rs)));
+ "Either stats unavailable or expectedEntries exceeded max allowable bloomfilter size. " + "Removing semijoin " + OperatorUtils.getOpNamePretty(rs) + " - " + OperatorUtils.getOpNamePretty(ts)); if (LOG.isDebugEnabled()) { LOG.debug("Insufficient rows (" + numRows + ") to justify semijoin optimization. Removing semijoin " + OperatorUtils.getOpNamePretty(rs) + " - " + OperatorUtils.getOpNamePretty(ts)); Set<ReduceSinkOperator> rsOps = OperatorUtils.findOperators( ((Operator<?>) stack.get(stack.size() - 5)).getParentOperators().get(0), ReduceSinkOperator.class);
.build(); Multimap<Class<? extends Operator<?>>, Operator<?>> ops = OperatorUtils.classifyOperators(ts, searchedClasses);
Operator<?> rootOp = OperatorUtils.findSingleOperatorUpstreamJoinAccounted( mapJoinOp.getParentOperators().get(joinConf.getPosBigTable()), ReduceSinkOperator.class); Operator<?> rootOp = OperatorUtils.findSingleOperatorUpstreamJoinAccounted( mapJoinOp.getParentOperators().get(joinConf.getPosBigTable()), TableScanOperator.class);
TableScanOperator top = OperatorUtils.findSingleOperatorUpstream(gop, TableScanOperator.class); GroupByOperator mGop = OperatorUtils.findMapSideGb(gop); if(mGop != null) { containsGroupingSet = mGop.getConf().isGroupingSetsPresent();
/** * Given an operator and a set of classes, it returns the number of operators it finds * upstream that instantiate any of the given classes. * * @param start the start operator * @param classes the set of classes * @return the number of operators */ public static int countOperatorsUpstream(Operator<?> start, Set<Class<? extends Operator<?>>> classes) { Multimap<Class<? extends Operator<?>>, Operator<?>> ops = classifyOperatorsUpstream(start, classes); int numberOperators = 0; Set<Operator<?>> uniqueOperators = new HashSet<Operator<?>>(); for (Operator<?> op : ops.values()) { if (uniqueOperators.add(op)) { numberOperators++; } } return numberOperators; }
public static void setChildrenCollector(List<Operator<? extends OperatorDesc>> childOperators, OutputCollector out) { if (childOperators == null) { return; } for (Operator<? extends OperatorDesc> op : childOperators) { if (op.getName().equals(ReduceSinkOperator.getOperatorName())) { op.setOutputCollector(out); } else { setChildrenCollector(op.getChildOperators(), out); } } }
l4j.info("Memory available for operators set to {}", LlapUtil.humanReadableByteCount(memoryAvailableToTask)); OperatorUtils.setMemoryAvailable(reducer.getChildOperators(), memoryAvailableToTask); OperatorUtils.setChildrenCollector(children, outMap);
public static void iterateParents(Operator<?> operator, Function<Operator<?>> function) { iterateParents(operator, function, new HashSet<Operator<?>>()); }
private float getSelectivitySimpleTree(Operator<? extends OperatorDesc> op) { TableScanOperator tsOp = OperatorUtils .findSingleOperatorUpstream(op, TableScanOperator.class); if (tsOp == null) { // complex tree with multiple parents return getSelectivityComplexTree(op); } else { // simple tree with single parent long inputRow = tsOp.getStatistics().getNumRows(); long outputRow = op.getStatistics().getNumRows(); return (float) outputRow / (float) inputRow; } }
if (OperatorUtils.findSingleOperator(reduceWork.getReducer(), GroupByOperator.class) != null) { continue; OperatorUtils.findSingleOperator(operator, ReduceSinkOperator.class); if (child == null || child.getConf().getNumReducers() != 1 || !child.getConf().getPartitionCols().isEmpty()) {