final Configuration conf = getOperator().getParameters(); final String shipStrategy = conf.getString(Optimizer.HINT_SHIP_STRATEGY, null); final ShipStrategyType preSet; Operator<?> children = ((SingleInputOperator<?, ?, ?>) getOperator()).getInput(); throw new CompilerException("Error: Node for '" + getOperator().getName() + "' has no input."); } else { pred = contractToNode.get(children);
/** * Creates a new node with a single input for the optimizer plan. * * @param programOperator The PACT that the node represents. */ protected SingleInputNode(SingleInputOperator<?, ?, ?> programOperator) { super(programOperator); int[] k = programOperator.getKeyColumns(0); this.keys = k == null || k.length == 0 ? null : new FieldSet(k); }
@Override public SemanticProperties getSemanticProperties() { return getOperator().getSemanticProperties(); }
private <IN, OUT> List<OUT> executeUnaryOperator(SingleInputOperator<?, ?, ?> operator, int superStep) throws Exception { Operator<?> inputOp = operator.getInput(); if (inputOp == null) { throw new InvalidProgramException("The unary operation " + operator.getName() + " has no input."); } @SuppressWarnings("unchecked") List<IN> inputData = (List<IN>) execute(inputOp, superStep); @SuppressWarnings("unchecked") SingleInputOperator<IN, OUT, ?> typedOp = (SingleInputOperator<IN, OUT, ?>) operator; // build the runtime context and compute broadcast variables, if necessary TaskInfo taskInfo = new TaskInfo(typedOp.getName(), 1, 0, 1, 0); RuntimeUDFContext ctx; MetricGroup metrics = new UnregisteredMetricsGroup(); if (RichFunction.class.isAssignableFrom(typedOp.getUserCodeWrapper().getUserCodeClass())) { ctx = superStep == 0 ? new RuntimeUDFContext(taskInfo, userCodeClassLoader, executionConfig, cachedFiles, accumulators, metrics) : new IterationRuntimeUDFContext(taskInfo, userCodeClassLoader, executionConfig, cachedFiles, accumulators, metrics); for (Map.Entry<String, Operator<?>> bcInputs : operator.getBroadcastInputs().entrySet()) { List<?> bcData = execute(bcInputs.getValue()); ctx.setBroadcastVariable(bcInputs.getKey(), bcData); } } else { ctx = null; } return typedOp.executeOnCollections(inputData, ctx, executionConfig); }
SingleInputOperator<?, ?, ?> siop = (SingleInputOperator<?, ?, ?>) op; if (dynamicPathOperations.contains(siop.getInput())) { dynamicPathOperations.add(op); } else { for (Operator<?> o : siop.getBroadcastInputs().values()) { if (dynamicPathOperations.contains(o)) { dynamicPathOperations.add(op);
@Override public SingleInputPlanNode instantiate(Channel in, SingleInputNode node) { return new SingleInputPlanNode(node, "Map ("+node.getOperator().getName()+")", in, DriverStrategy.MAP); }
Union unionOperator = (Union) ((SingleInputOperator) sink.getInput()).getInput();
sn.setComparator(createComparator(singleInputOperator.getOperatorInfo().getInputType(), sn.getKeys(i), getSortOrders(sn.getKeys(i), sn.getSortOrders(i))), i);
@Override public SingleInputPlanNode instantiate(Channel in, SingleInputNode node) { return new SingleInputPlanNode(node, "GroupCombine ("+node.getOperator().getName()+")", in, DriverStrategy.ALL_GROUP_COMBINE); }
private <IN, OUT> List<OUT> executeUnaryOperator(SingleInputOperator<?, ?, ?> operator, int superStep) throws Exception { Operator<?> inputOp = operator.getInput(); if (inputOp == null) { throw new InvalidProgramException("The unary operation " + operator.getName() + " has no input."); } @SuppressWarnings("unchecked") List<IN> inputData = (List<IN>) execute(inputOp, superStep); @SuppressWarnings("unchecked") SingleInputOperator<IN, OUT, ?> typedOp = (SingleInputOperator<IN, OUT, ?>) operator; // build the runtime context and compute broadcast variables, if necessary TaskInfo taskInfo = new TaskInfo(typedOp.getName(), 1, 0, 1, 0); RuntimeUDFContext ctx; MetricGroup metrics = new UnregisteredMetricsGroup(); if (RichFunction.class.isAssignableFrom(typedOp.getUserCodeWrapper().getUserCodeClass())) { ctx = superStep == 0 ? new RuntimeUDFContext(taskInfo, userCodeClassLoader, executionConfig, cachedFiles, accumulators, metrics) : new IterationRuntimeUDFContext(taskInfo, userCodeClassLoader, executionConfig, cachedFiles, accumulators, metrics); for (Map.Entry<String, Operator<?>> bcInputs : operator.getBroadcastInputs().entrySet()) { List<?> bcData = execute(bcInputs.getValue()); ctx.setBroadcastVariable(bcInputs.getKey(), bcData); } } else { ctx = null; } return typedOp.executeOnCollections(inputData, ctx, executionConfig); }
((PlanUnwrappingReduceOperator<?, ?>) po.getInput()).setCustomPartitioner(grouper.getCustomPartitioner());
SingleInputOperator<?, ?, ?> siop = (SingleInputOperator<?, ?, ?>) op; if (dynamicPathOperations.contains(siop.getInput())) { dynamicPathOperations.add(op); } else { for (Operator<?> o : siop.getBroadcastInputs().values()) { if (dynamicPathOperations.contains(o)) { dynamicPathOperations.add(op);
sn.setComparator(createComparator(singleInputOperator.getOperatorInfo().getInputType(), sn.getKeys(i), getSortOrders(sn.getKeys(i), sn.getSortOrders(i))), i);
@Override public SingleInputPlanNode instantiate(Channel in, SingleInputNode node) { return new SingleInputPlanNode(node, "GroupReduce ("+node.getOperator().getName()+")", in, DriverStrategy.SORTED_GROUP_REDUCE, this.keyList); }
final Configuration conf = getOperator().getParameters(); final String shipStrategy = conf.getString(Optimizer.HINT_SHIP_STRATEGY, null); final ShipStrategyType preSet; Operator<?> children = ((SingleInputOperator<?, ?, ?>) getOperator()).getInput(); throw new CompilerException("Error: Node for '" + getOperator().getName() + "' has no input."); } else { pred = contractToNode.get(children);
private <IN, OUT> List<OUT> executeUnaryOperator(SingleInputOperator<?, ?, ?> operator, int superStep) throws Exception { Operator<?> inputOp = operator.getInput(); if (inputOp == null) { throw new InvalidProgramException("The unary operation " + operator.getName() + " has no input."); } @SuppressWarnings("unchecked") List<IN> inputData = (List<IN>) execute(inputOp, superStep); @SuppressWarnings("unchecked") SingleInputOperator<IN, OUT, ?> typedOp = (SingleInputOperator<IN, OUT, ?>) operator; // build the runtime context and compute broadcast variables, if necessary TaskInfo taskInfo = new TaskInfo(typedOp.getName(), 1, 0, 1, 0); RuntimeUDFContext ctx; MetricGroup metrics = new UnregisteredMetricsGroup(); if (RichFunction.class.isAssignableFrom(typedOp.getUserCodeWrapper().getUserCodeClass())) { ctx = superStep == 0 ? new RuntimeUDFContext(taskInfo, classLoader, executionConfig, cachedFiles, accumulatorRegistry, metrics) : new IterationRuntimeUDFContext(taskInfo, classLoader, executionConfig, cachedFiles, accumulatorRegistry, metrics); for (Map.Entry<String, Operator<?>> bcInputs : operator.getBroadcastInputs().entrySet()) { List<?> bcData = execute(bcInputs.getValue()); ctx.setBroadcastVariable(bcInputs.getKey(), bcData); } } else { ctx = null; } return typedOp.executeOnCollections(inputData, ctx, executionConfig); }
Union secondUnionOperator = (Union) ((SingleInputOperator) sink.getInput()).getInput();
SingleInputOperator<?, ?, ?> siop = (SingleInputOperator<?, ?, ?>) op; if (dynamicPathOperations.contains(siop.getInput())) { dynamicPathOperations.add(op); } else { for (Operator<?> o : siop.getBroadcastInputs().values()) { if (dynamicPathOperations.contains(o)) { dynamicPathOperations.add(op);
@Override protected SemanticProperties getSemanticPropertiesForLocalPropertyFiltering() { // Local properties for MapPartition may not be preserved. SingleInputSemanticProperties origProps = ((SingleInputOperator<?,?,?>) getOperator()).getSemanticProperties(); SingleInputSemanticProperties filteredProps = new SingleInputSemanticProperties(); FieldSet readSet = origProps.getReadFields(0); if(readSet != null) { filteredProps.addReadFields(readSet); } return filteredProps; }
/** * Creates a new node with a single input for the optimizer plan. * * @param programOperator The PACT that the node represents. */ protected SingleInputNode(SingleInputOperator<?, ?, ?> programOperator) { super(programOperator); int[] k = programOperator.getKeyColumns(0); this.keys = k == null || k.length == 0 ? null : new FieldSet(k); }