private JobVertex createSingleInputVertex(SingleInputPlanNode node) throws CompilerException { final String taskName = node.getNodeName(); final DriverStrategy ds = node.getDriverStrategy(); Channel inConn = node.getInput(); PlanNode pred = inConn.getSource(); chaining = ds.getPushChainDriverClass() != null && inConn.getLocalStrategy() == LocalStrategy.NONE && pred.getOutgoingChannels().size() == 1 && node.getParallelism() == pred.getParallelism() && node.getBroadcastInputs().isEmpty(); if (this.currentIteration instanceof WorksetIterationPlanNode && node.getOutgoingChannels().size() > 0) if (node == wspn.getRootOfTerminationCriterion() && wspn.getRootOfStepFunction() == pred){ chaining = false; }else if(node.getOutgoingChannels().size() > 0 &&(wspn.getRootOfStepFunction() == pred || wspn.getRootOfTerminationCriterion() == pred)) { chaining = false; vertex.setResources(node.getMinResources(), node.getPreferredResources()); vertex.setInvokableClass((this.currentIteration != null && node.isOnDynamicPath()) ? IterationIntermediateTask.class : BatchTask.class); config.setStubWrapper(node.getProgramOperator().getUserCodeWrapper()); config.setStubParameters(node.getProgramOperator().getParameters()); config.setDriverComparator(node.getComparator(i), i);
final SingleInputPlanNode combiner = (SingleInputPlanNode) reducer.getPredecessor(); final SingleInputPlanNode mapper = or.getNode(MAPPER_NAME); assertEquals(1, mapper.getBroadcastInputs().size()); assertEquals(ShipStrategyType.FORWARD, mapper.getInput().getShipStrategy()); assertEquals(ShipStrategyType.BROADCAST, mapper.getBroadcastInputs().get(0).getShipStrategy()); assertEquals(LocalStrategy.NONE, mapper.getInput().getLocalStrategy()); assertEquals(LocalStrategy.NONE, mapper.getBroadcastInputs().get(0).getLocalStrategy()); assertEquals(DriverStrategy.MAP, mapper.getDriverStrategy()); assertNull(mapper.getInput().getLocalStrategyKeys()); assertNull(mapper.getInput().getLocalStrategySortOrder()); assertNull(mapper.getBroadcastInputs().get(0).getLocalStrategyKeys()); assertNull(mapper.getBroadcastInputs().get(0).getLocalStrategySortOrder()); assertEquals(ShipStrategyType.FORWARD, combiner.getInput().getShipStrategy()); assertEquals(LocalStrategy.NONE, combiner.getInput().getLocalStrategy()); assertEquals(DriverStrategy.SORTED_GROUP_COMBINE, combiner.getDriverStrategy()); assertNull(combiner.getInput().getLocalStrategyKeys()); assertNull(combiner.getInput().getLocalStrategySortOrder()); assertEquals(set0, combiner.getKeys(0)); assertEquals(set0, combiner.getKeys(1)); assertEquals(ShipStrategyType.PARTITION_HASH, reducer.getInput().getShipStrategy()); assertEquals(LocalStrategy.COMBININGSORT, reducer.getInput().getLocalStrategy()); assertEquals(DriverStrategy.SORTED_GROUP_REDUCE, reducer.getDriverStrategy()); assertEquals(set0, reducer.getKeys(0));
final Channel sipChannel = new Channel(sourceNode, TempMode.NONE); sipChannel.setShipStrategy(ShipStrategyType.FORWARD, DataExchangeMode.PIPELINED); final SingleInputPlanNode sipPlanNode = new SingleInputPlanNode(sipNode, SIP_NAME, sipChannel, DriverStrategy.MAP_PARTITION); sipNode.setParallelism(sourceParallelism); sipPlanNode.setParallelism(sourceParallelism); sipPlanNode.initProperties(new GlobalProperties(), new LocalProperties()); sipPlanNode.setCosts(defaultZeroCosts); sipChannel.setTarget(sipPlanNode); this.plan.getAllNodes().add(sipPlanNode); final Channel sicChannel = new Channel(sipPlanNode, TempMode.NONE); sicChannel.setShipStrategy(ShipStrategyType.FORWARD, DataExchangeMode.PIPELINED); final SingleInputPlanNode sicPlanNode = new SingleInputPlanNode(sicNode, SIC_NAME, sicChannel, DriverStrategy.ALL_GROUP_REDUCE); sicNode.setParallelism(1); sicPlanNode.setParallelism(1); sicPlanNode.initProperties(new GlobalProperties(), new LocalProperties()); sicPlanNode.setCosts(defaultZeroCosts); sicChannel.setTarget(sicPlanNode); sipPlanNode.addOutgoingChannel(sicChannel); this.plan.getAllNodes().add(sicPlanNode); final Channel rbChannel = new Channel(sicPlanNode, TempMode.NONE); rbChannel.setShipStrategy(ShipStrategyType.FORWARD, DataExchangeMode.PIPELINED); final SingleInputPlanNode rbPlanNode = new SingleInputPlanNode(rbNode, RB_NAME, rbChannel, DriverStrategy.MAP_PARTITION); rbNode.setParallelism(1); rbPlanNode.setParallelism(1); rbPlanNode.initProperties(new GlobalProperties(), new LocalProperties()); rbPlanNode.setCosts(defaultZeroCosts); rbChannel.setTarget(rbPlanNode);
@Override public SingleInputPlanNode instantiate(Channel in, SingleInputNode node) { node.setParallelism(in.getSource().getParallelism()); // sorting key info SingleInputPlanNode singleInputPlanNode = new SingleInputPlanNode( node, "GroupCombine (" + node.getOperator().getName() + ")", in, // reuse the combine strategy also used in the group reduce DriverStrategy.SORTED_GROUP_COMBINE, this.keyList); // set sorting comparator key info singleInputPlanNode.setDriverKeyInfo(this.ordering.getInvolvedIndexes(), this.ordering.getFieldSortDirections(), 0); // set grouping comparator key info singleInputPlanNode.setDriverKeyInfo(this.keyList, 1); return singleInputPlanNode; }
private void checkStandardStrategies(SingleInputPlanNode map, DualInputPlanNode join, SingleInputPlanNode combiner, SingleInputPlanNode reducer, SinkPlanNode sink) { // check ship strategies that are always fix Assert.assertEquals(ShipStrategyType.FORWARD, map.getInput().getShipStrategy()); Assert.assertEquals(ShipStrategyType.FORWARD, sink.getInput().getShipStrategy()); // check the driver strategies that are always fix Assert.assertEquals(DriverStrategy.FLAT_MAP, map.getDriverStrategy()); Assert.assertEquals(DriverStrategy.SORTED_GROUP_REDUCE, reducer.getDriverStrategy()); Assert.assertEquals(DriverStrategy.NONE, sink.getDriverStrategy()); if (combiner != null) { Assert.assertEquals(DriverStrategy.SORTED_GROUP_COMBINE, combiner.getDriverStrategy()); Assert.assertEquals(LocalStrategy.NONE, combiner.getInput().getLocalStrategy()); } }
traverseChannel(addMapper.getInput()); SingleInputPlanNode sn = (SingleInputPlanNode) node; if (!(sn.getOptimizerNode().getOperator() instanceof SingleInputOperator)) { if(sn.getOptimizerNode().getOperator() instanceof NoOpUnaryUdfOp) { traverseChannel(sn.getInput()); return; } else { SingleInputOperator<?, ?, ?> singleInputOperator = (SingleInputOperator<?, ?, ?>) sn.getOptimizerNode().getOperator(); for(int i=0;i<sn.getDriverStrategy().getNumRequiredComparators();i++) { sn.setComparator(createComparator(singleInputOperator.getOperatorInfo().getInputType(), sn.getKeys(i), getSortOrders(sn.getKeys(i), sn.getSortOrders(i))), i); traverseChannel(sn.getInput()); for (Channel c: sn.getBroadcastInputs()) { traverseChannel(c);
SingleInputPlanNode combineNode = (SingleInputPlanNode) reduceNode.getInput().getSource(); SingleInputPlanNode keyExtractor = (SingleInputPlanNode) combineNode.getInput().getSource(); SingleInputPlanNode keyProjector = (SingleInputPlanNode) sinkNode.getInput().getSource(); assertEquals(sourceNode, keyExtractor.getInput().getSource()); assertEquals(keyProjector, sinkNode.getInput().getSource()); assertEquals(DriverStrategy.SORTED_REDUCE, reduceNode.getDriverStrategy()); assertEquals(DriverStrategy.SORTED_PARTIAL_REDUCE, combineNode.getDriverStrategy()); assertEquals(new FieldList(0), reduceNode.getKeys(0)); assertEquals(new FieldList(0), combineNode.getKeys(0)); assertEquals(new FieldList(0), reduceNode.getInput().getLocalStrategyKeys()); assertEquals(6, keyExtractor.getParallelism()); assertEquals(6, combineNode.getParallelism()); assertEquals(8, reduceNode.getParallelism()); assertEquals(8, keyProjector.getParallelism()); assertEquals(8, sinkNode.getParallelism());
traverse(addMapper.getInput().getSource(), createEmptySchema(), false); try { addMapper.getInput().setSerializer(createSerializer(createEmptySchema())); } catch (MissingFieldTypeInfoException e) { throw new RuntimeException(e); traverse(addMapper.getInput().getSource(), createEmptySchema(), createUtilities); try { addMapper.getInput().setSerializer(createSerializer(createEmptySchema())); } catch (MissingFieldTypeInfoException e) { throw new RuntimeException(e); SingleInputNode optNode = sn.getSingleInputNode(); if (schema.getNumConnectionsThatContributed() < sn.getOutgoingChannels().size()) { return; for(int i=0;i<sn.getDriverStrategy().getNumRequiredComparators();i++) { try { sn.setComparator(createComparator(sn.getKeys(i), sn.getSortOrders(i), schema),i); } catch (MissingFieldTypeInfoException e) { throw new CompilerPostPassException("Could not set up runtime strategy for node '" + propagateToChannel(schema, sn.getInput(), createUtilities); } catch (MissingFieldTypeInfoException e) { throw new CompilerPostPassException("Could not set up runtime strategy for input channel to node '" + for (Channel c: sn.getBroadcastInputs()) { try {
private boolean checkHashJoinStrategies(DualInputPlanNode join, SingleInputPlanNode reducer, boolean buildFirst) { if ((buildFirst && DriverStrategy.HYBRIDHASH_BUILD_FIRST == join.getDriverStrategy()) || (!buildFirst && DriverStrategy.HYBRIDHASH_BUILD_SECOND == join.getDriverStrategy())) { // driver keys Assert.assertEquals(set0, join.getKeysForInput1()); Assert.assertEquals(set0, join.getKeysForInput2()); // local strategies Assert.assertEquals(LocalStrategy.NONE, join.getInput1().getLocalStrategy()); Assert.assertEquals(LocalStrategy.NONE, join.getInput2().getLocalStrategy()); Assert.assertEquals(LocalStrategy.COMBININGSORT, reducer.getInput().getLocalStrategy()); // local strategy keys Assert.assertEquals(set01, reducer.getInput().getLocalStrategyKeys()); Assert.assertEquals(set01, reducer.getKeys(0)); Assert.assertTrue(Arrays.equals(reducer.getInput().getLocalStrategySortOrder(), reducer.getSortOrders(0))); return true; } else { return false; } }
SingleInputPlanNode combineNode = (SingleInputPlanNode) reduceNode.getInput().getSource(); assertEquals(sourceNode, combineNode.getInput().getSource()); assertEquals(reduceNode, sinkNode.getInput().getSource()); assertEquals(DriverStrategy.ALL_REDUCE, reduceNode.getDriverStrategy()); assertEquals(DriverStrategy.ALL_REDUCE, combineNode.getDriverStrategy()); assertEquals(8, combineNode.getParallelism()); assertEquals(1, reduceNode.getParallelism()); assertEquals(1, sinkNode.getParallelism());
assertEquals(ShipStrategyType.PARTITION_HASH, worksetReducer.getInput().getShipStrategy()); assertEquals(new FieldList(1, 2), worksetReducer.getKeys(0)); ShipStrategyType ss1 = deltaMapper.getInput().getShipStrategy(); ShipStrategyType ss2 = deltaMapper.getOutgoingChannels().get(0).getShipStrategy();
@Override public SingleInputPlanNode instantiate(Channel in, SingleInputNode node) { if (in.getShipStrategy() == ShipStrategyType.FORWARD) { // locally connected, directly instantiate return new SingleInputPlanNode(node, "GroupReduce ("+node.getOperator().getName()+")", in, DriverStrategy.ALL_GROUP_REDUCE); } else { // non forward case.plug in a combiner Channel toCombiner = new Channel(in.getSource()); toCombiner.setShipStrategy(ShipStrategyType.FORWARD, DataExchangeMode.PIPELINED); // create an input node for combine with same parallelism as input node GroupReduceNode combinerNode = ((GroupReduceNode) node).getCombinerUtilityNode(); combinerNode.setParallelism(in.getSource().getParallelism()); SingleInputPlanNode combiner = new SingleInputPlanNode(combinerNode, "Combine ("+node.getOperator().getName()+")", toCombiner, DriverStrategy.ALL_GROUP_REDUCE_COMBINE); combiner.setCosts(new Costs(0, 0)); combiner.initProperties(toCombiner.getGlobalProperties(), toCombiner.getLocalProperties()); Channel toReducer = new Channel(combiner); toReducer.setShipStrategy(in.getShipStrategy(), in.getShipStrategyKeys(), in.getShipStrategySortOrder(), in.getDataExchangeMode()); toReducer.setLocalStrategy(in.getLocalStrategy(), in.getLocalStrategyKeys(), in.getLocalStrategySortOrder()); return new SingleInputPlanNode(node, "GroupReduce ("+node.getOperator().getName()+")", toReducer, DriverStrategy.ALL_GROUP_REDUCE); } }
SingleInputPlanNode keyRemover = (SingleInputPlanNode) mapper.getInput().getSource(); SingleInputPlanNode partitioner = (SingleInputPlanNode) keyRemover.getInput().getSource(); SingleInputPlanNode keyExtractor = (SingleInputPlanNode) partitioner.getInput().getSource(); SingleInputPlanNode balancer = (SingleInputPlanNode) keyExtractor.getInput().getSource(); assertEquals(ShipStrategyType.FORWARD, mapper.getInput().getShipStrategy()); assertEquals(parallelism, mapper.getParallelism()); assertEquals(ShipStrategyType.FORWARD, keyRemover.getInput().getShipStrategy()); assertEquals(parallelism, keyRemover.getParallelism()); assertEquals(ShipStrategyType.PARTITION_CUSTOM, partitioner.getInput().getShipStrategy()); assertEquals(part, partitioner.getInput().getPartitioner()); assertEquals(parallelism, partitioner.getParallelism()); assertEquals(ShipStrategyType.FORWARD, keyExtractor.getInput().getShipStrategy()); assertEquals(parallelism, keyExtractor.getParallelism()); assertEquals(ShipStrategyType.PARTITION_FORCED_REBALANCE, balancer.getInput().getShipStrategy()); assertEquals(parallelism, balancer.getParallelism());
private boolean checkBroadcastShipStrategies(DualInputPlanNode join, SingleInputPlanNode reducer, SingleInputPlanNode combiner) { if (ShipStrategyType.BROADCAST == join.getInput1().getShipStrategy() && ShipStrategyType.FORWARD == join.getInput2().getShipStrategy() && ShipStrategyType.PARTITION_HASH == reducer.getInput().getShipStrategy()) { // check combiner Assert.assertNotNull("Plan should have a combiner", combiner); Assert.assertEquals(ShipStrategyType.FORWARD, combiner.getInput().getShipStrategy()); return true; } else { return false; } }
SingleInputPlanNode beforeMapper = (SingleInputPlanNode) mapper.getInput().getSource(); assertEquals(TempMode.NONE, mapper.getInput().getTempMode()); assertEquals(TempMode.NONE, beforeMapper.getInput().getTempMode()); assertEquals(TempMode.NONE, mapper.getBroadcastInputs().get(0).getTempMode()); assertEquals(DataExchangeMode.PIPELINED, mapper.getInput().getDataExchangeMode()); assertEquals(DataExchangeMode.BATCH, beforeMapper.getInput().getDataExchangeMode()); assertEquals(DataExchangeMode.BATCH, mapper.getBroadcastInputs().get(0).getDataExchangeMode());
@Override public SourceAndDamReport hasDamOnPathDownTo(PlanNode source) { if (source == this) { return FOUND_SOURCE; } SourceAndDamReport res = this.input.getSource().hasDamOnPathDownTo(source); if (res == FOUND_SOURCE_AND_DAM) { return FOUND_SOURCE_AND_DAM; } else if (res == FOUND_SOURCE) { return (this.input.getLocalStrategy().dams() || this.input.getTempMode().breaksPipeline() || getDriverStrategy().firstDam() == DamBehavior.FULL_DAM) ? FOUND_SOURCE_AND_DAM : FOUND_SOURCE; } else { // NOT_FOUND // check the broadcast inputs for (NamedChannel nc : getBroadcastInputs()) { SourceAndDamReport bcRes = nc.getSource().hasDamOnPathDownTo(source); if (bcRes != NOT_FOUND) { // broadcast inputs are always dams return FOUND_SOURCE_AND_DAM; } } return NOT_FOUND; } }
SingleInputPlanNode outDopFull = resolver.getNode("outDopFull"); SingleInputPlanNode outDopHalf = resolver.getNode("outDopHalf"); NAryUnionPlanNode unionDopFull = (NAryUnionPlanNode) outDopFull.getInput().getSource(); NAryUnionPlanNode unionDopHalf = (NAryUnionPlanNode) outDopHalf.getInput().getSource(); assertEquals(2, inDopFull.getOutgoingChannels().size()); assertEquals(2, inDopHalf.getOutgoingChannels().size()); assertEquals(fullDop, inDopFull.getParallelism()); assertEquals(halfDop, inDopHalf.getParallelism()); assertEquals(fullDop, outDopFull.getParallelism()); assertEquals(halfDop, outDopHalf.getParallelism()); assertEquals(ShipStrategyType.FORWARD, outDopHalf.getInput().getShipStrategy()); assertEquals(ShipStrategyType.FORWARD, outDopFull.getInput().getShipStrategy()); if (inDopFull.getOutgoingChannels().get(0).getTarget() == unionDopFull) { fullFull = inDopFull.getOutgoingChannels().get(0); fullHalf = inDopFull.getOutgoingChannels().get(1); } else { fullFull = inDopFull.getOutgoingChannels().get(1); fullHalf = inDopFull.getOutgoingChannels().get(0); if (inDopHalf.getOutgoingChannels().get(0).getTarget() == unionDopFull) { halfFull = inDopHalf.getOutgoingChannels().get(0); halfHalf = inDopHalf.getOutgoingChannels().get(1); } else { halfFull = inDopHalf.getOutgoingChannels().get(1); halfHalf = inDopHalf.getOutgoingChannels().get(0);
@Override public Iterable<PlanNode> getPredecessors() { if (getBroadcastInputs() == null || getBroadcastInputs().isEmpty()) { return Collections.singleton(this.input.getSource()); } else { List<PlanNode> preds = new ArrayList<PlanNode>(); preds.add(input.getSource()); for (Channel c : getBroadcastInputs()) { preds.add(c.getSource()); } return preds; } }
@Override public SingleInputPlanNode instantiate(Channel in, SingleInputNode node) { return new SingleInputPlanNode(node, "Partition", in, DriverStrategy.UNARY_NO_OP); }