@Override public SingleInputPlanNode instantiate(Channel in, SingleInputNode node) { if (in.getShipStrategy() == ShipStrategyType.FORWARD) { // locally connected, directly instantiate return new SingleInputPlanNode(node, "GroupReduce ("+node.getOperator().getName()+")", in, DriverStrategy.ALL_GROUP_REDUCE); } else { // non forward case.plug in a combiner Channel toCombiner = new Channel(in.getSource()); toCombiner.setShipStrategy(ShipStrategyType.FORWARD, DataExchangeMode.PIPELINED); // create an input node for combine with same parallelism as input node GroupReduceNode combinerNode = ((GroupReduceNode) node).getCombinerUtilityNode(); combinerNode.setParallelism(in.getSource().getParallelism()); SingleInputPlanNode combiner = new SingleInputPlanNode(combinerNode, "Combine ("+node.getOperator().getName()+")", toCombiner, DriverStrategy.ALL_GROUP_REDUCE_COMBINE); combiner.setCosts(new Costs(0, 0)); combiner.initProperties(toCombiner.getGlobalProperties(), toCombiner.getLocalProperties()); Channel toReducer = new Channel(combiner); toReducer.setShipStrategy(in.getShipStrategy(), in.getShipStrategyKeys(), in.getShipStrategySortOrder(), in.getDataExchangeMode()); toReducer.setLocalStrategy(in.getLocalStrategy(), in.getLocalStrategyKeys(), in.getLocalStrategySortOrder()); return new SingleInputPlanNode(node, "GroupReduce ("+node.getOperator().getName()+")", toReducer, DriverStrategy.ALL_GROUP_REDUCE); } }
switch (channel.getShipStrategy()) { case FORWARD: distributionPattern = DistributionPattern.POINTWISE; break; default: throw new RuntimeException("Unknown runtime ship strategy: " + channel.getShipStrategy()); switch (channel.getDataExchangeMode()) { resultType = channel.getSource().isOnDynamicPath() ? ResultPartitionType.PIPELINED : ResultPartitionType.BLOCKING; channel.getDataExchangeMode() + " currently not supported."); sourceConfig.addOutputShipStrategy(channel.getShipStrategy()); if (outputIndex == 0) { sourceConfig.setOutputSerializer(channel.getSerializer()); if (channel.getShipStrategyComparator() != null) { sourceConfig.setOutputComparator(channel.getShipStrategyComparator(), outputIndex); if (channel.getShipStrategy() == ShipStrategyType.PARTITION_RANGE) { final DataDistribution dataDistribution = channel.getDataDistribution(); if (dataDistribution != null) { sourceConfig.setOutputDataDistribution(dataDistribution, outputIndex);
public DualInputPlanNode(OptimizerNode template, String nodeName, Channel input1, Channel input2, DriverStrategy diverStrategy, FieldList driverKeyFields1, FieldList driverKeyFields2, boolean[] driverSortOrders) { super(template, nodeName, diverStrategy); this.input1 = input1; this.input2 = input2; this.keys1 = driverKeyFields1; this.keys2 = driverKeyFields2; this.sortOrders = driverSortOrders; if (this.input1.getShipStrategy() == ShipStrategyType.BROADCAST) { this.input1.setReplicationFactor(getParallelism()); } if (this.input2.getShipStrategy() == ShipStrategyType.BROADCAST) { this.input2.setReplicationFactor(getParallelism()); } mergeBranchPlanMaps(input1.getSource(), input2.getSource()); }
public void collect(Channel in, List<Channel> inputs) { if (in.getSource() instanceof NAryUnionPlanNode) { // sanity check if (in.getShipStrategy() != ShipStrategyType.FORWARD) { throw new CompilerException("Bug: Plan generation for Unions picked a ship strategy between binary plan operators."); } if (!(in.getLocalStrategy() == null || in.getLocalStrategy() == LocalStrategy.NONE)) { throw new CompilerException("Bug: Plan generation for Unions picked a local strategy between binary plan operators."); } inputs.addAll(((NAryUnionPlanNode) in.getSource()).getListOfInputs()); } else { // is not a collapsed union node, so we take the channel directly inputs.add(in); } } }
public static TypeComparatorFactory<?> getShipComparator(Channel channel, ExecutionConfig executionConfig) { PlanNode source = channel.getSource(); Operator<?> javaOp = source.getProgramOperator(); TypeInformation<?> type = javaOp.getOperatorInfo().getOutputType(); return createComparator(type, channel.getShipStrategyKeys(), getSortOrders(channel.getShipStrategyKeys(), channel.getShipStrategySortOrder()), executionConfig); }
private void traverseChannel(Channel channel) { PlanNode source = channel.getSource(); Operator<?> javaOp = source.getProgramOperator(); channel.setSerializer(createSerializer(type)); if (channel.getShipStrategy().requiresComparator()) { channel.setShipStrategyComparator(createComparator(type, channel.getShipStrategyKeys(), getSortOrders(channel.getShipStrategyKeys(), channel.getShipStrategySortOrder()))); if (channel.getLocalStrategy().requiresComparator()) { channel.setLocalStrategyComparator(createComparator(type, channel.getLocalStrategyKeys(), getSortOrders(channel.getLocalStrategyKeys(), channel.getLocalStrategySortOrder()))); traverse(channel.getSource());
assertEquals(ShipStrategyType.FORWARD, mapper.getInput().getShipStrategy()); assertEquals(ShipStrategyType.BROADCAST, mapper.getBroadcastInputs().get(0).getShipStrategy()); assertEquals(LocalStrategy.NONE, mapper.getInput().getLocalStrategy()); assertEquals(LocalStrategy.NONE, mapper.getBroadcastInputs().get(0).getLocalStrategy()); assertNull(mapper.getInput().getLocalStrategyKeys()); assertNull(mapper.getInput().getLocalStrategySortOrder()); assertNull(mapper.getBroadcastInputs().get(0).getLocalStrategyKeys()); assertNull(mapper.getBroadcastInputs().get(0).getLocalStrategySortOrder()); assertEquals(ShipStrategyType.FORWARD, combiner.getInput().getShipStrategy()); assertEquals(LocalStrategy.NONE, combiner.getInput().getLocalStrategy()); assertEquals(DriverStrategy.SORTED_GROUP_COMBINE, combiner.getDriverStrategy()); assertNull(combiner.getInput().getLocalStrategyKeys()); assertNull(combiner.getInput().getLocalStrategySortOrder()); assertEquals(set0, combiner.getKeys(0)); assertEquals(set0, combiner.getKeys(1)); assertEquals(ShipStrategyType.PARTITION_HASH, reducer.getInput().getShipStrategy()); assertEquals(LocalStrategy.COMBININGSORT, reducer.getInput().getLocalStrategy()); assertEquals(DriverStrategy.SORTED_GROUP_REDUCE, reducer.getDriverStrategy()); assertEquals(set0, reducer.getKeys(0)); assertEquals(set0, reducer.getInput().getLocalStrategyKeys()); assertTrue(Arrays.equals(reducer.getInput().getLocalStrategySortOrder(), reducer.getSortOrders(0))); assertEquals(ShipStrategyType.FORWARD, sink.getInput().getShipStrategy()); assertEquals(LocalStrategy.NONE, sink.getInput().getLocalStrategy());
Assert.assertTrue(!neighborsJoin.getInput1().getTempMode().isCached()); Assert.assertTrue(!neighborsJoin.getInput2().getTempMode().isCached()); Assert.assertEquals(set0, neighborsJoin.getKeysForInput1()); Assert.assertEquals(set0, neighborsJoin.getKeysForInput2()); Assert.assertEquals(ShipStrategyType.FORWARD, sink.getInput().getShipStrategy()); Assert.assertEquals(ShipStrategyType.PARTITION_HASH, iter.getInitialSolutionSetInput().getShipStrategy()); Assert.assertEquals(set0, iter.getInitialSolutionSetInput().getShipStrategyKeys()); Assert.assertEquals(ShipStrategyType.PARTITION_HASH, iter.getInitialWorksetInput().getShipStrategy()); Assert.assertEquals(set0, iter.getInitialWorksetInput().getShipStrategyKeys()); Assert.assertEquals(ShipStrategyType.FORWARD, neighborsJoin.getInput1().getShipStrategy()); // workset Assert.assertEquals(ShipStrategyType.PARTITION_HASH, neighborsJoin.getInput2().getShipStrategy()); // edges Assert.assertEquals(set0, neighborsJoin.getInput2().getShipStrategyKeys()); Assert.assertEquals(ShipStrategyType.PARTITION_HASH, minIdReducer.getInput().getShipStrategy()); Assert.assertEquals(set0, minIdReducer.getInput().getShipStrategyKeys()); Assert.assertEquals(ShipStrategyType.FORWARD, minIdCombiner.getInput().getShipStrategy()); Assert.assertEquals(ShipStrategyType.FORWARD, updatingMatch.getInput1().getShipStrategy()); // solution set Assert.assertEquals(ShipStrategyType.FORWARD, updatingMatch.getInput2().getShipStrategy()); // min id Assert.assertEquals(LocalStrategy.NONE, sink.getInput().getLocalStrategy()); Assert.assertEquals(LocalStrategy.NONE, iter.getInitialSolutionSetInput().getLocalStrategy()); Assert.assertEquals(LocalStrategy.NONE, iter.getInitialWorksetInput().getLocalStrategy()); Assert.assertEquals(LocalStrategy.NONE, neighborsJoin.getInput1().getLocalStrategy()); // workset Assert.assertEquals(LocalStrategy.NONE, neighborsJoin.getInput2().getLocalStrategy()); // edges
channel.getShipStrategy() : inConn.getShipStrategy(); if (channel != null && channel.getShipStrategyKeys() != null && channel.getShipStrategyKeys().size() > 0) { shipStrategy += " on " + (channel.getShipStrategySortOrder() == null ? channel.getShipStrategyKeys().toString() : Utils.createOrdering(channel.getShipStrategyKeys(), channel.getShipStrategySortOrder()).toString()); switch (channel.getLocalStrategy()) { case NONE: break; break; default: throw new CompilerException("Unknown local strategy " + channel.getLocalStrategy().name()); if (channel != null && channel.getLocalStrategyKeys() != null && channel.getLocalStrategyKeys().size() > 0) { localStrategy += " on " + (channel.getLocalStrategySortOrder() == null ? channel.getLocalStrategyKeys().toString() : Utils.createOrdering(channel.getLocalStrategyKeys(), channel.getLocalStrategySortOrder()).toString()); if (channel != null && channel.getTempMode() != TempMode.NONE) { String tempMode = channel.getTempMode().toString(); writer.print(", \"temp_mode\": \"" + tempMode + "\""); String exchangeMode = channel.getDataExchangeMode().toString(); writer.print(", \"exchange_mode\": \"" + exchangeMode + "\"");
private void addLocalInfoFromChannelToConfig(Channel channel, TaskConfig config, int inputNum, boolean isBroadcastChannel) { config.setBroadcastInputSerializer(channel.getSerializer(), inputNum); if (channel.getLocalStrategy() != LocalStrategy.NONE || (channel.getTempMode() != null && channel.getTempMode() != TempMode.NONE)) { throw new CompilerException("Found local strategy or temp mode on a broadcast variable channel."); } else { config.setInputSerializer(channel.getSerializer(), inputNum); if (channel.getLocalStrategy() != LocalStrategy.NONE) { config.setInputLocalStrategy(inputNum, channel.getLocalStrategy()); if (channel.getLocalStrategyComparator() != null) { config.setInputComparator(channel.getLocalStrategyComparator(), inputNum); if (channel.getTempMode() != null) { final TempMode tm = channel.getTempMode(); (channel.isOnDynamicPath() || channel.getDataExchangeMode() != DataExchangeMode.BATCH) ) { config.setInputAsynchronouslyMaterialized(inputNum, true); needsMemory = true; if (tm == TempMode.NONE || channel.getRelativeTempMemory() <= 0) { throw new CompilerException("Bug in compiler: Inconsistent description of input materialization."); config.setRelativeInputMaterializationMemory(inputNum, channel.getRelativeTempMemory());
Channel toMap1 = new Channel(target); SingleInputPlanNode map1 = new SingleInputPlanNode(getMapNode(), "Mapper 1", toMap1, DriverStrategy.MAP); Channel toMap2 = new Channel(map1); SingleInputPlanNode map2 = new SingleInputPlanNode(getMapNode(), "Mapper 2", toMap2, DriverStrategy.MAP); Channel toMap3 = new Channel(map2); SingleInputPlanNode map3 = new SingleInputPlanNode(getMapNode(), "Mapper 3", toMap3, DriverStrategy.MAP); toMap1.setShipStrategy(ShipStrategyType.FORWARD, DataExchangeMode.PIPELINED); toMap1.setLocalStrategy(LocalStrategy.SORT, new FieldList(5, 7), new boolean[] {false, false}); toMap2.setShipStrategy(ShipStrategyType.FORWARD, DataExchangeMode.PIPELINED); toMap2.setLocalStrategy(LocalStrategy.NONE); toMap3.setShipStrategy(ShipStrategyType.FORWARD, DataExchangeMode.PIPELINED); toMap3.setLocalStrategy(LocalStrategy.NONE); toMap1.setRequiredGlobalProps(null); toMap1.setRequiredLocalProps(null); toMap2.setRequiredGlobalProps(null); toMap2.setRequiredLocalProps(null); toMap3.setRequiredGlobalProps(null); toMap3.setRequiredLocalProps(reqLp); toMap1.setShipStrategy(ShipStrategyType.PARTITION_HASH, new FieldList(1, 2), DataExchangeMode.PIPELINED); toMap1.setLocalStrategy(LocalStrategy.NONE);
assertTrue(op.getDataSinks().iterator().next().getInput().getSource() instanceof WorksetIterationPlanNode); WorksetIterationPlanNode wipn = (WorksetIterationPlanNode) op.getDataSinks().iterator().next().getInput().getSource(); assertEquals(ShipStrategyType.FORWARD, wipn.getInput1().getShipStrategy()); assertEquals(DataExchangeMode.BATCH, wipn.getInput1().getDataExchangeMode()); assertEquals(DataExchangeMode.BATCH, wipn.getInput2().getDataExchangeMode()); assertEquals(TempMode.NONE, wipn.getInput1().getTempMode()); assertEquals(TempMode.NONE, wipn.getInput2().getTempMode());
this.worksetNode.setCandidateProperties(worksetIn.getGlobalProperties(), worksetIn.getLocalProperties(), worksetIn); this.solutionSetNode.setCandidateProperties(this.partitionedProperties, new LocalProperties(), solutionSetIn); Channel toNoOp = new Channel(candidate); globPropsReqWorkset.parameterizeChannel(toNoOp, false, nextWorksetRootConnection.getDataExchangeMode(), false); rebuildWorksetPropertiesNode, "Rebuild Workset Properties", toNoOp, DriverStrategy.UNARY_NO_OP); rebuildWorksetPropertiesPlanNode.initProperties(toNoOp.getGlobalProperties(), toNoOp.getLocalProperties()); estimator.costOperator(rebuildWorksetPropertiesPlanNode); if (siSolutionDeltaCandidate.getInput().getShipStrategy() == ShipStrategyType.FORWARD && this.solutionDeltaImmediatelyAfterSolutionJoin) siSolutionDeltaCandidate.getInput().getLocalStrategy() != LocalStrategy.NONE) solutionSetCandidate = siSolutionDeltaCandidate.getInput().getSource(); immediateDeltaUpdate = true; } else { siSolutionDeltaCandidate.getInput().setTempMode(TempMode.PIPELINE_BREAKER); immediateDeltaUpdate = false;
SingleInputPlanNode mapper = (SingleInputPlanNode) sink.getInput().getSource(); SingleInputPlanNode beforeMapper = (SingleInputPlanNode) mapper.getInput().getSource(); assertEquals(TempMode.NONE, mapper.getInput().getTempMode()); assertEquals(TempMode.NONE, beforeMapper.getInput().getTempMode()); assertEquals(TempMode.NONE, mapper.getBroadcastInputs().get(0).getTempMode()); assertEquals(DataExchangeMode.PIPELINED, mapper.getInput().getDataExchangeMode()); assertEquals(DataExchangeMode.BATCH, beforeMapper.getInput().getDataExchangeMode()); assertEquals(DataExchangeMode.BATCH, mapper.getBroadcastInputs().get(0).getDataExchangeMode());
Channel c1 = new Channel(child1, this.input1.getMaterializationMode()); if (this.input1.getShipStrategy() == null) { if (dopChange1 && !c1.getShipStrategy().isNetworkStrategy()) { c1.getGlobalProperties().reset(); DataExchangeMode exMode = DataExchangeMode.select(input1Mode, shipStrategy, input1breakPipeline); if (this.keys1 != null) { c1.setShipStrategy(this.input1.getShipStrategy(), this.keys1.toFieldList(), exMode); } else { c1.setShipStrategy(this.input1.getShipStrategy(), exMode); c1.adjustGlobalPropertiesForFullParallelismChange(); Channel c2 = new Channel(child2, this.input2.getMaterializationMode()); if (this.input2.getShipStrategy() == null) { if (dopChange2 && !c2.getShipStrategy().isNetworkStrategy()) { c2.getGlobalProperties().reset(); DataExchangeMode exMode = DataExchangeMode.select(input2Mode, shipStrategy, input2breakPipeline); if (this.keys2 != null) { c2.setShipStrategy(this.input2.getShipStrategy(), this.keys2.toFieldList(), exMode); } else { c2.setShipStrategy(this.input2.getShipStrategy(), exMode); c2.adjustGlobalPropertiesForFullParallelismChange(); GlobalProperties p1 = c1.getGlobalProperties();
@Override public SingleInputPlanNode instantiate(Channel in, SingleInputNode node) { // create in input node for combine with the same parallelism as input node GroupReduceNode combinerNode = new GroupReduceNode((GroupReduceOperatorBase<?, ?, ?>) node.getOperator()); combinerNode.setParallelism(in.getSource().getParallelism()); SingleInputPlanNode combiner = new SingleInputPlanNode(combinerNode, "Combine ("+node.getOperator().getName()+")", in, DriverStrategy.SORTED_GROUP_COMBINE); // sorting key info combiner.setDriverKeyInfo(in.getLocalStrategyKeys(), in.getLocalStrategySortOrder(), 0); // set grouping comparator key info combiner.setDriverKeyInfo(this.keyList, 1); return combiner; }
final Channel c = new Channel(child, this.inConn.getMaterializationMode()); igps.parameterizeChannel(c, parallelismChange, executionMode, breaksPipeline); if (parallelismChange && !c.getShipStrategy().isNetworkStrategy()) { c.getGlobalProperties().reset(); if (rgps.isMetBy(c.getGlobalProperties())) { c.setRequiredGlobalProps(rgps); addLocalCandidates(c, broadcastPlanChannels, igps, outputPlans, estimator); break; final Channel c = new Channel(child, this.inConn.getMaterializationMode()); final ShipStrategyType shipStrategy = this.inConn.getShipStrategy(); final DataExchangeMode exMode = DataExchangeMode.select(executionMode, shipStrategy, breaksPipeline); c.setShipStrategy(shipStrategy, this.keys.toFieldList(), exMode); } else { c.setShipStrategy(shipStrategy, exMode); c.adjustGlobalPropertiesForFullParallelismChange(); if (rgps.isMetBy(c.getGlobalProperties())) { addLocalCandidates(c, broadcastPlanChannels, rgps, outputPlans, estimator); break;
private List<Channel> rewriteRangePartitionChannel(Channel channel) { final List<Channel> sourceNewOutputChannels = new ArrayList<>(); final PlanNode sourceNode = channel.getSource(); final PlanNode targetNode = channel.getTarget(); final int sourceParallelism = sourceNode.getParallelism(); final int targetParallelism = targetNode.getParallelism(); final MapPartitionOperatorBase sipOperatorBase = new MapPartitionOperatorBase(sampleInPartition, sipOperatorInformation, SIP_NAME); final MapPartitionNode sipNode = new MapPartitionNode(sipOperatorBase); final Channel sipChannel = new Channel(sourceNode, TempMode.NONE); sipChannel.setShipStrategy(ShipStrategyType.FORWARD, DataExchangeMode.PIPELINED); final SingleInputPlanNode sipPlanNode = new SingleInputPlanNode(sipNode, SIP_NAME, sipChannel, DriverStrategy.MAP_PARTITION); sipNode.setParallelism(sourceParallelism); sipPlanNode.initProperties(new GlobalProperties(), new LocalProperties()); sipPlanNode.setCosts(defaultZeroCosts); sipChannel.setTarget(sipPlanNode); this.plan.getAllNodes().add(sipPlanNode); sourceNewOutputChannels.add(sipChannel); final GroupReduceOperatorBase sicOperatorBase = new GroupReduceOperatorBase(sampleInCoordinator, sicOperatorInformation, SIC_NAME); final GroupReduceNode sicNode = new GroupReduceNode(sicOperatorBase); final Channel sicChannel = new Channel(sipPlanNode, TempMode.NONE); sicChannel.setShipStrategy(ShipStrategyType.FORWARD, DataExchangeMode.PIPELINED); final SingleInputPlanNode sicPlanNode = new SingleInputPlanNode(sicNode, SIC_NAME, sicChannel, DriverStrategy.ALL_GROUP_REDUCE); sicNode.setParallelism(1); sicPlanNode.initProperties(new GlobalProperties(), new LocalProperties()); sicPlanNode.setCosts(defaultZeroCosts); sicChannel.setTarget(sicPlanNode);
private boolean checkBroadcastMergeJoin(DualInputPlanNode join, SingleInputPlanNode reducer) { if (DriverStrategy.INNER_MERGE == join.getDriverStrategy()) { // driver keys Assert.assertEquals(set0, join.getKeysForInput1()); Assert.assertEquals(set0, join.getKeysForInput2()); // local strategies Assert.assertEquals(LocalStrategy.SORT, join.getInput1().getLocalStrategy()); Assert.assertEquals(LocalStrategy.SORT, join.getInput2().getLocalStrategy()); Assert.assertEquals(LocalStrategy.COMBININGSORT, reducer.getInput().getLocalStrategy()); // local strategy keys Assert.assertEquals(set0, join.getInput1().getLocalStrategyKeys()); Assert.assertEquals(set0, join.getInput2().getLocalStrategyKeys()); Assert.assertTrue(Arrays.equals(join.getInput1().getLocalStrategySortOrder(), join.getInput2().getLocalStrategySortOrder())); Assert.assertEquals(set01, reducer.getInput().getLocalStrategyKeys()); Assert.assertEquals(set01, reducer.getKeys(0)); Assert.assertTrue(Arrays.equals(reducer.getInput().getLocalStrategySortOrder(), reducer.getSortOrders(0))); return true; } else { return false; } }
@Override public SourceAndDamReport hasDamOnPathDownTo(PlanNode source) { if (source == this) { return FOUND_SOURCE; } SourceAndDamReport res = this.initialInput.getSource().hasDamOnPathDownTo(source); if (res == FOUND_SOURCE_AND_DAM) { return FOUND_SOURCE_AND_DAM; } else if (res == FOUND_SOURCE) { return (this.initialInput.getLocalStrategy().dams() || this.initialInput.getTempMode().breaksPipeline() || getDriverStrategy().firstDam() == DamBehavior.FULL_DAM) ? FOUND_SOURCE_AND_DAM : FOUND_SOURCE; } else { return NOT_FOUND; } } }