sourceNewOutputChannels.add(ariChannel); final NamedChannel broadcastChannel = new NamedChannel("RangeBoundaries", rbPlanNode); broadcastChannel.setShipStrategy(ShipStrategyType.BROADCAST, DataExchangeMode.PIPELINED); broadcastChannel.setTarget(ariPlanNode); List<NamedChannel> broadcastChannels = new ArrayList<>(1); broadcastChannels.add(broadcastChannel);
for (NamedChannel broadcastInput: node.getBroadcastInputs()) { int broadcastInputIndexDelta = translateChannel(broadcastInput, broadcastInputIndex, targetVertex, targetVertexConfig, true); targetVertexConfig.setBroadcastInputName(broadcastInput.getName(), broadcastInputIndex); targetVertexConfig.setBroadcastInputSerializer(broadcastInput.getSerializer(), broadcastInputIndex); broadcastInputIndex += broadcastInputIndexDelta;
/** * Sets a list of all broadcast inputs attached to this node. */ public void setBroadcastInputs(List<NamedChannel> broadcastInputs) { if (broadcastInputs != null) { this.broadcastInputs = broadcastInputs; // update the branch map for (NamedChannel nc : broadcastInputs) { PlanNode source = nc.getSource(); mergeBranchPlanMaps(branchPlan, source.branchPlan); } } // do a sanity check that if we are branching, we have now candidates for each branch point if (this.template.hasUnclosedBranches()) { if (this.branchPlan == null) { throw new CompilerException("Branching and rejoining logic did not find a candidate for the branching point."); } for (UnclosedBranchDescriptor uc : this.template.getOpenBranches()) { OptimizerNode brancher = uc.getBranchingNode(); if (this.branchPlan.get(brancher) == null) { throw new CompilerException("Branching and rejoining logic did not find a candidate for the branching point."); } } } }
NamedChannel c = new NamedChannel(broadcastConnectionName, plan); DataExchangeMode exMode = DataExchangeMode.select(broadcastConnection.getDataExchangeMode(), ShipStrategyType.BROADCAST, broadcastConnection.isBreakingPipeline()); c.setShipStrategy(ShipStrategyType.BROADCAST, exMode); broadcastChannels.add(c);
@Test public void testNoBreakerForIndependentVariable() { try { ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment(); DataSet<String> source1 = env.fromElements("test"); DataSet<String> source2 = env.fromElements("test"); source1.map(new IdentityMapper<String>()).withBroadcastSet(source2, "some name") .output(new DiscardingOutputFormat<String>()); Plan p = env.createProgramPlan(); OptimizedPlan op = compileNoStats(p); SinkPlanNode sink = op.getDataSinks().iterator().next(); SingleInputPlanNode mapper = (SingleInputPlanNode) sink.getInput().getSource(); assertEquals(TempMode.NONE, mapper.getInput().getTempMode()); assertEquals(TempMode.NONE, mapper.getBroadcastInputs().get(0).getTempMode()); assertEquals(DataExchangeMode.PIPELINED, mapper.getInput().getDataExchangeMode()); assertEquals(DataExchangeMode.PIPELINED, mapper.getBroadcastInputs().get(0).getDataExchangeMode()); } catch (Exception e) { e.printStackTrace(); fail(e.getMessage()); } }
final NamedChannel c = new NamedChannel(broadcastConnectionName, plan); DataExchangeMode exMode = DataExchangeMode.select(broadcastConnection.getDataExchangeMode(), ShipStrategyType.BROADCAST, broadcastConnection.isBreakingPipeline()); c.setShipStrategy(ShipStrategyType.BROADCAST, exMode); broadcastChannels.add(c);
assertEquals(TempMode.NONE, mapper.getBroadcastInputs().get(0).getTempMode()); assertEquals(DataExchangeMode.BATCH, mapper.getBroadcastInputs().get(0).getDataExchangeMode());
sourceNewOutputChannels.add(ariChannel); final NamedChannel broadcastChannel = new NamedChannel("RangeBoundaries", rbPlanNode); broadcastChannel.setShipStrategy(ShipStrategyType.BROADCAST, DataExchangeMode.PIPELINED); broadcastChannel.setTarget(ariPlanNode); List<NamedChannel> broadcastChannels = new ArrayList<>(1); broadcastChannels.add(broadcastChannel);
NamedChannel c = new NamedChannel(broadcastConnectionName, plan); DataExchangeMode exMode = DataExchangeMode.select(broadcastConnection.getDataExchangeMode(), ShipStrategyType.BROADCAST, broadcastConnection.isBreakingPipeline()); c.setShipStrategy(ShipStrategyType.BROADCAST, exMode); broadcastChannels.add(c);
Costs bcInputCost = nc.getSource().getCumulativeCostsShare(); if (bcInputCost != null) { this.cumulativeCosts.addCosts(bcInputCost);
for (NamedChannel broadcastInput: node.getBroadcastInputs()) { int broadcastInputIndexDelta = translateChannel(broadcastInput, broadcastInputIndex, targetVertex, targetVertexConfig, true); targetVertexConfig.setBroadcastInputName(broadcastInput.getName(), broadcastInputIndex); targetVertexConfig.setBroadcastInputSerializer(broadcastInput.getSerializer(), broadcastInputIndex); broadcastInputIndex += broadcastInputIndexDelta;
assertEquals(TempMode.NONE, mapper.getBroadcastInputs().get(0).getTempMode()); assertEquals(DataExchangeMode.BATCH, mapper.getBroadcastInputs().get(0).getDataExchangeMode());
sourceNewOutputChannels.add(ariChannel); final NamedChannel broadcastChannel = new NamedChannel("RangeBoundaries", rbPlanNode); broadcastChannel.setShipStrategy(ShipStrategyType.BROADCAST, DataExchangeMode.PIPELINED); broadcastChannel.setTarget(ariPlanNode); List<NamedChannel> broadcastChannels = new ArrayList<>(1); broadcastChannels.add(broadcastChannel);
NamedChannel c = new NamedChannel(broadcastConnectionName, plan); DataExchangeMode exMode = DataExchangeMode.select(broadcastConnection.getDataExchangeMode(), ShipStrategyType.BROADCAST, broadcastConnection.isBreakingPipeline()); c.setShipStrategy(ShipStrategyType.BROADCAST, exMode); broadcastChannels.add(c);
for (NamedChannel broadcastInput: node.getBroadcastInputs()) { int broadcastInputIndexDelta = translateChannel(broadcastInput, broadcastInputIndex, targetVertex, targetVertexConfig, true); targetVertexConfig.setBroadcastInputName(broadcastInput.getName(), broadcastInputIndex); targetVertexConfig.setBroadcastInputSerializer(broadcastInput.getSerializer(), broadcastInputIndex); broadcastInputIndex += broadcastInputIndexDelta;
sourceNewOutputChannels.add(ariChannel); final NamedChannel broadcastChannel = new NamedChannel("RangeBoundaries", rbPlanNode); broadcastChannel.setShipStrategy(ShipStrategyType.BROADCAST, DataExchangeMode.PIPELINED); broadcastChannel.setTarget(ariPlanNode); List<NamedChannel> broadcastChannels = new ArrayList<>(1); broadcastChannels.add(broadcastChannel);
NamedChannel c = new NamedChannel(broadcastConnectionName, plan); DataExchangeMode exMode = DataExchangeMode.select(broadcastConnection.getDataExchangeMode(), ShipStrategyType.BROADCAST, broadcastConnection.isBreakingPipeline()); c.setShipStrategy(ShipStrategyType.BROADCAST, exMode); broadcastChannels.add(c);
@Override public SourceAndDamReport hasDamOnPathDownTo(PlanNode source) { if (source == this) { return FOUND_SOURCE; } SourceAndDamReport res = this.input.getSource().hasDamOnPathDownTo(source); if (res == FOUND_SOURCE_AND_DAM) { return FOUND_SOURCE_AND_DAM; } else if (res == FOUND_SOURCE) { return (this.input.getLocalStrategy().dams() || this.input.getTempMode().breaksPipeline() || getDriverStrategy().firstDam() == DamBehavior.FULL_DAM) ? FOUND_SOURCE_AND_DAM : FOUND_SOURCE; } else { // NOT_FOUND // check the broadcast inputs for (NamedChannel nc : getBroadcastInputs()) { SourceAndDamReport bcRes = nc.getSource().hasDamOnPathDownTo(source); if (bcRes != NOT_FOUND) { // broadcast inputs are always dams return FOUND_SOURCE_AND_DAM; } } return NOT_FOUND; } }
for (NamedChannel broadcastInput: node.getBroadcastInputs()) { int broadcastInputIndexDelta = translateChannel(broadcastInput, broadcastInputIndex, targetVertex, targetVertexConfig, true); targetVertexConfig.setBroadcastInputName(broadcastInput.getName(), broadcastInputIndex); targetVertexConfig.setBroadcastInputSerializer(broadcastInput.getSerializer(), broadcastInputIndex); broadcastInputIndex += broadcastInputIndexDelta;