protected ResultPartitionWriter createResultPartition( JobID jobId, ResultPartitionID partitionId, NetworkEnvironment environment, int channels) throws Exception { ResultPartition resultPartition = new ResultPartition( "sender task", new NoOpTaskActions(), jobId, partitionId, ResultPartitionType.PIPELINED_BOUNDED, channels, 1, environment.getResultPartitionManager(), new NoOpResultPartitionConsumableNotifier(), ioManager, false); environment.setupPartition(resultPartition); return resultPartition; }
public SerializingLongReceiver createReceiver() throws Exception { TaskManagerLocation senderLocation = new TaskManagerLocation( ResourceID.generate(), LOCAL_ADDRESS, senderEnv.getConnectionManager().getDataPort()); InputGate receiverGate = createInputGate( jobId, dataSetID, executionAttemptID, senderLocation, receiverEnv, channels); SerializingLongReceiver receiver = new SerializingLongReceiver(receiverGate, channels * partitionIds.length); receiver.start(); return receiver; }
final NetworkBufferPool bufferPool = new NetworkBufferPool(bufferPoolSize, segmentSize); final NettyConnectionManager nettyConnectionManager = new NettyConnectionManager( new NettyConfig(LOCAL_ADDRESS, 0, segmentSize, slots, config)); return new NetworkEnvironment( bufferPool, nettyConnectionManager, new ResultPartitionManager(), new TaskEventDispatcher(), new KvStateRegistry(), null,
TaskEventDispatcher taskEventDispatcher = new TaskEventDispatcher(); NetworkEnvironment network = mock(NetworkEnvironment.class); when(network.createKvStateTaskRegistry(any(JobID.class), any(JobVertexID.class))).thenReturn(mockKvRegistry); when(network.getTaskEventDispatcher()).thenReturn(taskEventDispatcher); new TestingTaskManagerRuntimeInfo(), UnregisteredMetricGroups.createUnregisteredTaskMetricGroup(), new NoOpResultPartitionConsumableNotifier(), mock(PartitionProducerStateChecker.class), Executors.directExecutor());
.map(partitionId -> new InputChannelDeploymentDescriptor( partitionId, localMode ? ResultPartitionLocation.createLocal() : ResultPartitionLocation.createRemote(new ConnectionID(senderLocation, finalChannel)))) .toArray(InputChannelDeploymentDescriptor[]::new); channelDescriptors); SingleInputGate gate = SingleInputGate.create( "receiving task[" + channel + "]", jobId, UnregisteredMetricGroups.createUnregisteredTaskMetricGroup().getIOMetricGroup()); environment.setupInputGate(gate); gates[channel] = gate; return new UnionInputGate(gates); } else { return gates[0];
ResultPartitionConsumableNotifier consumableNotifier = new NoOpResultPartitionConsumableNotifier(); PartitionProducerStateChecker partitionProducerStateChecker = mock(PartitionProducerStateChecker.class); Executor executor = mock(Executor.class); TaskEventDispatcher taskEventDispatcher = new TaskEventDispatcher(); when(network.getResultPartitionManager()).thenReturn(partitionManager); when(network.getDefaultIOMode()).thenReturn(IOManager.IOMode.SYNC); when(network.createKvStateTaskRegistry(any(JobID.class), any(JobVertexID.class))) .thenReturn(mock(TaskKvStateRegistry.class)); when(network.getTaskEventDispatcher()).thenReturn(taskEventDispatcher);
DataExchangeMode exMode = DataExchangeMode.select(exchangeMode, shipType, breakPipeline); channel.setShipStrategy(shipType, partitionKeys, sortDirection, partitioner, exMode);
@Override public Integer getValue() { return network.getNetworkBufferPool().getNumberOfAvailableMemorySegments(); } });
@Override public Long getValue() { return (long) network.getNetworkBufferPool().getTotalNumberOfMemorySegments(); } });
public void notifyCreditAvailable(RemoteInputChannel inputChannel) { clientHandler.notifyCreditAvailable(inputChannel); }
if (input.getDataExchangeMode().equals(DataExchangeMode.BATCH)) { in.setDataExchangeMode(DataExchangeMode.BATCH);
String exchangeMode = channel.getDataExchangeMode().toString(); writer.print(", \"exchange_mode\": \"" + exchangeMode + "\"");
int mode) throws IOException { TaskEventDispatcher taskEventDispatcher = new TaskEventDispatcher(); NetworkEnvironment networkEnvironment = mock(NetworkEnvironment.class); when(networkEnvironment.createKvStateTaskRegistry(any(JobID.class), any(JobVertexID.class))) .thenReturn(mock(TaskKvStateRegistry.class)); when(networkEnvironment.getTaskEventDispatcher()).thenReturn(taskEventDispatcher); new TestingTaskManagerRuntimeInfo(), UnregisteredMetricGroups.createUnregisteredTaskMetricGroup(), new NoOpResultPartitionConsumableNotifier(), mock(PartitionProducerStateChecker.class), mock(Executor.class));
ShipStrategyType.FORWARD; DataExchangeMode em = DataExchangeMode.select(exchangeMode, shipStrategy, breakPipeline); channel.setShipStrategy(shipStrategy, em); return; DataExchangeMode em = DataExchangeMode.select(exchangeMode, ShipStrategyType.FORWARD, breakPipeline); channel.setShipStrategy(ShipStrategyType.FORWARD, em); return; DataExchangeMode exMode = DataExchangeMode.select(exchangeMode, shipType, breakPipeline); channel.setShipStrategy(shipType, partitionKeys, sortDirection, partitioner, exMode);
public void notifyCreditAvailable(RemoteInputChannel inputChannel) { clientHandler.notifyCreditAvailable(inputChannel); }
TaskEventDispatcher taskEventDispatcher = new TaskEventDispatcher(); final NetworkEnvironment networkEnv = mock(NetworkEnvironment.class); when(networkEnv.createKvStateTaskRegistry(any(JobID.class), any(JobVertexID.class))).thenReturn(mock(TaskKvStateRegistry.class)); when(networkEnv.getTaskEventDispatcher()).thenReturn(taskEventDispatcher); taskManagerRuntimeInfo, UnregisteredMetricGroups.createUnregisteredTaskMetricGroup(), new NoOpResultPartitionConsumableNotifier(), mock(PartitionProducerStateChecker.class), Executors.directExecutor());
for (PlanNode plan: broadcastPlanCandidates) { NamedChannel c = new NamedChannel(broadcastConnectionName, plan); DataExchangeMode exMode = DataExchangeMode.select(broadcastConnection.getDataExchangeMode(), ShipStrategyType.BROADCAST, broadcastConnection.isBreakingPipeline()); c.setShipStrategy(ShipStrategyType.BROADCAST, exMode); final DataExchangeMode exMode = DataExchangeMode.select(executionMode, shipStrategy, breaksPipeline);
for (PlanNode plan: broadcastPlanCandidates) { final NamedChannel c = new NamedChannel(broadcastConnectionName, plan); DataExchangeMode exMode = DataExchangeMode.select(broadcastConnection.getDataExchangeMode(), ShipStrategyType.BROADCAST, broadcastConnection.isBreakingPipeline()); c.setShipStrategy(ShipStrategyType.BROADCAST, exMode); DataExchangeMode exMode = DataExchangeMode.select(input1Mode, shipType, input1breaksPipeline); if (this.keys1 != null) { c1.setShipStrategy(shipType, this.keys1.toFieldList(), exMode); DataExchangeMode exMode = DataExchangeMode.select(input2Mode, shipType, input2breaksPipeline); if (this.keys2 != null) { c2.setShipStrategy(shipType, this.keys2.toFieldList(), exMode);
DataExchangeMode exMode = DataExchangeMode.select(input1Mode, shipStrategy, input1breakPipeline); if (this.keys1 != null) { c1.setShipStrategy(this.input1.getShipStrategy(), this.keys1.toFieldList(), exMode); DataExchangeMode exMode = DataExchangeMode.select(input2Mode, shipStrategy, input2breakPipeline); if (this.keys2 != null) { c2.setShipStrategy(this.input2.getShipStrategy(), this.keys2.toFieldList(), exMode);