new NettyConfig(LOCAL_ADDRESS, 0, segmentSize, slots, config)); return new NetworkEnvironment( bufferPool, nettyConnectionManager,
when(network.getResultPartitionManager()).thenReturn(partitionManager); when(network.getDefaultIOMode()).thenReturn(IOManager.IOMode.SYNC); when(network.createKvStateTaskRegistry(any(JobID.class), any(JobVertexID.class))) .thenReturn(mock(TaskKvStateRegistry.class)); when(network.getTaskEventDispatcher()).thenReturn(taskEventDispatcher);
public SerializingLongReceiver createReceiver() throws Exception { TaskManagerLocation senderLocation = new TaskManagerLocation( ResourceID.generate(), LOCAL_ADDRESS, senderEnv.getConnectionManager().getDataPort()); InputGate receiverGate = createInputGate( jobId, dataSetID, executionAttemptID, senderLocation, receiverEnv, channels); SerializingLongReceiver receiver = new SerializingLongReceiver(receiverGate, channels * partitionIds.length); receiver.start(); return receiver; }
protected ResultPartitionWriter createResultPartition( JobID jobId, ResultPartitionID partitionId, NetworkEnvironment environment, int channels) throws Exception { ResultPartition resultPartition = new ResultPartition( "sender task", new NoOpTaskActions(), jobId, partitionId, ResultPartitionType.PIPELINED_BOUNDED, channels, 1, environment.getResultPartitionManager(), new NoOpResultPartitionConsumableNotifier(), ioManager, false); environment.setupPartition(resultPartition); return resultPartition; }
TaskEventDispatcher taskEventDispatcher = new TaskEventDispatcher(); NetworkEnvironment network = mock(NetworkEnvironment.class); when(network.createKvStateTaskRegistry(any(JobID.class), any(JobVertexID.class))).thenReturn(mockKvRegistry); when(network.getTaskEventDispatcher()).thenReturn(taskEventDispatcher);
icdd.length, taskActions, metrics, networkEnvironment.isCreditBased()); networkEnvironment.getResultPartitionManager(), networkEnvironment.getTaskEventDispatcher(), networkEnvironment.getPartitionRequestInitialBackoff(), networkEnvironment.getPartitionRequestMaxBackoff(), metrics ); inputChannels[i] = new RemoteInputChannel(inputGate, i, partitionId, partitionLocation.getConnectionId(), networkEnvironment.getConnectionManager(), networkEnvironment.getPartitionRequestInitialBackoff(), networkEnvironment.getPartitionRequestMaxBackoff(), metrics ); networkEnvironment.getResultPartitionManager(), networkEnvironment.getTaskEventDispatcher(), networkEnvironment.getConnectionManager(), networkEnvironment.getPartitionRequestInitialBackoff(), networkEnvironment.getPartitionRequestMaxBackoff(), metrics );
network.registerTask(this); TaskKvStateRegistry kvStateRegistry = network.createKvStateTaskRegistry(jobId, getJobVertexId()); producedPartitions, inputGates, network.getTaskEventDispatcher(), checkpointResponder, taskManagerConfig, network.unregisterTask(this);
network.start(); network.getConnectionManager().getDataPort());
network.registerTask(this); .createKvStateTaskRegistry(jobId, getJobVertexId()); network.unregisterTask(this);
@Override public Integer getValue() { return network.getNetworkBufferPool().getNumberOfAvailableMemorySegments(); } });
@Override public void failPartition(ExecutionAttemptID executionAttemptID) { log.info("Discarding the results produced by task execution {}.", executionAttemptID); try { networkEnvironment.getResultPartitionManager().releasePartitionsProducedBy(executionAttemptID); } catch (Throwable t) { // TODO: Do we still need this catch branch? onFatalError(t); } // TODO: Maybe it's better to return an Acknowledge here to notify the JM about the success/failure with an Exception }
icdd.length, taskActions, metrics, networkEnvironment.isCreditBased()); networkEnvironment.getResultPartitionManager(), networkEnvironment.getTaskEventDispatcher(), networkEnvironment.getPartitionRequestInitialBackoff(), networkEnvironment.getPartitionRequestMaxBackoff(), metrics ); inputChannels[i] = new RemoteInputChannel(inputGate, i, partitionId, partitionLocation.getConnectionId(), networkEnvironment.getConnectionManager(), networkEnvironment.getPartitionRequestInitialBackoff(), networkEnvironment.getPartitionRequestMaxBackoff(), metrics ); networkEnvironment.getResultPartitionManager(), networkEnvironment.getTaskEventDispatcher(), networkEnvironment.getConnectionManager(), networkEnvironment.getPartitionRequestInitialBackoff(), networkEnvironment.getPartitionRequestMaxBackoff(), metrics );
network.registerTask(this); TaskKvStateRegistry kvStateRegistry = network.createKvStateTaskRegistry(jobId, getJobVertexId()); producedPartitions, inputGates, network.getTaskEventDispatcher(), checkpointResponder, taskManagerConfig, network.unregisterTask(this);
network.start(); network.getConnectionManager().getDataPort());
when(networkEnvironment.createKvStateTaskRegistry(any(JobID.class), any(JobVertexID.class))) .thenReturn(mock(TaskKvStateRegistry.class)); when(networkEnvironment.getTaskEventDispatcher()).thenReturn(taskEventDispatcher);
@Override public Long getValue() { return (long) network.getNetworkBufferPool().getTotalNumberOfMemorySegments(); } });
@Override public void failPartition(ExecutionAttemptID executionAttemptID) { log.info("Discarding the results produced by task execution {}.", executionAttemptID); try { networkEnvironment.getResultPartitionManager().releasePartitionsProducedBy(executionAttemptID); } catch (Throwable t) { // TODO: Do we still need this catch branch? onFatalError(t); } // TODO: Maybe it's better to return an Acknowledge here to notify the JM about the success/failure with an Exception }
owningTaskName, jobId, consumedResultId, consumedPartitionType, consumedSubpartitionIndex, icdd.length, taskActions, metrics, partitionRequestManager, executorService, networkEnvironment.isCreditBased(), isPartitionRequestRestricted); networkEnvironment.getResultPartitionManager(), networkEnvironment.getTaskEventDispatcher(), networkEnvironment.getPartitionRequestInitialBackoff(), networkEnvironment.getPartitionRequestMaxBackoff(), metrics ); inputChannels[i] = new RemoteInputChannel(inputGate, i, partitionId, partitionLocation.getConnectionId(), networkEnvironment.getConnectionManager(), networkEnvironment.getPartitionRequestInitialBackoff(), networkEnvironment.getPartitionRequestMaxBackoff(), metrics ); networkEnvironment.getResultPartitionManager(), networkEnvironment.getTaskEventDispatcher(), networkEnvironment.getConnectionManager(), networkEnvironment.getPartitionRequestInitialBackoff(), networkEnvironment.getPartitionRequestMaxBackoff(), metrics );
network.registerTask(this); TaskKvStateRegistry kvStateRegistry = network.createKvStateTaskRegistry(jobId, getJobVertexId()); producedPartitions, inputGates, network.getTaskEventDispatcher(), checkpointResponder, taskManagerConfig, network.unregisterTask(this);