bufferPool, nettyConnectionManager, new ResultPartitionManager(), new TaskEventDispatcher(), new KvStateRegistry(),
/** * Notification when a subpartition is released. */ void onConsumedSubpartition(int subpartitionIndex) { if (isReleased.get()) { return; } int refCnt = pendingReferences.decrementAndGet(); if (refCnt == 0) { partitionManager.onConsumedPartition(this); } else if (refCnt < 0) { throw new IllegalStateException("All references released."); } LOG.debug("{}: Received release notification for subpartition {} (reference count now at: {}).", this, subpartitionIndex, pendingReferences); }
public void releasePartitionsProducedBy(ExecutionAttemptID executionId) { releasePartitionsProducedBy(executionId, null); }
@VisibleForTesting public void setupPartition(ResultPartition partition) throws IOException { BufferPool bufferPool = null; try { int maxNumberOfMemorySegments = partition.getPartitionType().isBounded() ? partition.getNumberOfSubpartitions() * networkBuffersPerChannel + extraNetworkBuffersPerGate : Integer.MAX_VALUE; // If the partition type is back pressure-free, we register with the buffer pool for // callbacks to release memory. bufferPool = networkBufferPool.createBufferPool(partition.getNumberOfSubpartitions(), maxNumberOfMemorySegments, partition.getPartitionType().hasBackPressure() ? Optional.empty() : Optional.of(partition)); partition.registerBufferPool(bufferPool); resultPartitionManager.registerResultPartition(partition); } catch (Throwable t) { if (bufferPool != null) { bufferPool.lazyDestroy(); } if (t instanceof IOException) { throw (IOException) t; } else { throw new IOException(t.getMessage(), t); } } taskEventDispatcher.registerPartition(partition.getPartitionId()); }
ResultSubpartitionView subpartitionView = partitionManager.createSubpartitionView( partitionId, subpartitionIndex, this);
resultPartitionManager.shutdown();
@VisibleForTesting public void setupPartition(InternalResultPartition partition) throws IOException { BufferPool bufferPool = null; try { int extraNetworkBufferThisPartition = extraNetworkBuffersPerGate >= 0 ? extraNetworkBuffersPerGate : partition.getNumberOfSubpartitions() * networkBuffersPerSubpartition; int numberOfMemorySegments = partition.getNumberOfSubpartitions() * networkBuffersPerSubpartition + extraNetworkBufferThisPartition; bufferPool = networkBufferPool.createBufferPool(numberOfMemorySegments, numberOfMemorySegments); partition.registerBufferPool(bufferPool); resultPartitionManager.registerResultPartition(partition); } catch (Throwable t) { if (bufferPool != null) { bufferPool.lazyDestroy(); } if (t instanceof IOException) { throw (IOException) t; } else { throw new IOException(t.getMessage(), t); } } taskEventDispatcher.registerPartition(partition.getPartitionId()); }
ResultSubpartitionView subpartitionView = partitionManager.createSubpartitionView( partitionId, subpartitionIndex, this);
resultPartitionManager.shutdown();
public void releasePartitionsProducedBy(ExecutionAttemptID executionId) { releasePartitionsProducedBy(executionId, null); }
public NetworkEnvironment( int numBuffers, int memorySegmentSize, int partitionRequestInitialBackoff, int partitionRequestMaxBackoff, int networkBuffersPerChannel, int extraNetworkBuffersPerGate, boolean enableCreditBased) { this( new NetworkBufferPool(numBuffers, memorySegmentSize), new LocalConnectionManager(), new ResultPartitionManager(), new TaskEventDispatcher(), new KvStateRegistry(), null, null, IOManager.IOMode.SYNC, partitionRequestInitialBackoff, partitionRequestMaxBackoff, networkBuffersPerChannel, extraNetworkBuffersPerGate, enableCreditBased); }
/** * Notification when a subpartition is released. */ void onConsumedSubpartition(int subpartitionIndex) { if (isReleased.get()) { return; } int refCnt = pendingReferences.decrementAndGet(); if (refCnt == 0) { partitionManager.onConsumedPartition(this); } else if (refCnt < 0) { throw new IllegalStateException("All references released."); } LOG.debug("{}: Received release notification for subpartition {} (reference count now at: {}).", this, subpartitionIndex, pendingReferences); }
@VisibleForTesting public void setupPartition(ResultPartition partition) throws IOException { BufferPool bufferPool = null; try { int maxNumberOfMemorySegments = partition.getPartitionType().isBounded() ? partition.getNumberOfSubpartitions() * networkBuffersPerChannel + extraNetworkBuffersPerGate : Integer.MAX_VALUE; // If the partition type is back pressure-free, we register with the buffer pool for // callbacks to release memory. bufferPool = networkBufferPool.createBufferPool(partition.getNumberOfSubpartitions(), maxNumberOfMemorySegments, partition.getPartitionType().hasBackPressure() ? Optional.empty() : Optional.of(partition)); partition.registerBufferPool(bufferPool); resultPartitionManager.registerResultPartition(partition); } catch (Throwable t) { if (bufferPool != null) { bufferPool.lazyDestroy(); } if (t instanceof IOException) { throw (IOException) t; } else { throw new IOException(t.getMessage(), t); } } taskEventDispatcher.registerPartition(partition.getPartitionId()); }
ResultSubpartitionView subpartitionView = partitionManager.createSubpartitionView( partitionId, subpartitionIndex, this);
resultPartitionManager.shutdown();
public void releasePartitionsProducedBy(ExecutionAttemptID executionId) { releasePartitionsProducedBy(executionId, null); }
public NetworkEnvironment( int numBuffers, int memorySegmentSize, int partitionRequestInitialBackoff, int partitionRequestMaxBackoff, int networkBuffersPerChannel, int extraNetworkBuffersPerGate, boolean enableCreditBased) { this( new NetworkBufferPool(numBuffers, memorySegmentSize), new LocalConnectionManager(), new ResultPartitionManager(), new TaskEventDispatcher(), new KvStateRegistry(), null, null, IOManager.IOMode.SYNC, partitionRequestInitialBackoff, partitionRequestMaxBackoff, networkBuffersPerChannel, extraNetworkBuffersPerGate, enableCreditBased); }
/** * Notification when a subpartition is released. */ void onConsumedSubpartition(int subpartitionIndex) { if (isReleased.get()) { return; } int refCnt = pendingReferences.decrementAndGet(); if (refCnt == 0) { partitionManager.onConsumedPartition(this); } else if (refCnt < 0) { throw new IllegalStateException("All references released."); } LOG.debug("{}: Received release notification for subpartition {} (reference count now at: {}).", this, subpartitionIndex, pendingReferences); }
partition.registerBufferPool(bufferPool); resultPartitionManager.registerResultPartition(partition); } catch (Throwable t) { if (bufferPool != null) {
ResultSubpartitionView subpartitionView = partitionManager.createSubpartitionView( partitionId, subpartitionIndex, this);