public ResultPartitionID getPartitionId() { return partition.getPartitionId(); }
public void release() { release(null); }
/** * Finishes the result partition. * * <p>After this operation, it is not possible to add further data to the result partition. * * <p>For BLOCKING results, this will trigger the deployment of consuming tasks. */ public void finish() throws IOException { boolean success = false; try { checkInProduceState(); for (ResultSubpartition subpartition : subpartitions) { subpartition.finish(); } success = true; } finally { if (success) { isFinished = true; notifyPipelinedConsumers(); } } }
@VisibleForTesting public void setupPartition(ResultPartition partition) throws IOException { BufferPool bufferPool = null; try { int maxNumberOfMemorySegments = partition.getPartitionType().isBounded() ? partition.getNumberOfSubpartitions() * networkBuffersPerChannel + extraNetworkBuffersPerGate : Integer.MAX_VALUE; // If the partition type is back pressure-free, we register with the buffer pool for // callbacks to release memory. bufferPool = networkBufferPool.createBufferPool(partition.getNumberOfSubpartitions(), maxNumberOfMemorySegments, partition.getPartitionType().hasBackPressure() ? Optional.empty() : Optional.of(partition)); partition.registerBufferPool(bufferPool); resultPartitionManager.registerResultPartition(partition); } catch (Throwable t) { if (bufferPool != null) { bufferPool.lazyDestroy(); } if (t instanceof IOException) { throw (IOException) t; } else { throw new IOException(t.getMessage(), t); } } taskEventDispatcher.registerPartition(partition.getPartitionId()); }
void onConsumedPartition(ResultPartition partition) { final ResultPartition previous; LOG.debug("Received consume notification from {}.", partition); synchronized (registeredPartitions) { ResultPartitionID partitionId = partition.getPartitionId(); previous = registeredPartitions.remove(partitionId.getProducerId(), partitionId.getPartitionId()); } // Release the partition if it was successfully removed if (partition == previous) { partition.release(); LOG.debug("Released {}.", partition); } } }
@Override public PipelinedSubpartitionView createReadView(BufferAvailabilityListener availabilityListener) throws IOException { final boolean notifyDataAvailable; synchronized (buffers) { checkState(!isReleased); checkState(readView == null, "Subpartition %s of is being (or already has been) consumed, " + "but pipelined subpartitions can only be consumed once.", index, parent.getPartitionId()); LOG.debug("{}: Creating read view for subpartition {} of partition {}.", parent.getOwningTaskName(), index, parent.getPartitionId()); readView = new PipelinedSubpartitionView(this, availabilityListener); notifyDataAvailable = !buffers.isEmpty(); } if (notifyDataAvailable) { notifyDataAvailable(); } return readView; }
/** * Iterates over all sub-partitions and collects the maximum number of queued buffers in a * sub-partition in a best-effort way. * * @return maximum number of queued buffers per sub-partition */ int refreshAndGetMax() { int max = 0; for (ResultSubpartition part : partition.getAllPartitions()) { int size = part.unsynchronizedGetNumberOfQueuedBuffers(); max = Math.max(max, size); } return max; }
taskEventDispatcher.unregisterPartition(partition.getPartitionId()); partition.destroyBufferPool();
partition.finish();
public BufferProvider getBufferProvider() { return partition.getBufferProvider(); }
partition.destroyBufferPool(); } catch (Throwable t) { LOG.error("Failed to release result partition buffer pool for task {}.", taskName, t);
protected Throwable getFailureCause() { return parent.getFailureCause(); }
@Override public ResultSubpartitionView createSubpartitionView( ResultPartitionID partitionId, int subpartitionIndex, BufferAvailabilityListener availabilityListener) throws IOException { synchronized (registeredPartitions) { final ResultPartition partition = registeredPartitions.get(partitionId.getProducerId(), partitionId.getPartitionId()); if (partition == null) { throw new PartitionNotFoundException(partitionId); } LOG.debug("Requesting subpartition {} of {}.", subpartitionIndex, partition); return partition.createSubpartitionView(subpartitionIndex, availabilityListener); } }
@Override public Float getValue() { int usedBuffers = 0; int bufferPoolSize = 0; for (ResultPartition resultPartition : task.getProducedPartitions()) { usedBuffers += resultPartition.getBufferPool().bestEffortGetNumOfUsedBuffers(); bufferPoolSize += resultPartition.getBufferPool().getNumBuffers(); } if (bufferPoolSize != 0) { return ((float) usedBuffers) / bufferPoolSize; } else { return 0.0f; } } }
@VisibleForTesting public void setupPartition(ResultPartition partition) throws IOException { BufferPool bufferPool = null; try { int maxNumberOfMemorySegments = partition.getPartitionType().isBounded() ? partition.getNumberOfSubpartitions() * networkBuffersPerChannel + extraNetworkBuffersPerGate : Integer.MAX_VALUE; // If the partition type is back pressure-free, we register with the buffer pool for // callbacks to release memory. bufferPool = networkBufferPool.createBufferPool(partition.getNumberOfSubpartitions(), maxNumberOfMemorySegments, partition.getPartitionType().hasBackPressure() ? Optional.empty() : Optional.of(partition)); partition.registerBufferPool(bufferPool); resultPartitionManager.registerResultPartition(partition); } catch (Throwable t) { if (bufferPool != null) { bufferPool.lazyDestroy(); } if (t instanceof IOException) { throw (IOException) t; } else { throw new IOException(t.getMessage(), t); } } taskEventDispatcher.registerPartition(partition.getPartitionId()); }
void onConsumedPartition(ResultPartition partition) { final ResultPartition previous; LOG.debug("Received consume notification from {}.", partition); synchronized (registeredPartitions) { ResultPartitionID partitionId = partition.getPartitionId(); previous = registeredPartitions.remove(partitionId.getProducerId(), partitionId.getPartitionId()); } // Release the partition if it was successfully removed if (partition == previous) { partition.release(); LOG.debug("Released {}.", partition); } } }
@Override public PipelinedSubpartitionView createReadView(BufferAvailabilityListener availabilityListener) throws IOException { final boolean notifyDataAvailable; synchronized (buffers) { checkState(!isReleased); checkState(readView == null, "Subpartition %s of is being (or already has been) consumed, " + "but pipelined subpartitions can only be consumed once.", index, parent.getPartitionId()); LOG.debug("{}: Creating read view for subpartition {} of partition {}.", parent.getOwningTaskName(), index, parent.getPartitionId()); readView = new PipelinedSubpartitionView(this, availabilityListener); notifyDataAvailable = !buffers.isEmpty(); } if (notifyDataAvailable) { notifyDataAvailable(); } return readView; }
/** * Iterates over all sub-partitions and collects the maximum number of queued buffers in a * sub-partition in a best-effort way. * * @return maximum number of queued buffers per sub-partition */ int refreshAndGetMax() { int max = 0; for (ResultSubpartition part : partition.getAllPartitions()) { int size = part.unsynchronizedGetNumberOfQueuedBuffers(); max = Math.max(max, size); } return max; }
taskEventDispatcher.unregisterPartition(partition.getPartitionId()); partition.destroyBufferPool();
partition.finish();