public ResultPartitionID getPartitionId() { return partition.getPartitionId(); }
@Override public String toString() { boolean hasSpilled = spilledView != null; return String.format("SpillableSubpartitionView(index: %d, buffers: %d, spilled? %b) of ResultPartition %s", parent.index, numBuffers, hasSpilled, parent.parent.getPartitionId()); } }
@Override public String toString() { return String.format("PipelinedSubpartitionView(index: %d) of ResultPartition %s", parent.index, parent.parent.getPartitionId()); } }
@Override public String toString() { boolean hasSpilled = spilledView != null; return String.format("SpillableSubpartitionView(index: %d, buffers: %d, spilled? %b) of ResultPartition %s", parent.index, numBuffers, hasSpilled, parent.parent.getPartitionId()); } }
@Override public String toString() { return String.format("PipelinedSubpartitionView(index: %d) of ResultPartition %s", parent.index, parent.parent.getPartitionId()); } }
@Override public String toString() { return String.format("PipelinedSubpartitionView(index: %d) of ResultPartition %s", parent.index, parent.parent.getPartitionId()); } }
@Override public String toString() { boolean hasSpilled = spilledView != null; return String.format("SpillableSubpartitionView(index: %d, buffers: %d, spilled? %b) of ResultPartition %s", parent.index, numBuffers, hasSpilled, parent.parent.getPartitionId()); } }
@Override public String toString() { return String.format("SpilledSubpartitionView(index: %d, buffers: %d) of ResultPartition %s", parent.index, numberOfSpilledBuffers, parent.parent.getPartitionId()); }
@Override public String toString() { return String.format("SpilledSubpartitionView(index: %d, buffers: %d) of ResultPartition %s", parent.index, numberOfSpilledBuffers, parent.parent.getPartitionId()); }
@Override public String toString() { return String.format("SpilledSubpartitionView(index: %d, buffers: %d) of ResultPartition %s", parent.index, numberOfSpilledBuffers, parent.parent.getPartitionId()); }
@Override public PipelinedSubpartitionView createReadView(BufferAvailabilityListener availabilityListener) throws IOException { final int queueSize; synchronized (buffers) { checkState(!isReleased); checkState(readView == null, "Subpartition %s of is being (or already has been) consumed, " + "but pipelined subpartitions can only be consumed once.", index, parent.getPartitionId()); LOG.debug("Creating read view for subpartition {} of partition {}.", index, parent.getPartitionId()); queueSize = buffers.size(); readView = new PipelinedSubpartitionView(this, availabilityListener); } readView.notifyBuffersAvailable(queueSize); return readView; }
public void registerResultPartition(ResultPartition partition) throws IOException { synchronized (registeredPartitions) { checkState(!isShutdown, "Result partition manager already shut down."); ResultPartitionID partitionId = partition.getPartitionId(); ResultPartition previous = registeredPartitions.put( partitionId.getProducerId(), partitionId.getPartitionId(), partition); if (previous != null) { throw new IllegalStateException("Result partition already registered."); } LOG.debug("Registered {}.", partition); } }
public void registerResultPartition(ResultPartition partition) throws IOException { synchronized (registeredPartitions) { checkState(!isShutdown, "Result partition manager already shut down."); ResultPartitionID partitionId = partition.getPartitionId(); ResultPartition previous = registeredPartitions.put( partitionId.getProducerId(), partitionId.getPartitionId(), partition); if (previous != null) { throw new IllegalStateException("Result partition already registered."); } LOG.debug("Registered {}.", partition); } }
public void registerResultPartition(ResultPartition partition) throws IOException { synchronized (registeredPartitions) { checkState(!isShutdown, "Result partition manager already shut down."); ResultPartitionID partitionId = partition.getPartitionId(); ResultPartition previous = registeredPartitions.put( partitionId.getProducerId(), partitionId.getPartitionId(), partition); if (previous != null) { throw new IllegalStateException("Result partition already registered."); } LOG.debug("Registered {}.", partition); } }
@Override public PipelinedSubpartitionView createReadView(BufferAvailabilityListener availabilityListener) throws IOException { final boolean notifyDataAvailable; synchronized (buffers) { checkState(!isReleased); checkState(readView == null, "Subpartition %s of is being (or already has been) consumed, " + "but pipelined subpartitions can only be consumed once.", index, parent.getPartitionId()); LOG.debug("{}: Creating read view for subpartition {} of partition {}.", parent.getOwningTaskName(), index, parent.getPartitionId()); readView = new PipelinedSubpartitionView(this, availabilityListener); notifyDataAvailable = !buffers.isEmpty(); } if (notifyDataAvailable) { notifyDataAvailable(); } return readView; }
@Override public PipelinedSubpartitionView createReadView(BufferAvailabilityListener availabilityListener) throws IOException { final boolean notifyDataAvailable; synchronized (buffers) { checkState(!isReleased); checkState(readView == null, "Subpartition %s of is being (or already has been) consumed, " + "but pipelined subpartitions can only be consumed once.", index, parent.getPartitionId()); LOG.debug("{}: Creating read view for subpartition {} of partition {}.", parent.getOwningTaskName(), index, parent.getPartitionId()); readView = new PipelinedSubpartitionView(this, availabilityListener); notifyDataAvailable = !buffers.isEmpty(); } if (notifyDataAvailable) { notifyDataAvailable(); } return readView; }
void onConsumedPartition(ResultPartition partition) { final ResultPartition previous; LOG.debug("Received consume notification from {}.", partition); synchronized (registeredPartitions) { ResultPartitionID partitionId = partition.getPartitionId(); previous = registeredPartitions.remove(partitionId.getProducerId(), partitionId.getPartitionId()); } // Release the partition if it was successfully removed if (partition == previous) { partition.release(); LOG.debug("Released {}.", partition); } } }
void onConsumedPartition(ResultPartition partition) { final ResultPartition previous; LOG.debug("Received consume notification from {}.", partition); synchronized (registeredPartitions) { ResultPartitionID partitionId = partition.getPartitionId(); previous = registeredPartitions.remove(partitionId.getProducerId(), partitionId.getPartitionId()); } // Release the partition if it was successfully removed if (partition == previous) { partition.release(); LOG.debug("Released {}.", partition); } } }
void onConsumedPartition(ResultPartition partition) { final ResultPartition previous; LOG.debug("Received consume notification from {}.", partition); synchronized (registeredPartitions) { ResultPartitionID partitionId = partition.getPartitionId(); previous = registeredPartitions.remove(partitionId.getProducerId(), partitionId.getPartitionId()); } // Release the partition if it was successfully removed if (partition == previous) { partition.release(); LOG.debug("Released {}.", partition); } } }
@Override public int releaseMemory() throws IOException { synchronized (buffers) { ResultSubpartitionView view = readView; if (view != null && view.getClass() == SpillableSubpartitionView.class) { // If there is a spillable view, it's the responsibility of the // view to release memory. SpillableSubpartitionView spillableView = (SpillableSubpartitionView) view; return spillableView.releaseMemory(); } else if (spillWriter == null) { // No view and in-memory => spill to disk spillWriter = ioManager.createBufferFileWriter(ioManager.createChannel()); int numberOfBuffers = buffers.size(); long spilledBytes = spillFinishedBufferConsumers(isFinished); int spilledBuffers = numberOfBuffers - buffers.size(); LOG.debug("{}: Spilling {} bytes ({} buffers} for sub partition {} of {}.", parent.getOwningTaskName(), spilledBytes, spilledBuffers, index, parent.getPartitionId()); return spilledBuffers; } } // Else: We have already spilled and don't hold any buffers return 0; }