@Override public void finish() throws IOException { add(EventSerializer.toBufferConsumer(EndOfPartitionEvent.INSTANCE), true); LOG.debug("{}: Finished {}.", parent.getOwningTaskName(), this); }
@Override public void finish() throws IOException { add(EventSerializer.toBufferConsumer(EndOfPartitionEvent.INSTANCE), true); LOG.debug("{}: Finished {}.", parent.getOwningTaskName(), this); }
@Override public int releaseMemory() throws IOException { synchronized (buffers) { ResultSubpartitionView view = readView; if (view != null && view.getClass() == SpillableSubpartitionView.class) { // If there is a spillable view, it's the responsibility of the // view to release memory. SpillableSubpartitionView spillableView = (SpillableSubpartitionView) view; return spillableView.releaseMemory(); } else if (spillWriter == null) { // No view and in-memory => spill to disk spillWriter = ioManager.createBufferFileWriter(ioManager.createChannel()); int numberOfBuffers = buffers.size(); long spilledBytes = spillFinishedBufferConsumers(isFinished); int spilledBuffers = numberOfBuffers - buffers.size(); LOG.debug("{}: Spilling {} bytes ({} buffers} for sub partition {} of {}.", parent.getOwningTaskName(), spilledBytes, spilledBuffers, index, parent.getPartitionId()); return spilledBuffers; } } // Else: We have already spilled and don't hold any buffers return 0; }
@Override public synchronized void finish() throws IOException { synchronized (buffers) { if (add(EventSerializer.toBufferConsumer(EndOfPartitionEvent.INSTANCE), true)) { isFinished = true; } flush(); } // If we are spilling/have spilled, wait for the writer to finish if (spillWriter != null) { spillWriter.close(); } LOG.debug("{}: Finished {}.", parent.getOwningTaskName(), this); }
@Override public synchronized void finish() throws IOException { synchronized (buffers) { if (add(EventSerializer.toBufferConsumer(EndOfPartitionEvent.INSTANCE), true)) { isFinished = true; } flush(); } // If we are spilling/have spilled, wait for the writer to finish if (spillWriter != null) { spillWriter.close(); } LOG.debug("{}: Finished {}.", parent.getOwningTaskName(), this); }
@Override public int releaseMemory() throws IOException { synchronized (buffers) { ResultSubpartitionView view = readView; if (view != null && view.getClass() == SpillableSubpartitionView.class) { // If there is a spillable view, it's the responsibility of the // view to release memory. SpillableSubpartitionView spillableView = (SpillableSubpartitionView) view; return spillableView.releaseMemory(); } else if (spillWriter == null) { // No view and in-memory => spill to disk spillWriter = ioManager.createBufferFileWriter(ioManager.createChannel()); int numberOfBuffers = buffers.size(); long spilledBytes = spillFinishedBufferConsumers(isFinished); int spilledBuffers = numberOfBuffers - buffers.size(); LOG.debug("{}: Spilling {} bytes ({} buffers} for sub partition {} of {}.", parent.getOwningTaskName(), spilledBytes, spilledBuffers, index, parent.getPartitionId()); return spilledBuffers; } } // Else: We have already spilled and don't hold any buffers return 0; }
@Override public void release() { // view reference accessible outside the lock, but assigned inside the locked scope final PipelinedSubpartitionView view; synchronized (buffers) { if (isReleased) { return; } // Release all available buffers for (BufferConsumer buffer : buffers) { buffer.close(); } buffers.clear(); view = readView; readView = null; // Make sure that no further buffers are added to the subpartition isReleased = true; } LOG.debug("{}: Released {}.", parent.getOwningTaskName(), this); if (view != null) { view.releaseAllResources(); } }
@Override public void release() { // view reference accessible outside the lock, but assigned inside the locked scope final PipelinedSubpartitionView view; synchronized (buffers) { if (isReleased) { return; } // Release all available buffers for (BufferConsumer buffer : buffers) { buffer.close(); } buffers.clear(); view = readView; readView = null; // Make sure that no further buffers are added to the subpartition isReleased = true; } LOG.debug("{}: Released {}.", parent.getOwningTaskName(), this); if (view != null) { view.releaseAllResources(); } }
@Override public PipelinedSubpartitionView createReadView(BufferAvailabilityListener availabilityListener) throws IOException { final boolean notifyDataAvailable; synchronized (buffers) { checkState(!isReleased); checkState(readView == null, "Subpartition %s of is being (or already has been) consumed, " + "but pipelined subpartitions can only be consumed once.", index, parent.getPartitionId()); LOG.debug("{}: Creating read view for subpartition {} of partition {}.", parent.getOwningTaskName(), index, parent.getPartitionId()); readView = new PipelinedSubpartitionView(this, availabilityListener); notifyDataAvailable = !buffers.isEmpty(); } if (notifyDataAvailable) { notifyDataAvailable(); } return readView; }
@Override public PipelinedSubpartitionView createReadView(BufferAvailabilityListener availabilityListener) throws IOException { final boolean notifyDataAvailable; synchronized (buffers) { checkState(!isReleased); checkState(readView == null, "Subpartition %s of is being (or already has been) consumed, " + "but pipelined subpartitions can only be consumed once.", index, parent.getPartitionId()); LOG.debug("{}: Creating read view for subpartition {} of partition {}.", parent.getOwningTaskName(), index, parent.getPartitionId()); readView = new PipelinedSubpartitionView(this, availabilityListener); notifyDataAvailable = !buffers.isEmpty(); } if (notifyDataAvailable) { notifyDataAvailable(); } return readView; }
LOG.debug("{}: Released {}.", parent.getOwningTaskName(), this);
LOG.debug("{}: Released {}.", parent.getOwningTaskName(), this);