@Override public synchronized boolean add(BufferConsumer bufferConsumer) throws IOException { return add(bufferConsumer, false); }
@Override public Throwable getFailureCause() { return parent.getFailureCause(); }
@Override public String toString() { return String.format("SpillableSubpartition#%d [%d number of buffers (%d bytes)," + "%d number of buffers in backlog, finished? %s, read view? %s, spilled? %s]", index, getTotalNumberOfBuffers(), getTotalNumberOfBytes(), getBuffersInBacklog(), isFinished, readView != null, spillWriter != null); }
@Override public String toString() { return String.format("SpillableSubpartition [%d number of buffers (%d bytes)," + "finished? %s, read view? %s, spilled? %s]", getTotalNumberOfBuffers(), getTotalNumberOfBytes(), isFinished, readView != null, spillWriter != null); }
private boolean add(BufferConsumer bufferConsumer, boolean forceFinishRemainingBuffers) throws IOException { checkNotNull(bufferConsumer); synchronized (buffers) { if (isFinished || isReleased) { bufferConsumer.close(); return false; } buffers.add(bufferConsumer); // The number of buffers are needed later when creating // the read views. If you ever remove this line here, // make sure to still count the number of buffers. updateStatistics(bufferConsumer); increaseBuffersInBacklog(bufferConsumer); if (spillWriter != null) { spillFinishedBufferConsumers(forceFinishRemainingBuffers); } } return true; }
BufferConsumer bufferConsumer = buffers.getFirst(); Buffer buffer = bufferConsumer.build(); updateStatistics(buffer); int bufferSize = buffer.getSize(); spilledBytes += bufferSize; } else { decreaseBuffersInBacklog(buffer); buffer.recycleBuffer(); increaseBuffersInBacklog(bufferConsumer); } else { buffer.recycleBuffer();
@Override public synchronized void finish() throws IOException { synchronized (buffers) { if (add(EventSerializer.toBufferConsumer(EndOfPartitionEvent.INSTANCE), true)) { isFinished = true; } flush(); } // If we are spilling/have spilled, wait for the writer to finish if (spillWriter != null) { spillWriter.close(); } }
"We can only read from SpillableSubpartition after it was finished"); newBacklog = parent.decreaseBuffersInBacklogUnsafe(nextBuffer.isBuffer()); nextBuffer.close(); nextBuffer = buffers.poll(); parent.updateStatistics(current);
@Override public boolean isReleased() { return parent.isReleased() || isReleased.get(); }
@Override public void notifySubpartitionConsumed() throws IOException { parent.onConsumedSubpartition(); }
@Override public boolean add(Buffer buffer) throws IOException { checkNotNull(buffer); synchronized (buffers) { if (isFinished || isReleased) { return false; } // The number of buffers are needed later when creating // the read views. If you ever remove this line here, // make sure to still count the number of buffers. updateStatistics(buffer); if (spillWriter == null) { buffers.add(buffer); return true; } } // Didn't return early => go to disk spillWriter.writeBlock(buffer); return true; }
@Override public ResultSubpartitionView createReadView(BufferAvailabilityListener availabilityListener) throws IOException { synchronized (buffers) { if (!isFinished) { throw new IllegalStateException("Subpartition has not been finished yet, " + "but blocking subpartitions can only be consumed after they have " + "been finished."); } if (readView != null) { throw new IllegalStateException("Subpartition is being or already has been " + "consumed, but we currently allow subpartitions to only be consumed once."); } if (spillWriter != null) { readView = new SpilledSubpartitionView( this, parent.getBufferProvider().getMemorySegmentSize(), spillWriter, getTotalNumberOfBuffers(), availabilityListener); } else { readView = new SpillableSubpartitionView( this, buffers, ioManager, parent.getBufferProvider().getMemorySegmentSize(), availabilityListener); } return readView; } }
case BLOCKING: for (int i = 0; i < subpartitions.length; i++) { subpartitions[i] = new SpillableSubpartition(i, this, ioManager);
@Nullable @Override public BufferAndBacklog getNextBuffer() throws IOException, InterruptedException { if (isSpillInProgress) { return null; } Buffer current; boolean nextBufferIsEvent; synchronized (this) { if (nextBuffer == null) { current = requestAndFillBuffer(); } else { current = nextBuffer; } nextBuffer = requestAndFillBuffer(); nextBufferIsEvent = nextBuffer != null && !nextBuffer.isBuffer(); } if (current == null) { return null; } int newBacklog = parent.decreaseBuffersInBacklog(current); return new BufferAndBacklog(current, newBacklog > 0 || nextBufferIsEvent, newBacklog, nextBufferIsEvent); }
private boolean add(BufferConsumer bufferConsumer, boolean forceFinishRemainingBuffers) throws IOException { checkNotNull(bufferConsumer); synchronized (buffers) { if (isFinished || isReleased) { bufferConsumer.close(); return false; } buffers.add(bufferConsumer); // The number of buffers are needed later when creating // the read views. If you ever remove this line here, // make sure to still count the number of buffers. updateStatistics(bufferConsumer); increaseBuffersInBacklog(bufferConsumer); if (spillWriter != null) { spillFinishedBufferConsumers(forceFinishRemainingBuffers); } } return true; }
BufferConsumer bufferConsumer = buffers.getFirst(); Buffer buffer = bufferConsumer.build(); updateStatistics(buffer); int bufferSize = buffer.getSize(); spilledBytes += bufferSize; } else { decreaseBuffersInBacklog(buffer); buffer.recycleBuffer(); increaseBuffersInBacklog(bufferConsumer); } else { buffer.recycleBuffer();
@Override public synchronized void finish() throws IOException { synchronized (buffers) { if (add(EventSerializer.toBufferConsumer(EndOfPartitionEvent.INSTANCE), true)) { isFinished = true; } flush(); } // If we are spilling/have spilled, wait for the writer to finish if (spillWriter != null) { spillWriter.close(); } LOG.debug("{}: Finished {}.", parent.getOwningTaskName(), this); }
"We can only read from SpillableSubpartition after it was finished"); newBacklog = parent.decreaseBuffersInBacklogUnsafe(nextBuffer.isBuffer()); nextBuffer.close(); nextBuffer = buffers.poll(); parent.updateStatistics(current);
@Override public boolean isReleased() { return parent.isReleased() || isReleased.get(); }
@Override public void notifySubpartitionConsumed() throws IOException { parent.onConsumedSubpartition(); }