public int getNumberOfQueuedBuffers() { int totalBuffers = 0; for (ResultSubpartition subpartition : subpartitions) { totalBuffers += subpartition.unsynchronizedGetNumberOfQueuedBuffers(); } return totalBuffers; }
@Override public void flushAll() { for (ResultSubpartition subpartition : subpartitions) { subpartition.flush(); } }
/** * Returns the requested subpartition. */ public ResultSubpartitionView createSubpartitionView(int index, BufferAvailabilityListener availabilityListener) throws IOException { int refCnt = pendingReferences.get(); checkState(refCnt != -1, "Partition released."); checkState(refCnt > 0, "Partition not pinned."); checkElementIndex(index, subpartitions.length, "Subpartition not found."); ResultSubpartitionView readView = subpartitions[index].createReadView(availabilityListener); LOG.debug("Created {}", readView); return readView; }
@Override public void addBufferConsumer(BufferConsumer bufferConsumer, int subpartitionIndex) throws IOException { checkNotNull(bufferConsumer); ResultSubpartition subpartition; try { checkInProduceState(); subpartition = subpartitions[subpartitionIndex]; } catch (Exception ex) { bufferConsumer.close(); throw ex; } if (subpartition.add(bufferConsumer)) { notifyPipelinedConsumers(); } }
/** * Releases the result partition. */ public void release(Throwable cause) { if (isReleased.compareAndSet(false, true)) { LOG.debug("{}: Releasing {}.", owningTaskName, this); // Set the error cause if (cause != null) { this.cause = cause; } // Release all subpartitions for (ResultSubpartition subpartition : subpartitions) { try { subpartition.release(); } // Catch this in order to ensure that release is called on all subpartitions catch (Throwable t) { LOG.error("Error during release of result subpartition: " + t.getMessage(), t); } } } }
/** * Releases buffers held by this result partition. * * <p>This is a callback from the buffer pool, which is registered for result partitions, which * are back pressure-free. */ @Override public void releaseMemory(int toRelease) throws IOException { checkArgument(toRelease > 0); for (ResultSubpartition subpartition : subpartitions) { toRelease -= subpartition.releaseMemory(); // Only release as much memory as needed if (toRelease <= 0) { break; } } }
@Override public void finish() throws IOException { boolean success = false; try { checkInProduceState(); for (ResultSubpartition subpartition : subpartitions) { subpartition.finish(); } success = true; } finally { if (success) { isFinished = true; notifyPipelinedConsumers(); } } }
/** * Decreases the number of non-event buffers by one after fetching a non-event * buffer from this subpartition (for access by the subpartition views). * * @return backlog after the operation */ public int decreaseBuffersInBacklog(Buffer buffer) { synchronized (buffers) { return decreaseBuffersInBacklogUnsafe(buffer != null && buffer.isBuffer()); } }
/** * Returns the requested subpartition. */ public ResultSubpartitionView createSubpartitionView(int index, BufferAvailabilityListener availabilityListener) throws IOException { int refCnt = pendingReferences.get(); checkState(refCnt != -1, "Partition released."); checkState(refCnt > 0, "Partition not pinned."); checkElementIndex(index, subpartitions.length, "Subpartition not found."); ResultSubpartitionView readView = subpartitions[index].createReadView(availabilityListener); LOG.debug("Created {}", readView); return readView; }
@Override public void addBufferConsumer(BufferConsumer bufferConsumer, int subpartitionIndex) throws IOException { checkNotNull(bufferConsumer); ResultSubpartition subpartition; try { checkInProduceState(); subpartition = subpartitions[subpartitionIndex]; } catch (Exception ex) { bufferConsumer.close(); throw ex; } if (subpartition.add(bufferConsumer)) { notifyPipelinedConsumers(); } }
/** * Releases the result partition. */ public void release(Throwable cause) { if (isReleased.compareAndSet(false, true)) { LOG.debug("{}: Releasing {}.", owningTaskName, this); // Set the error cause if (cause != null) { this.cause = cause; } // Release all subpartitions for (ResultSubpartition subpartition : subpartitions) { try { subpartition.release(); } // Catch this in order to ensure that release is called on all subpartitions catch (Throwable t) { LOG.error("Error during release of result subpartition: " + t.getMessage(), t); } } } }
/** * Releases buffers held by this result partition. * * <p> This is a callback from the buffer pool, which is registered for result partitions, which * are back pressure-free. */ @Override public void releaseMemory(int toRelease) throws IOException { checkArgument(toRelease > 0); for (ResultSubpartition subpartition : subpartitions) { toRelease -= subpartition.releaseMemory(); // Only release as much memory as needed if (toRelease <= 0) { break; } } }
/** * Finishes the result partition. * * <p>After this operation, it is not possible to add further data to the result partition. * * <p>For BLOCKING results, this will trigger the deployment of consuming tasks. */ public void finish() throws IOException { boolean success = false; try { checkInProduceState(); for (ResultSubpartition subpartition : subpartitions) { subpartition.finish(); } success = true; } finally { if (success) { isFinished = true; notifyPipelinedConsumers(); } } }
/** * Decreases the number of non-event buffers by one after fetching a non-event * buffer from this subpartition (for access by the subpartition views). * * @return backlog after the operation */ public int decreaseBuffersInBacklog(Buffer buffer) { synchronized (buffers) { return decreaseBuffersInBacklogUnsafe(buffer != null && buffer.isBuffer()); } }
public int getNumberOfQueuedBuffers() { int totalBuffers = 0; for (ResultSubpartition subpartition : subpartitions) { totalBuffers += subpartition.unsynchronizedGetNumberOfQueuedBuffers(); } return totalBuffers; }
/** * Returns the requested subpartition. */ public ResultSubpartitionView createSubpartitionView(int index, BufferAvailabilityListener availabilityListener) throws IOException { int refCnt = pendingReferences.get(); checkState(refCnt != -1, "Partition released."); checkState(refCnt > 0, "Partition not pinned."); checkElementIndex(index, subpartitions.length, "Subpartition not found."); ResultSubpartitionView readView = subpartitions[index].createReadView(availabilityListener); LOG.debug("Created {}", readView); return readView; }
@VisibleForTesting public void addBufferConsumer(BufferConsumer bufferConsumer, int subpartitionIndex) throws IOException { checkNotNull(bufferConsumer); ResultSubpartition subpartition; try { checkInProduceState(); subpartition = subpartitions[subpartitionIndex]; } catch (Exception ex) { bufferConsumer.close(); throw ex; } if (subpartition.add(bufferConsumer)) { notifyPipelinedConsumers(); } }
@Override protected void releaseInternal() { // Release all subpartitions for (ResultSubpartition subpartition : subpartitions) { try { subpartition.release(); } // Catch this in order to ensure that release is called on all subpartitions catch (Throwable t) { LOG.error("Error during release of result subpartition: " + t.getMessage(), t); } } if (partitionType == ResultPartitionType.BLOCKING && bufferPool != null) { bufferPool.notifyBufferPoolOwnerReleased(); } }
/** * Releases buffers held by this result partition. * * <p> This is a callback from the buffer pool, which is registered for result partitions, which * are back pressure-free. */ @Override public void releaseMemory(int toRelease) throws IOException { checkArgument(toRelease > 0); for (ResultSubpartition subpartition : subpartitions) { toRelease -= subpartition.releaseMemory(); // Only release as much memory as needed if (toRelease <= 0) { break; } } }
/** * Finishes the result partition. * * <p>After this operation, it is not possible to add further data to the result partition. * * <p>For BLOCKING results, this will trigger the deployment of consuming tasks. */ public void finish() throws IOException { boolean success = false; try { checkInProduceState(); for (ResultSubpartition subpartition : subpartitions) { subpartition.finish(); } success = true; } finally { if (success) { isFinished = true; notifyPipelinedConsumers(); } } }