dsType.isBlocking()); } else { assertFalse("Expected non-batch exchange, but result type is " + dsType + ".", dsType.isBlocking());
boolean isConsumable() { if (resultType.isPipelined()) { return true; } else { return numberOfRunningProducers.get() == 0; } }
@VisibleForTesting public void setupPartition(ResultPartition partition) throws IOException { BufferPool bufferPool = null; try { int maxNumberOfMemorySegments = partition.getPartitionType().isBounded() ? partition.getNumberOfSubpartitions() * networkBuffersPerChannel + extraNetworkBuffersPerGate : Integer.MAX_VALUE; // If the partition type is back pressure-free, we register with the buffer pool for // callbacks to release memory. bufferPool = networkBufferPool.createBufferPool(partition.getNumberOfSubpartitions(), maxNumberOfMemorySegments, partition.getPartitionType().hasBackPressure() ? Optional.empty() : Optional.of(partition)); partition.registerBufferPool(bufferPool); resultPartitionManager.registerResultPartition(partition); } catch (Throwable t) { if (bufferPool != null) { bufferPool.lazyDestroy(); } if (t instanceof IOException) { throw (IOException) t; } else { throw new IOException(t.getMessage(), t); } } taskEventDispatcher.registerPartition(partition.getPartitionId()); }
gen.writeStringField("exchange", edge.getSource().getResultType().name().toLowerCase());
/** * Registers a buffer pool with this result partition. * <p> * There is one pool for each result partition, which is shared by all its sub partitions. * <p> * The pool is registered with the partition *after* it as been constructed in order to conform * to the life-cycle of task registrations in the {@link TaskManager}. */ public void registerBufferPool(BufferPool bufferPool) { checkArgument(bufferPool.getNumberOfRequiredMemorySegments() >= getNumberOfSubpartitions(), "Bug in result partition setup logic: Buffer pool has not enough guaranteed buffers for this result partition."); checkState(this.bufferPool == null, "Bug in result partition setup logic: Already registered buffer pool."); this.bufferPool = checkNotNull(bufferPool); // If the partition type is back pressure-free, we register with the buffer pool for // callbacks to release memory. if (!partitionType.hasBackPressure()) { bufferPool.setBufferPoolOwner(this); } }
@VisibleForTesting public void setupInputGate(SingleInputGate gate) throws IOException { BufferPool bufferPool = null; int maxNumberOfMemorySegments; try { if (enableCreditBased) { maxNumberOfMemorySegments = gate.getConsumedPartitionType().isBounded() ? extraNetworkBuffersPerGate : Integer.MAX_VALUE; // assign exclusive buffers to input channels directly and use the rest for floating buffers gate.assignExclusiveSegments(networkBufferPool, networkBuffersPerChannel); bufferPool = networkBufferPool.createBufferPool(0, maxNumberOfMemorySegments); } else { maxNumberOfMemorySegments = gate.getConsumedPartitionType().isBounded() ? gate.getNumberOfInputChannels() * networkBuffersPerChannel + extraNetworkBuffersPerGate : Integer.MAX_VALUE; bufferPool = networkBufferPool.createBufferPool(gate.getNumberOfInputChannels(), maxNumberOfMemorySegments); } gate.setBufferPool(bufferPool); } catch (Throwable t) { if (bufferPool != null) { bufferPool.lazyDestroy(); } ExceptionUtils.rethrowIOException(t); } }
@VisibleForTesting public void setupPartition(ResultPartition partition) throws IOException { BufferPool bufferPool = null; try { int maxNumberOfMemorySegments = partition.getPartitionType().isBounded() ? partition.getNumberOfSubpartitions() * networkBuffersPerChannel + extraNetworkBuffersPerGate : Integer.MAX_VALUE; // If the partition type is back pressure-free, we register with the buffer pool for // callbacks to release memory. bufferPool = networkBufferPool.createBufferPool(partition.getNumberOfSubpartitions(), maxNumberOfMemorySegments, partition.getPartitionType().hasBackPressure() ? Optional.empty() : Optional.of(partition)); partition.registerBufferPool(bufferPool); resultPartitionManager.registerResultPartition(partition); } catch (Throwable t) { if (bufferPool != null) { bufferPool.lazyDestroy(); } if (t instanceof IOException) { throw (IOException) t; } else { throw new IOException(t.getMessage(), t); } } taskEventDispatcher.registerPartition(partition.getPartitionId()); }
gen.writeStringField("exchange", edge.getSource().getResultType().name().toLowerCase());
/** * Registers a buffer pool with this result partition. * <p> * There is one pool for each result partition, which is shared by all its sub partitions. * <p> * The pool is registered with the partition *after* it as been constructed in order to conform * to the life-cycle of task registrations in the {@link TaskManager}. */ public void registerBufferPool(BufferPool bufferPool) { checkArgument(bufferPool.getNumberOfRequiredMemorySegments() >= getNumberOfSubpartitions(), "Bug in result partition setup logic: Buffer pool has not enough guaranteed buffers for this result partition."); checkState(this.bufferPool == null, "Bug in result partition setup logic: Already registered buffer pool."); this.bufferPool = checkNotNull(bufferPool); // If the partition type is back pressure-free, we register with the buffer pool for // callbacks to release memory. if (!partitionType.hasBackPressure()) { bufferPool.setBufferPoolOwner(this); } }
@VisibleForTesting public void setupInputGate(SingleInputGate gate) throws IOException { BufferPool bufferPool = null; int maxNumberOfMemorySegments; try { if (enableCreditBased) { maxNumberOfMemorySegments = gate.getConsumedPartitionType().isBounded() ? extraNetworkBuffersPerGate : Integer.MAX_VALUE; // assign exclusive buffers to input channels directly and use the rest for floating buffers gate.assignExclusiveSegments(networkBufferPool, networkBuffersPerChannel); bufferPool = networkBufferPool.createBufferPool(0, maxNumberOfMemorySegments); } else { maxNumberOfMemorySegments = gate.getConsumedPartitionType().isBounded() ? gate.getNumberOfInputChannels() * networkBuffersPerChannel + extraNetworkBuffersPerGate : Integer.MAX_VALUE; bufferPool = networkBufferPool.createBufferPool(gate.getNumberOfInputChannels(), maxNumberOfMemorySegments); } gate.setBufferPool(bufferPool); } catch (Throwable t) { if (bufferPool != null) { bufferPool.lazyDestroy(); } ExceptionUtils.rethrowIOException(t); } }
boolean markFinished() { // Sanity check that this is only called on blocking partitions. if (!getResultType().isBlocking()) { throw new IllegalStateException("Tried to mark a non-blocking result partition as finished"); } final int refCnt = totalResult.decrementNumberOfRunningProducersAndGetRemaining(); if (refCnt == 0) { return true; } else if (refCnt < 0) { throw new IllegalStateException("Decremented number of unfinished producers below 0. " + "This is most likely a bug in the execution state/intermediate result " + "partition management."); } return false; } }
boolean isConsumable() { if (resultType.isPipelined()) { return true; } else { return numberOfRunningProducers.get() == 0; } }
gen.writeStringField("exchange", edge.getSource().getResultType().name().toLowerCase());
int maxNumberOfMemorySegments = partition.getPartitionType().isBounded() ? partition.getNumberOfSubpartitions() * networkBuffersPerChannel + extraNetworkBuffersPerGate : Integer.MAX_VALUE; int maxNumberOfMemorySegments = gate.getConsumedPartitionType().isBounded() ? gate.getNumberOfInputChannels() * networkBuffersPerChannel + extraNetworkBuffersPerGate : Integer.MAX_VALUE;
/** * Returns all blocking result partitions whose receivers can be scheduled/updated. */ List<IntermediateResultPartition> finishAllBlockingPartitions() { List<IntermediateResultPartition> finishedBlockingPartitions = null; for (IntermediateResultPartition partition : resultPartitions.values()) { if (partition.getResultType().isBlocking() && partition.markFinished()) { if (finishedBlockingPartitions == null) { finishedBlockingPartitions = new LinkedList<IntermediateResultPartition>(); } finishedBlockingPartitions.add(partition); } } if (finishedBlockingPartitions == null) { return Collections.emptyList(); } else { return finishedBlockingPartitions; } }
boolean isConsumable() { if (resultType.isPipelined()) { return true; } else { return numberOfRunningProducers.get() == 0; } }
gen.writeStringField("exchange", edge.getSource().getResultType().name().toLowerCase());
boolean markFinished() { // Sanity check that this is only called on blocking partitions. if (!getResultType().isBlocking()) { throw new IllegalStateException("Tried to mark a non-blocking result partition as finished"); } final int refCnt = totalResult.decrementNumberOfRunningProducersAndGetRemaining(); if (refCnt == 0) { return true; } else if (refCnt < 0) { throw new IllegalStateException("Decremented number of unfinished producers below 0. " + "This is most likely a bug in the execution state/intermediate result " + "partition management."); } return false; } }
/** * Notifies pipelined consumers of this result partition once. */ private void notifyPipelinedConsumers() { if (sendScheduleOrUpdateConsumersMessage && !hasNotifiedPipelinedConsumers && partitionType.isPipelined()) { partitionConsumableNotifier.notifyPartitionConsumable(jobId, partitionId, taskActions); hasNotifiedPipelinedConsumers = true; } } }
/** * Returns all blocking result partitions whose receivers can be scheduled/updated. */ List<IntermediateResultPartition> finishAllBlockingPartitions() { List<IntermediateResultPartition> finishedBlockingPartitions = null; for (IntermediateResultPartition partition : resultPartitions.values()) { if (partition.getResultType().isBlocking() && partition.markFinished()) { if (finishedBlockingPartitions == null) { finishedBlockingPartitions = new LinkedList<IntermediateResultPartition>(); } finishedBlockingPartitions.add(partition); } } if (finishedBlockingPartitions == null) { return Collections.emptyList(); } else { return finishedBlockingPartitions; } }