dsType.isBlocking()); } else { assertFalse("Expected non-batch exchange, but result type is " + dsType + ".", dsType.isBlocking());
/** * Returns all blocking result partitions whose receivers can be scheduled/updated. */ List<IntermediateResultPartition> finishAllBlockingPartitions() { List<IntermediateResultPartition> finishedBlockingPartitions = null; for (IntermediateResultPartition partition : resultPartitions.values()) { if (partition.getResultType().isBlocking() && partition.markFinished()) { if (finishedBlockingPartitions == null) { finishedBlockingPartitions = new LinkedList<IntermediateResultPartition>(); } finishedBlockingPartitions.add(partition); } } if (finishedBlockingPartitions == null) { return Collections.emptyList(); } else { return finishedBlockingPartitions; } }
boolean markFinished() { // Sanity check that this is only called on blocking partitions. if (!getResultType().isBlocking()) { throw new IllegalStateException("Tried to mark a non-blocking result partition as finished"); } final int refCnt = totalResult.decrementNumberOfRunningProducersAndGetRemaining(); if (refCnt == 0) { return true; } else if (refCnt < 0) { throw new IllegalStateException("Decremented number of unfinished producers below 0. " + "This is most likely a bug in the execution state/intermediate result " + "partition management."); } return false; } }
/** * Returns all blocking result partitions whose receivers can be scheduled/updated. */ List<IntermediateResultPartition> finishAllBlockingPartitions() { List<IntermediateResultPartition> finishedBlockingPartitions = null; for (IntermediateResultPartition partition : resultPartitions.values()) { if (partition.getResultType().isBlocking() && partition.markFinished()) { if (finishedBlockingPartitions == null) { finishedBlockingPartitions = new LinkedList<IntermediateResultPartition>(); } finishedBlockingPartitions.add(partition); } } if (finishedBlockingPartitions == null) { return Collections.emptyList(); } else { return finishedBlockingPartitions; } }
/** * Returns all blocking result partitions whose receivers can be scheduled/updated. */ List<IntermediateResultPartition> finishAllBlockingPartitions() { List<IntermediateResultPartition> finishedBlockingPartitions = null; for (IntermediateResultPartition partition : resultPartitions.values()) { if (partition.getResultType().isBlocking() && partition.markFinished()) { if (finishedBlockingPartitions == null) { finishedBlockingPartitions = new LinkedList<IntermediateResultPartition>(); } finishedBlockingPartitions.add(partition); } } if (finishedBlockingPartitions == null) { return Collections.emptyList(); } else { return finishedBlockingPartitions; } }
boolean markFinished() { // Sanity check that this is only called on blocking partitions. if (!getResultType().isBlocking()) { throw new IllegalStateException("Tried to mark a non-blocking result partition as finished"); } final int refCnt = totalResult.decrementNumberOfRunningProducersAndGetRemaining(); if (refCnt == 0) { return true; } else if (refCnt < 0) { throw new IllegalStateException("Decremented number of unfinished producers below 0. " + "This is most likely a bug in the execution state/intermediate result " + "partition management."); } return false; } }
boolean markFinished() { // Sanity check that this is only called on blocking partitions. if (!getResultType().isBlocking()) { throw new IllegalStateException("Tried to mark a non-blocking result partition as finished"); } final int refCnt = totalResult.decrementNumberOfRunningProducersAndGetRemaining(); if (refCnt == 0) { return true; } else if (refCnt < 0) { throw new IllegalStateException("Decremented number of unfinished producers below 0. " + "This is most likely a bug in the execution state/intermediate result " + "partition management."); } return false; } }
/** * Finish all blocking result partitions whose receivers can be scheduled/updated and notify. */ void finishPartitionsAndNotify() { for (IntermediateResultPartition partition : resultPartitions.values()) { partition.markFinished(); // Blocking partitions are consumable on finished if (partition.getResultType().isBlocking()) { notifyAndUpdateConsumers(partition); } } }
public void markFinished() { if (getResultType().isBlocking() && !isConsumable()) { getIntermediateResult().incrementNumberOfConsumablePartitions(); } isFinished = true; } }
public SerializerManager(ResultPartitionType resultPartitionType, Configuration configuration) { // Conditions to enable compression: (1) the edge is BLOCKING // && (2) use external shuffle service // && (3) set TaskManagerOptions.TASK_EXTERNAL_SHUFFLE_ENABLE_COMPRESSION true // in flink configuration. if (!resultPartitionType.isBlocking()) { this.useCompression = false; } else { BlockingShuffleType shuffleType = BlockingShuffleType.getBlockingShuffleTypeFromConfiguration(configuration, LOG); this.useCompression = (shuffleType == BlockingShuffleType.YARN) && configuration.getBoolean(TaskManagerOptions.TASK_EXTERNAL_SHUFFLE_ENABLE_COMPRESSION); } if (!this.useCompression) { this.compressionFactory = Optional.empty(); this.compressionBufferSize = -1; } else { this.compressionFactory = Optional.of(BlockCompressionFactoryLoader.createBlockCompressionFactory( configuration.getString(TaskManagerOptions.TASK_EXTERNAL_SHUFFLE_COMPRESSION_CODEC), configuration)); this.compressionBufferSize = configuration.getInteger( TaskManagerOptions.TASK_EXTERNAL_SHUFFLE_COMPRESSION_BUFFER_SIZE); } }
private int calculateTaskExtraManagedMemory() { Configuration config = getJobVertex().getGraph().getJobManagerConfiguration(); // Calculates managed memory for external result partition. BlockingShuffleType shuffleType = BlockingShuffleType.getBlockingShuffleTypeFromConfiguration(config, LOG); int numExternalResultPartitions = 0; for (IntermediateResultPartition irp : getProducedPartitions().values()) { if (shuffleType == BlockingShuffleType.YARN && irp.getIntermediateResult().getResultType().isBlocking()) { numExternalResultPartitions++; } } int mapOutputMemoryInMB = config.getInteger(TaskManagerOptions.TASK_MANAGER_OUTPUT_MEMORY_MB); return mapOutputMemoryInMB * numExternalResultPartitions; }
int numInternalResultPartitions = 0; for (IntermediateResultPartition irp : getProducedPartitions().values()) { if (!(shuffleType == BlockingShuffleType.YARN && irp.getIntermediateResult().getResultType().isBlocking())) { for (List<ExecutionEdge> consumer : irp.getConsumers()) { numInternalSubpartitions += consumer.size(); boolean isExternalBlocking = edges[0].getSource().getIntermediateResult().getResultType().isBlocking() && shuffleType == BlockingShuffleType.YARN;
ResultPartitionID partitionId = new ResultPartitionID(desc.getPartitionId(), executionId); ResultPartitionType partitionType = desc.getPartitionType(); if (partitionType.isBlocking() && shuffleType == BlockingShuffleType.YARN) { ExternalResultPartition resultPartition = new ExternalResultPartition( taskManagerConfig.getConfiguration(),
shuffleType == BlockingShuffleType.YARN && consumedPartitionType.isBlocking(); final SingleInputGate inputGate = new SingleInputGate( owningTaskName, jobId, consumedResultId, consumedPartitionType,