ResultPartitionType getResultType() { return totalResult.getResultType(); }
ResultPartitionType getResultType() { return totalResult.getResultType(); }
public ResultPartitionType getResultType() { return totalResult.getResultType(); }
ResultPartitionType getResultType() { return totalResult.getResultType(); }
public static ResultPartitionDeploymentDescriptor from( IntermediateResultPartition partition, int maxParallelism, boolean lazyScheduling) { final IntermediateDataSetID resultId = partition.getIntermediateResult().getId(); final IntermediateResultPartitionID partitionId = partition.getPartitionId(); final ResultPartitionType partitionType = partition.getIntermediateResult().getResultType(); // The produced data is partitioned among a number of subpartitions. // // If no consumers are known at this point, we use a single subpartition, otherwise we have // one for each consuming sub task. int numberOfSubpartitions = 1; if (!partition.getConsumers().isEmpty() && !partition.getConsumers().get(0).isEmpty()) { if (partition.getConsumers().size() > 1) { throw new IllegalStateException("Currently, only a single consumer group per partition is supported."); } numberOfSubpartitions = partition.getConsumers().get(0).size(); } return new ResultPartitionDeploymentDescriptor( resultId, partitionId, partitionType, numberOfSubpartitions, maxParallelism, lazyScheduling); } }
public static ResultPartitionDeploymentDescriptor from( IntermediateResultPartition partition, int maxParallelism, boolean lazyScheduling) { final IntermediateDataSetID resultId = partition.getIntermediateResult().getId(); final IntermediateResultPartitionID partitionId = partition.getPartitionId(); final ResultPartitionType partitionType = partition.getIntermediateResult().getResultType(); // The produced data is partitioned among a number of subpartitions. // // If no consumers are known at this point, we use a single subpartition, otherwise we have // one for each consuming sub task. int numberOfSubpartitions = 1; if (!partition.getConsumers().isEmpty() && !partition.getConsumers().get(0).isEmpty()) { if (partition.getConsumers().size() > 1) { throw new IllegalStateException("Currently, only a single consumer group per partition is supported."); } numberOfSubpartitions = partition.getConsumers().get(0).size(); } return new ResultPartitionDeploymentDescriptor( resultId, partitionId, partitionType, numberOfSubpartitions, maxParallelism, lazyScheduling); } }
public static ResultPartitionDeploymentDescriptor from( IntermediateResultPartition partition, int maxParallelism, boolean lazyScheduling) { final IntermediateDataSetID resultId = partition.getIntermediateResult().getId(); final IntermediateResultPartitionID partitionId = partition.getPartitionId(); final ResultPartitionType partitionType = partition.getIntermediateResult().getResultType(); // The produced data is partitioned among a number of subpartitions. // // If no consumers are known at this point, we use a single subpartition, otherwise we have // one for each consuming sub task. int numberOfSubpartitions = 1; if (!partition.getConsumers().isEmpty() && !partition.getConsumers().get(0).isEmpty()) { if (partition.getConsumers().size() > 1) { throw new IllegalStateException("Currently, only a single consumer group per partition is supported."); } numberOfSubpartitions = partition.getConsumers().get(0).size(); } return new ResultPartitionDeploymentDescriptor( resultId, partitionId, partitionType, numberOfSubpartitions, maxParallelism, lazyScheduling); } }
public static ResultPartitionDeploymentDescriptor from( IntermediateResultPartition partition, int maxParallelism, boolean lazyScheduling) { final IntermediateDataSetID resultId = partition.getIntermediateResult().getId(); final IntermediateResultPartitionID partitionId = partition.getPartitionId(); final ResultPartitionType partitionType = partition.getIntermediateResult().getResultType(); // The produced data is partitioned among a number of subpartitions. // // If no consumers are known at this point, we use a single subpartition, otherwise we have // one for each consuming sub task. int numberOfSubpartitions = 1; if (!partition.getConsumers().isEmpty() && !partition.getConsumers().get(0).isEmpty()) { if (partition.getConsumers().size() > 1) { throw new IllegalStateException("Currently, only a single consumer group per partition is supported."); } numberOfSubpartitions = partition.getConsumers().get(0).size(); } return new ResultPartitionDeploymentDescriptor( resultId, partitionId, partitionType, numberOfSubpartitions, maxParallelism, lazyScheduling); } }
/** * Gets the result partition location. This method is used when generating {@link InputChannelDeploymentDescriptor}. * * @param producerLocation The partition producer location. * @param consumerLocation The partition consumer location. * @param intermediateResult The intermediate result to be consumed. * @return The result partition location. */ public ResultPartitionLocation getResultPartitionLocation( TaskManagerLocation producerLocation, TaskManagerLocation consumerLocation, IntermediateResult intermediateResult) { BlockingShuffleType shuffleType = BlockingShuffleType.getBlockingShuffleTypeFromConfiguration(configuration, LOG); if (intermediateResult.getResultType() == ResultPartitionType.BLOCKING && shuffleType == BlockingShuffleType.YARN) { // use the yarn shuffle service return externalResultPartitionLocationTracker.getResultPartitionLocation( producerLocation, consumerLocation, intermediateResult); } else { // use internal shuffle service return internalResultPartitionLocationTracker.getResultPartitionLocation( producerLocation, consumerLocation, intermediateResult); } } }
if (input.getResultType().isPipelined()) { hasPipelinedInputs = true; break; if (inputs.get(inputNum).getResultType().isPipelined()) {
/** * Schedules or updates the consumer tasks of the result partition with the given ID. */ void scheduleOrUpdateConsumers(ResultPartitionID partitionId) { final Execution execution = currentExecution; // Abort this request if there was a concurrent reset if (!partitionId.getProducerId().equals(execution.getAttemptId())) { return; } final IntermediateResultPartition partition = resultPartitions.get(partitionId.getPartitionId()); if (partition == null) { throw new IllegalStateException("Unknown partition " + partitionId + "."); } if (partition.getIntermediateResult().getResultType().isPipelined()) { // Schedule or update receivers of this partition execution.scheduleOrUpdateConsumers(partition.getConsumers()); } else { throw new IllegalArgumentException("ScheduleOrUpdateConsumers msg is only valid for" + "pipelined partitions."); } }
/** * Schedules or updates the consumer tasks of the result partition with the given ID. */ void scheduleOrUpdateConsumers(ResultPartitionID partitionId) { final Execution execution = currentExecution; // Abort this request if there was a concurrent reset if (!partitionId.getProducerId().equals(execution.getAttemptId())) { return; } final IntermediateResultPartition partition = resultPartitions.get(partitionId.getPartitionId()); if (partition == null) { throw new IllegalStateException("Unknown partition " + partitionId + "."); } if (partition.getIntermediateResult().getResultType().isPipelined()) { // Schedule or update receivers of this partition execution.scheduleOrUpdateConsumers(partition.getConsumers()); } else { throw new IllegalArgumentException("ScheduleOrUpdateConsumers msg is only valid for" + "pipelined partitions."); } }
public void notifyResultPartitionConsumable( ExecutionVertexID executionVertexID, IntermediateDataSetID resultID, int partitionNumber, TaskManagerLocation location) { if (!operationLogManager.isReplaying() && executionGraph.getAllIntermediateResults().get(resultID).getResultType().isPipelined()) { operationLogManager.writeOpLog(new ResultPartitionOperationLog( executionVertexID, resultID, location)); } graphManagerPlugin.onResultPartitionConsumable( new ResultPartitionConsumableEvent(resultID, partitionNumber)); }
/** * Schedules or updates the consumer tasks of the result partition with the given ID. */ void scheduleOrUpdateConsumers(ResultPartitionID partitionId) { final Execution execution = currentExecution; // Abort this request if there was a concurrent reset if (!partitionId.getProducerId().equals(execution.getAttemptId())) { return; } final IntermediateResultPartition partition = resultPartitions.get(partitionId.getPartitionId()); if (partition == null) { throw new IllegalStateException("Unknown partition " + partitionId + "."); } if (partition.getIntermediateResult().getResultType().isPipelined()) { // Schedule or update receivers of this partition execution.scheduleOrUpdateConsumers(partition.getConsumers()); } else { throw new IllegalArgumentException("ScheduleOrUpdateConsumers msg is only valid for" + "pipelined partitions."); } }
/** * Schedules or updates the consumer tasks of the result partition with the given ID. */ void scheduleOrUpdateConsumers(ResultPartitionID partitionId) { final Execution execution = currentExecution; // Abort this request if there was a concurrent reset if (!partitionId.getProducerId().equals(execution.getAttemptId())) { return; } final IntermediateResultPartition partition = resultPartitions.get(partitionId.getPartitionId()); if (partition == null) { throw new IllegalStateException("Unknown partition " + partitionId + "."); } if (partition.getIntermediateResult().getResultType().isPipelined()) { // Schedule or update receivers of this partition partition.markDataProduced(); // Notify the scheduler to handle the consumable partition notifyAndUpdateConsumers(partition); } else { throw new IllegalArgumentException("ScheduleOrUpdateConsumers msg is only valid for" + "pipelined partitions."); } }
private int calculateTaskExtraManagedMemory() { Configuration config = getJobVertex().getGraph().getJobManagerConfiguration(); // Calculates managed memory for external result partition. BlockingShuffleType shuffleType = BlockingShuffleType.getBlockingShuffleTypeFromConfiguration(config, LOG); int numExternalResultPartitions = 0; for (IntermediateResultPartition irp : getProducedPartitions().values()) { if (shuffleType == BlockingShuffleType.YARN && irp.getIntermediateResult().getResultType().isBlocking()) { numExternalResultPartitions++; } } int mapOutputMemoryInMB = config.getInteger(TaskManagerOptions.TASK_MANAGER_OUTPUT_MEMORY_MB); return mapOutputMemoryInMB * numExternalResultPartitions; }
int numInternalResultPartitions = 0; for (IntermediateResultPartition irp : getProducedPartitions().values()) { if (!(shuffleType == BlockingShuffleType.YARN && irp.getIntermediateResult().getResultType().isBlocking())) { for (List<ExecutionEdge> consumer : irp.getConsumers()) { numInternalSubpartitions += consumer.size(); boolean isExternalBlocking = edges[0].getSource().getIntermediateResult().getResultType().isBlocking() && shuffleType == BlockingShuffleType.YARN;
final ResultPartitionType partitionType = consumedIntermediateResult.getResultType();
final ResultPartitionType partitionType = consumedIntermediateResult.getResultType();
final ResultPartitionType partitionType = consumedIntermediateResult.getResultType();