public void resetForNewExecution() { if (isConsumable()) { getIntermediateResult().decrementNumberOfConsumablePartitions(); } isFinished = false; dataProduced = false; }
public void markDataProduced() { if (!isConsumable()) { getIntermediateResult().incrementNumberOfConsumablePartitions(); } dataProduced = true; }
void resetResultPartitionID(ResultPartitionID[] partitionIds) { Map<IntermediateResultPartitionID, IntermediateResultPartition> newResultPartitions = new LinkedHashMap<>(resultPartitions.size()); Iterator<IntermediateResultPartition> iterator = resultPartitions.values().iterator(); for (int i = 0; i < resultPartitions.size(); i++) { IntermediateResultPartition resultPartition = iterator.next(); IntermediateResultPartitionID originId = resultPartition.getPartitionId(); resultPartition.setPartitionId(partitionIds[i].getPartitionId()); resultPartition.getIntermediateResult().resetLookupHelper(originId, partitionIds[i].getPartitionId()); newResultPartitions.put(resultPartition.getPartitionId(), resultPartition); } this.resultPartitions = newResultPartitions; }
public static ResultPartitionDeploymentDescriptor from( IntermediateResultPartition partition, int maxParallelism, boolean lazyScheduling) { final IntermediateDataSetID resultId = partition.getIntermediateResult().getId(); final IntermediateResultPartitionID partitionId = partition.getPartitionId(); final ResultPartitionType partitionType = partition.getIntermediateResult().getResultType(); // The produced data is partitioned among a number of subpartitions. // // If no consumers are known at this point, we use a single subpartition, otherwise we have // one for each consuming sub task. int numberOfSubpartitions = 1; if (!partition.getConsumers().isEmpty() && !partition.getConsumers().get(0).isEmpty()) { if (partition.getConsumers().size() > 1) { throw new IllegalStateException("Currently, only a single consumer group per partition is supported."); } numberOfSubpartitions = partition.getConsumers().get(0).size(); } return new ResultPartitionDeploymentDescriptor( resultId, partitionId, partitionType, numberOfSubpartitions, maxParallelism, lazyScheduling); } }
public static ResultPartitionDeploymentDescriptor from( IntermediateResultPartition partition, int maxParallelism, boolean lazyScheduling) { final IntermediateDataSetID resultId = partition.getIntermediateResult().getId(); final IntermediateResultPartitionID partitionId = partition.getPartitionId(); final ResultPartitionType partitionType = partition.getIntermediateResult().getResultType(); // The produced data is partitioned among a number of subpartitions. // // If no consumers are known at this point, we use a single subpartition, otherwise we have // one for each consuming sub task. int numberOfSubpartitions = 1; if (!partition.getConsumers().isEmpty() && !partition.getConsumers().get(0).isEmpty()) { if (partition.getConsumers().size() > 1) { throw new IllegalStateException("Currently, only a single consumer group per partition is supported."); } numberOfSubpartitions = partition.getConsumers().get(0).size(); } return new ResultPartitionDeploymentDescriptor( resultId, partitionId, partitionType, numberOfSubpartitions, maxParallelism, lazyScheduling); } }
public static ResultPartitionDeploymentDescriptor from( IntermediateResultPartition partition, int maxParallelism, boolean lazyScheduling) { final IntermediateDataSetID resultId = partition.getIntermediateResult().getId(); final IntermediateResultPartitionID partitionId = partition.getPartitionId(); final ResultPartitionType partitionType = partition.getIntermediateResult().getResultType(); // The produced data is partitioned among a number of subpartitions. // // If no consumers are known at this point, we use a single subpartition, otherwise we have // one for each consuming sub task. int numberOfSubpartitions = 1; if (!partition.getConsumers().isEmpty() && !partition.getConsumers().get(0).isEmpty()) { if (partition.getConsumers().size() > 1) { throw new IllegalStateException("Currently, only a single consumer group per partition is supported."); } numberOfSubpartitions = partition.getConsumers().get(0).size(); } return new ResultPartitionDeploymentDescriptor( resultId, partitionId, partitionType, numberOfSubpartitions, maxParallelism, lazyScheduling); } }
public static ResultPartitionDeploymentDescriptor from( IntermediateResultPartition partition, int maxParallelism, boolean lazyScheduling) { final IntermediateDataSetID resultId = partition.getIntermediateResult().getId(); final IntermediateResultPartitionID partitionId = partition.getPartitionId(); final ResultPartitionType partitionType = partition.getIntermediateResult().getResultType(); // The produced data is partitioned among a number of subpartitions. // // If no consumers are known at this point, we use a single subpartition, otherwise we have // one for each consuming sub task. int numberOfSubpartitions = 1; if (!partition.getConsumers().isEmpty() && !partition.getConsumers().get(0).isEmpty()) { if (partition.getConsumers().size() > 1) { throw new IllegalStateException("Currently, only a single consumer group per partition is supported."); } numberOfSubpartitions = partition.getConsumers().get(0).size(); } return new ResultPartitionDeploymentDescriptor( resultId, partitionId, partitionType, numberOfSubpartitions, maxParallelism, lazyScheduling); } }
public void markFinished() { if (getResultType().isBlocking() && !isConsumable()) { getIntermediateResult().incrementNumberOfConsumablePartitions(); } isFinished = true; } }
/** * Schedules or updates the consumer tasks of the result partition with the given ID. */ void scheduleOrUpdateConsumers(ResultPartitionID partitionId) { final Execution execution = currentExecution; // Abort this request if there was a concurrent reset if (!partitionId.getProducerId().equals(execution.getAttemptId())) { return; } final IntermediateResultPartition partition = resultPartitions.get(partitionId.getPartitionId()); if (partition == null) { throw new IllegalStateException("Unknown partition " + partitionId + "."); } if (partition.getIntermediateResult().getResultType().isPipelined()) { // Schedule or update receivers of this partition execution.scheduleOrUpdateConsumers(partition.getConsumers()); } else { throw new IllegalArgumentException("ScheduleOrUpdateConsumers msg is only valid for" + "pipelined partitions."); } }
/** * Schedules or updates the consumer tasks of the result partition with the given ID. */ void scheduleOrUpdateConsumers(ResultPartitionID partitionId) { final Execution execution = currentExecution; // Abort this request if there was a concurrent reset if (!partitionId.getProducerId().equals(execution.getAttemptId())) { return; } final IntermediateResultPartition partition = resultPartitions.get(partitionId.getPartitionId()); if (partition == null) { throw new IllegalStateException("Unknown partition " + partitionId + "."); } if (partition.getIntermediateResult().getResultType().isPipelined()) { // Schedule or update receivers of this partition execution.scheduleOrUpdateConsumers(partition.getConsumers()); } else { throw new IllegalArgumentException("ScheduleOrUpdateConsumers msg is only valid for" + "pipelined partitions."); } }
/** * Schedules or updates the consumer tasks of the result partition with the given ID. */ void scheduleOrUpdateConsumers(ResultPartitionID partitionId) { final Execution execution = currentExecution; // Abort this request if there was a concurrent reset if (!partitionId.getProducerId().equals(execution.getAttemptId())) { return; } final IntermediateResultPartition partition = resultPartitions.get(partitionId.getPartitionId()); if (partition == null) { throw new IllegalStateException("Unknown partition " + partitionId + "."); } if (partition.getIntermediateResult().getResultType().isPipelined()) { // Schedule or update receivers of this partition partition.markDataProduced(); // Notify the scheduler to handle the consumable partition notifyAndUpdateConsumers(partition); } else { throw new IllegalArgumentException("ScheduleOrUpdateConsumers msg is only valid for" + "pipelined partitions."); } }
/** * Schedules or updates the consumer tasks of the result partition with the given ID. */ void scheduleOrUpdateConsumers(ResultPartitionID partitionId) { final Execution execution = currentExecution; // Abort this request if there was a concurrent reset if (!partitionId.getProducerId().equals(execution.getAttemptId())) { return; } final IntermediateResultPartition partition = resultPartitions.get(partitionId.getPartitionId()); if (partition == null) { throw new IllegalStateException("Unknown partition " + partitionId + "."); } if (partition.getIntermediateResult().getResultType().isPipelined()) { // Schedule or update receivers of this partition execution.scheduleOrUpdateConsumers(partition.getConsumers()); } else { throw new IllegalArgumentException("ScheduleOrUpdateConsumers msg is only valid for" + "pipelined partitions."); } }
/** * Creates a partial input channel for the given partition and producing task. */ public static PartialInputChannelDeploymentDescriptor fromEdge( IntermediateResultPartition partition, Execution producer) { final ResultPartitionID partitionId = new ResultPartitionID( partition.getPartitionId(), producer.getAttemptId()); final IntermediateResult result = partition.getIntermediateResult(); final TaskManagerLocation partitionConnectionInfo = producer.getAssignedResourceLocation(); return new PartialInputChannelDeploymentDescriptor( result, partitionId, partitionConnectionInfo); } }
private int calculateTaskExtraManagedMemory() { Configuration config = getJobVertex().getGraph().getJobManagerConfiguration(); // Calculates managed memory for external result partition. BlockingShuffleType shuffleType = BlockingShuffleType.getBlockingShuffleTypeFromConfiguration(config, LOG); int numExternalResultPartitions = 0; for (IntermediateResultPartition irp : getProducedPartitions().values()) { if (shuffleType == BlockingShuffleType.YARN && irp.getIntermediateResult().getResultType().isBlocking()) { numExternalResultPartitions++; } } int mapOutputMemoryInMB = config.getInteger(TaskManagerOptions.TASK_MANAGER_OUTPUT_MEMORY_MB); return mapOutputMemoryInMB * numExternalResultPartitions; }
protected void notifyAndUpdateConsumers(IntermediateResultPartition partition) { getExecutionGraph().getGraphManager().notifyResultPartitionConsumable( getExecutionVertexID(), partition.getIntermediateResult().getId(), partition.getPartitionNumber(), getCurrentAssignedResourceLocation()); getExecutionGraph().getFutureExecutor().execute(() -> { currentExecution.updateConsumers(partition.getConsumers()); }); }
/** * Creates a partial input channel for the given partition and producing task. */ public static PartialInputChannelDeploymentDescriptor fromEdge( IntermediateResultPartition partition, Execution producer) { final ResultPartitionID partitionId = new ResultPartitionID( partition.getPartitionId(), producer.getAttemptId()); final IntermediateResult result = partition.getIntermediateResult(); final IntermediateDataSetID resultId = result.getId(); final TaskManagerLocation partitionConnectionInfo = producer.getAssignedResourceLocation(); final int partitionConnectionIndex = result.getConnectionIndex(); return new PartialInputChannelDeploymentDescriptor( resultId, partitionId, partitionConnectionInfo, partitionConnectionIndex); } }
/** * Creates a partial input channel for the given partition and producing task. */ public static PartialInputChannelDeploymentDescriptor fromEdge( IntermediateResultPartition partition, Execution producer) { final ResultPartitionID partitionId = new ResultPartitionID( partition.getPartitionId(), producer.getAttemptId()); final IntermediateResult result = partition.getIntermediateResult(); final IntermediateDataSetID resultId = result.getId(); final TaskManagerLocation partitionConnectionInfo = producer.getAssignedResourceLocation(); final int partitionConnectionIndex = result.getConnectionIndex(); return new PartialInputChannelDeploymentDescriptor( resultId, partitionId, partitionConnectionInfo, partitionConnectionIndex); } }
/** * Creates a partial input channel for the given partition and producing task. */ public static PartialInputChannelDeploymentDescriptor fromEdge( IntermediateResultPartition partition, Execution producer) { final ResultPartitionID partitionId = new ResultPartitionID( partition.getPartitionId(), producer.getAttemptId()); final IntermediateResult result = partition.getIntermediateResult(); final IntermediateDataSetID resultId = result.getId(); final TaskManagerLocation partitionConnectionInfo = producer.getAssignedResourceLocation(); final int partitionConnectionIndex = result.getConnectionIndex(); return new PartialInputChannelDeploymentDescriptor( resultId, partitionId, partitionConnectionInfo, partitionConnectionIndex); } }
.getIntermediateResult().getPartitions();
/** * Recover the pipelined result partition consume status after job master failover. * * @param resultId The intermediate data set id in the log. */ public void recoverResultPartitionStatus( IntermediateDataSetID resultId, TaskManagerLocation location) { IntermediateResultPartition partitionToRecover = null; for (IntermediateResultPartition irp : getProducedPartitions().values()) { if (irp.getIntermediateResult().getId().equals(resultId)) { partitionToRecover = irp; } } if (partitionToRecover == null) { throw new FlinkRuntimeException("Can not find the intermediate result " + resultId + " on " + getTaskNameWithSubtaskIndex()); } if (!(ExecutionState.RUNNING.equals(currentExecution.getState()) && partitionToRecover.getResultType().isPipelined())) { throw new FlinkRuntimeException("Invalid state " + currentExecution.getState() + " for " + getTaskNameWithSubtaskIndex()); } currentExecution.getTaskManagerLocationFuture().complete(location); scheduleOrUpdateConsumers(new ResultPartitionID(partitionToRecover.getPartitionId(), currentExecution.getAttemptId())); }