assertEquals(1, mapTaskVertex.getInputEdges(0).length); ExecutionEdge inputEdge = mapTaskVertex.getInputEdges(0)[0]; assertEquals(sourceVertex.getID(), inputEdge.getSource().getProducer().getJobvertexId()); int inputPartition = inputEdge.getSource().getPartitionNumber(); if (!mapInputPartitionCounts.containsKey(inputPartition)) { mapInputPartitionCounts.put(inputPartition, 1); ExecutionEdge inputEdge1 = sinkTaskVertex.getInputEdges(0)[0]; ExecutionEdge inputEdge2 = sinkTaskVertex.getInputEdges(0)[1]; assertEquals(mapVertex.getID(), inputEdge1.getSource().getProducer().getJobvertexId()); assertEquals(mapVertex.getID(), inputEdge2.getSource().getProducer().getJobvertexId()); int inputPartition1 = inputEdge1.getSource().getPartitionNumber(); assertFalse(mapSubpartitions.contains(inputPartition1)); mapSubpartitions.add(inputPartition1); int inputPartition2 = inputEdge2.getSource().getPartitionNumber(); assertFalse(mapSubpartitions.contains(inputPartition2)); mapSubpartitions.add(inputPartition2);
public int registerConsumer() { final int index = numConsumers; numConsumers++; for (IntermediateResultPartition p : partitions) { if (p.addConsumerGroup() != index) { throw new RuntimeException("Inconsistent consumer mapping between intermediate result partitions."); } } return index; }
public static ResultPartitionDeploymentDescriptor from( IntermediateResultPartition partition, int maxParallelism, boolean lazyScheduling) { final IntermediateDataSetID resultId = partition.getIntermediateResult().getId(); final IntermediateResultPartitionID partitionId = partition.getPartitionId(); final ResultPartitionType partitionType = partition.getIntermediateResult().getResultType(); // The produced data is partitioned among a number of subpartitions. // // If no consumers are known at this point, we use a single subpartition, otherwise we have // one for each consuming sub task. int numberOfSubpartitions = 1; if (!partition.getConsumers().isEmpty() && !partition.getConsumers().get(0).isEmpty()) { if (partition.getConsumers().size() > 1) { throw new IllegalStateException("Currently, only a single consumer group per partition is supported."); } numberOfSubpartitions = partition.getConsumers().get(0).size(); } return new ResultPartitionDeploymentDescriptor( resultId, partitionId, partitionType, numberOfSubpartitions, maxParallelism, lazyScheduling); } }
public void resetForNewExecution() { if (isConsumable()) { getIntermediateResult().decrementNumberOfConsumablePartitions(); } isFinished = false; dataProduced = false; }
public void markFinished() { if (getResultType().isBlocking() && !isConsumable()) { getIntermediateResult().incrementNumberOfConsumablePartitions(); } isFinished = true; } }
final Execution producer = consumedPartition.getProducer().getCurrentExecutionAttempt(); if ((consumedPartition.getResultType().isPipelined() || consumedPartition.isConsumable()) && producerSlot != null && (producerState == ExecutionState.RUNNING || partitionTaskManagerLocation, consumerLocation, consumedPartition.getIntermediateResult()); String msg = String.format("Trying to eagerly schedule a task whose inputs are not ready " + "(result type: %s, partition consumable: %s, producer state: %s, producer slot: %s).", consumedPartition.getResultType(), consumedPartition.isConsumable(), producerState, producerSlot); consumedPartition.getPartitionId(), producer.getAttemptId());
/** * Schedules or updates the consumer tasks of the result partition with the given ID. */ void scheduleOrUpdateConsumers(ResultPartitionID partitionId) { final Execution execution = currentExecution; // Abort this request if there was a concurrent reset if (!partitionId.getProducerId().equals(execution.getAttemptId())) { return; } final IntermediateResultPartition partition = resultPartitions.get(partitionId.getPartitionId()); if (partition == null) { throw new IllegalStateException("Unknown partition " + partitionId + "."); } if (partition.getIntermediateResult().getResultType().isPipelined()) { // Schedule or update receivers of this partition execution.scheduleOrUpdateConsumers(partition.getConsumers()); } else { throw new IllegalArgumentException("ScheduleOrUpdateConsumers msg is only valid for" + "pipelined partitions."); } }
final Execution partitionExecution = partition.getProducer() .getCurrentExecutionAttempt(); final TaskManagerLocation partitionTaskManagerLocation = partition.getProducer() .getCurrentAssignedResource().getTaskManagerLocation(); final ResourceID partitionTaskManager = partitionTaskManagerLocation.getResourceID(); final ResultPartitionID partitionId = new ResultPartitionID(partition.getPartitionId(), attemptId); partition.getIntermediateResult().getConnectionIndex()); Collections.singleton( new PartitionInfo( partition.getIntermediateResult().getId(), descriptor))); final Execution partitionExecution = partition.getProducer() .getCurrentExecutionAttempt();
SimpleSlot sourceSlot = sources[k].getSource().getProducer().getCurrentAssignedResource(); if (sourceSlot != null) {
/** * Finish all blocking result partitions whose receivers can be scheduled/updated and notify. */ void finishPartitionsAndNotify() { for (IntermediateResultPartition partition : resultPartitions.values()) { partition.markFinished(); // Blocking partitions are consumable on finished if (partition.getResultType().isBlocking()) { notifyAndUpdateConsumers(partition); } } }
/** * Creates a partial input channel for the given partition and producing task. */ public static PartialInputChannelDeploymentDescriptor fromEdge( IntermediateResultPartition partition, Execution producer) { final ResultPartitionID partitionId = new ResultPartitionID( partition.getPartitionId(), producer.getAttemptId()); final IntermediateResult result = partition.getIntermediateResult(); final TaskManagerLocation partitionConnectionInfo = producer.getAssignedResourceLocation(); return new PartialInputChannelDeploymentDescriptor( result, partitionId, partitionConnectionInfo); } }
IntermediateResultPartition irp = new IntermediateResultPartition(result, this, subTaskIndex); result.setPartition(subTaskIndex, irp); resultPartitions.put(irp.getPartitionId(), irp);
/** * Recover the pipelined result partition consume status after job master failover. * * @param resultId The intermediate data set id in the log. */ public void recoverResultPartitionStatus( IntermediateDataSetID resultId, TaskManagerLocation location) { IntermediateResultPartition partitionToRecover = null; for (IntermediateResultPartition irp : getProducedPartitions().values()) { if (irp.getIntermediateResult().getId().equals(resultId)) { partitionToRecover = irp; } } if (partitionToRecover == null) { throw new FlinkRuntimeException("Can not find the intermediate result " + resultId + " on " + getTaskNameWithSubtaskIndex()); } if (!(ExecutionState.RUNNING.equals(currentExecution.getState()) && partitionToRecover.getResultType().isPipelined())) { throw new FlinkRuntimeException("Invalid state " + currentExecution.getState() + " for " + getTaskNameWithSubtaskIndex()); } currentExecution.getTaskManagerLocationFuture().complete(location); scheduleOrUpdateConsumers(new ResultPartitionID(partitionToRecover.getPartitionId(), currentExecution.getAttemptId())); }
protected void notifyAndUpdateConsumers(IntermediateResultPartition partition) { getExecutionGraph().getGraphManager().notifyResultPartitionConsumable( getExecutionVertexID(), partition.getIntermediateResult().getId(), partition.getPartitionNumber(), getCurrentAssignedResourceLocation()); getExecutionGraph().getFutureExecutor().execute(() -> { currentExecution.updateConsumers(partition.getConsumers()); }); }
void resetResultPartitionID(ResultPartitionID[] partitionIds) { Map<IntermediateResultPartitionID, IntermediateResultPartition> newResultPartitions = new LinkedHashMap<>(resultPartitions.size()); Iterator<IntermediateResultPartition> iterator = resultPartitions.values().iterator(); for (int i = 0; i < resultPartitions.size(); i++) { IntermediateResultPartition resultPartition = iterator.next(); IntermediateResultPartitionID originId = resultPartition.getPartitionId(); resultPartition.setPartitionId(partitionIds[i].getPartitionId()); resultPartition.getIntermediateResult().resetLookupHelper(originId, partitionIds[i].getPartitionId()); newResultPartitions.put(resultPartition.getPartitionId(), resultPartition); } this.resultPartitions = newResultPartitions; }
if (partition.getResultType().isPipelined()) { if (partition.hasDataProduced() != partitionsConsumable[i]) { LOG.info("Reconcile execution {} failed due to partition {} consumable not equals to {}.", getTaskNameWithSubtaskIndex(), partition.getPartitionId(), partition.hasDataProduced());
public void setPartition(int partitionNumber, IntermediateResultPartition partition) { if (partition == null || partitionNumber < 0 || partitionNumber >= numParallelProducers) { throw new IllegalArgumentException(); } if (partitions[partitionNumber] != null) { throw new IllegalStateException("Partition #" + partitionNumber + " has already been assigned."); } partitions[partitionNumber] = partition; partitionLookupHelper.put(partition.getPartitionId(), partitionNumber); partitionsAssigned++; }
public boolean isConsumable() { if (getResultType().isPipelined()) { return dataProduced; } else { return isFinished; } }
public void connectSource(int inputNumber, IntermediateResult source, JobEdge edge, int consumerNumber) { final DistributionPattern pattern = edge.getDistributionPattern(); final IntermediateResultPartition[] sourcePartitions = source.getPartitions(); ExecutionEdge[] edges; switch (pattern) { case POINTWISE: edges = connectPointwise(sourcePartitions, inputNumber); break; case ALL_TO_ALL: edges = connectAllToAll(sourcePartitions, inputNumber); break; default: throw new RuntimeException("Unrecognized distribution pattern."); } this.inputEdges[inputNumber] = edges; // add the consumers to the source // for now (until the receiver initiated handshake is in place), we need to register the // edges as the execution graph for (ExecutionEdge ee : edges) { ee.getSource().addConsumer(ee, consumerNumber); } }
/** * Schedules or updates the consumer tasks of the result partition with the given ID. */ void scheduleOrUpdateConsumers(ResultPartitionID partitionId) { final Execution execution = currentExecution; // Abort this request if there was a concurrent reset if (!partitionId.getProducerId().equals(execution.getAttemptId())) { return; } final IntermediateResultPartition partition = resultPartitions.get(partitionId.getPartitionId()); if (partition == null) { throw new IllegalStateException("Unknown partition " + partitionId + "."); } if (partition.getIntermediateResult().getResultType().isPipelined()) { // Schedule or update receivers of this partition partition.markDataProduced(); // Notify the scheduler to handle the consumable partition notifyAndUpdateConsumers(partition); } else { throw new IllegalArgumentException("ScheduleOrUpdateConsumers msg is only valid for" + "pipelined partitions."); } }