public boolean isConsumable() { if (getResultType().isPipelined()) { return dataProduced; } else { return isFinished; } }
boolean markFinished() { // Sanity check that this is only called on blocking partitions. if (!getResultType().isBlocking()) { throw new IllegalStateException("Tried to mark a non-blocking result partition as finished"); } final int refCnt = totalResult.decrementNumberOfRunningProducersAndGetRemaining(); if (refCnt == 0) { return true; } else if (refCnt < 0) { throw new IllegalStateException("Decremented number of unfinished producers below 0. " + "This is most likely a bug in the execution state/intermediate result " + "partition management."); } return false; } }
/** * Returns all blocking result partitions whose receivers can be scheduled/updated. */ List<IntermediateResultPartition> finishAllBlockingPartitions() { List<IntermediateResultPartition> finishedBlockingPartitions = null; for (IntermediateResultPartition partition : resultPartitions.values()) { if (partition.getResultType().isBlocking() && partition.markFinished()) { if (finishedBlockingPartitions == null) { finishedBlockingPartitions = new LinkedList<IntermediateResultPartition>(); } finishedBlockingPartitions.add(partition); } } if (finishedBlockingPartitions == null) { return Collections.emptyList(); } else { return finishedBlockingPartitions; } }
boolean markFinished() { // Sanity check that this is only called on blocking partitions. if (!getResultType().isBlocking()) { throw new IllegalStateException("Tried to mark a non-blocking result partition as finished"); } final int refCnt = totalResult.decrementNumberOfRunningProducersAndGetRemaining(); if (refCnt == 0) { return true; } else if (refCnt < 0) { throw new IllegalStateException("Decremented number of unfinished producers below 0. " + "This is most likely a bug in the execution state/intermediate result " + "partition management."); } return false; } }
/** * Returns all blocking result partitions whose receivers can be scheduled/updated. */ List<IntermediateResultPartition> finishAllBlockingPartitions() { List<IntermediateResultPartition> finishedBlockingPartitions = null; for (IntermediateResultPartition partition : resultPartitions.values()) { if (partition.getResultType().isBlocking() && partition.markFinished()) { if (finishedBlockingPartitions == null) { finishedBlockingPartitions = new LinkedList<IntermediateResultPartition>(); } finishedBlockingPartitions.add(partition); } } if (finishedBlockingPartitions == null) { return Collections.emptyList(); } else { return finishedBlockingPartitions; } }
/** * Returns all blocking result partitions whose receivers can be scheduled/updated. */ List<IntermediateResultPartition> finishAllBlockingPartitions() { List<IntermediateResultPartition> finishedBlockingPartitions = null; for (IntermediateResultPartition partition : resultPartitions.values()) { if (partition.getResultType().isBlocking() && partition.markFinished()) { if (finishedBlockingPartitions == null) { finishedBlockingPartitions = new LinkedList<IntermediateResultPartition>(); } finishedBlockingPartitions.add(partition); } } if (finishedBlockingPartitions == null) { return Collections.emptyList(); } else { return finishedBlockingPartitions; } }
boolean markFinished() { // Sanity check that this is only called on blocking partitions. if (!getResultType().isBlocking()) { throw new IllegalStateException("Tried to mark a non-blocking result partition as finished"); } final int refCnt = totalResult.decrementNumberOfRunningProducersAndGetRemaining(); if (refCnt == 0) { return true; } else if (refCnt < 0) { throw new IllegalStateException("Decremented number of unfinished producers below 0. " + "This is most likely a bug in the execution state/intermediate result " + "partition management."); } return false; } }
/** * Finish all blocking result partitions whose receivers can be scheduled/updated and notify. */ void finishPartitionsAndNotify() { for (IntermediateResultPartition partition : resultPartitions.values()) { partition.markFinished(); // Blocking partitions are consumable on finished if (partition.getResultType().isBlocking()) { notifyAndUpdateConsumers(partition); } } }
public void markFinished() { if (getResultType().isBlocking() && !isConsumable()) { getIntermediateResult().incrementNumberOfConsumablePartitions(); } isFinished = true; } }
if (partition.getResultType().isPipelined()) { if (partition.hasDataProduced() != partitionsConsumable[i]) { LOG.info("Reconcile execution {} failed due to partition {} consumable not equals to {}.",
/** * Recover the pipelined result partition consume status after job master failover. * * @param resultId The intermediate data set id in the log. */ public void recoverResultPartitionStatus( IntermediateDataSetID resultId, TaskManagerLocation location) { IntermediateResultPartition partitionToRecover = null; for (IntermediateResultPartition irp : getProducedPartitions().values()) { if (irp.getIntermediateResult().getId().equals(resultId)) { partitionToRecover = irp; } } if (partitionToRecover == null) { throw new FlinkRuntimeException("Can not find the intermediate result " + resultId + " on " + getTaskNameWithSubtaskIndex()); } if (!(ExecutionState.RUNNING.equals(currentExecution.getState()) && partitionToRecover.getResultType().isPipelined())) { throw new FlinkRuntimeException("Invalid state " + currentExecution.getState() + " for " + getTaskNameWithSubtaskIndex()); } currentExecution.getTaskManagerLocationFuture().complete(location); scheduleOrUpdateConsumers(new ResultPartitionID(partitionToRecover.getPartitionId(), currentExecution.getAttemptId())); }
if ((consumedPartition.getResultType().isPipelined() || consumedPartition.isConsumable()) && producerSlot != null && String msg = String.format("Trying to eagerly schedule a task whose inputs are not ready " + "(result type: %s, partition consumable: %s, producer state: %s, producer slot: %s).", consumedPartition.getResultType(), consumedPartition.isConsumable(), producerState,