public ArchivedExecutionJobVertex(ExecutionJobVertex jobVertex) { this.taskVertices = new ArchivedExecutionVertex[jobVertex.getTaskVertices().length]; for (int x = 0; x < taskVertices.length; x++) { taskVertices[x] = jobVertex.getTaskVertices()[x].archive(); } archivedUserAccumulators = jobVertex.getAggregatedUserAccumulatorsStringified(); this.id = jobVertex.getJobVertexId(); this.name = jobVertex.getJobVertex().getName(); this.parallelism = jobVertex.getParallelism(); this.maxParallelism = jobVertex.getMaxParallelism(); }
private ExecutionVertex[] collectExecutionVertices(List<ExecutionJobVertex> jobVertices) { if (jobVertices.size() == 1) { ExecutionJobVertex jv = jobVertices.get(0); if (jv.getGraph() != this) { throw new IllegalArgumentException("Can only use ExecutionJobVertices of this ExecutionGraph"); } return jv.getTaskVertices(); } else { ArrayList<ExecutionVertex> all = new ArrayList<>(); for (ExecutionJobVertex jv : jobVertices) { if (jv.getGraph() != this) { throw new IllegalArgumentException("Can only use ExecutionJobVertices of this ExecutionGraph"); } all.addAll(Arrays.asList(jv.getTaskVertices())); } return all.toArray(new ExecutionVertex[all.size()]); } }
ExecutionJobVertex execSinkVertex = eg.getJobVertex(sinkVertex.getID()); assertEquals(0, execSourceVertex.getInputs().size()); assertEquals(1, execMapVertex.getInputs().size()); assertEquals(4, execMapVertex.getParallelism()); ExecutionVertex[] mapTaskVertices = execMapVertex.getTaskVertices(); assertEquals(1, execSinkVertex.getInputs().size()); assertEquals(2, execSinkVertex.getParallelism()); ExecutionVertex[] sinkTaskVertices = execSinkVertex.getTaskVertices();
/** * Creates an empty map with a {@link TaskStateStats} instance per task * that is involved in the checkpoint. * * @return An empty map with an {@link TaskStateStats} entry for each task that is involved in the checkpoint. */ private ConcurrentHashMap<JobVertexID, TaskStateStats> createEmptyTaskStateStatsMap() { ConcurrentHashMap<JobVertexID, TaskStateStats> taskStatsMap = new ConcurrentHashMap<>(jobVertices.size()); for (ExecutionJobVertex vertex : jobVertices) { TaskStateStats taskStats = new TaskStateStats(vertex.getJobVertexId(), vertex.getParallelism()); taskStatsMap.put(vertex.getJobVertexId(), taskStats); } return taskStatsMap; }
if (ejv.getCoLocationGroup() != null) { makeAllOneRegion(newJobVerticesTopological); return; final List<IntermediateResult> inputs = ejv.getInputs(); final int numInputs = inputs.size(); boolean hasPipelinedInputs = false; for (ExecutionVertex ev : ejv.getTaskVertices()) { for (ExecutionVertex ev : ejv.getTaskVertices()) { ArrayList<ExecutionVertex> region = new ArrayList<>(1); region.add(ev);
ExecutionJobVertex ejv = new ExecutionJobVertex( this, jobVertex, createTimestamp); ejv.connectToPredecessors(this.intermediateResults); for (IntermediateResult res : ejv.getProducedDataSets()) { IntermediateResult previousDataSet = this.intermediateResults.putIfAbsent(res.getId(), res); if (previousDataSet != null) { this.numVerticesTotal += ejv.getParallelism(); newExecJobVertices.add(ejv);
private void makeAllOneRegion(List<ExecutionJobVertex> jobVertices) { LOG.warn("Cannot decompose ExecutionGraph into individual failover regions due to use of " + "Co-Location constraints (iterations). Job will fail over as one holistic unit."); final ArrayList<ExecutionVertex> allVertices = new ArrayList<>(); for (ExecutionJobVertex ejv : jobVertices) { // safe some incremental size growing allVertices.ensureCapacity(allVertices.size() + ejv.getParallelism()); for (ExecutionVertex ev : ejv.getTaskVertices()) { allVertices.add(ev); } } final FailoverRegion singleRegion = new FailoverRegion(executionGraph, executor, allVertices, regionFailLimit); for (ExecutionVertex ev : allVertices) { vertexToRegion.put(ev, singleRegion); } }
/** * Cancels all currently running vertex executions. * * @return A future that is complete once all tasks have canceled. */ public CompletableFuture<Void> cancelWithFuture() { // we collect all futures from the task cancellations CompletableFuture<ExecutionState>[] futures = Arrays.stream(getTaskVertices()) .map(ExecutionVertex::cancel) .<CompletableFuture<ExecutionState>>toArray(CompletableFuture[]::new); // return a conjunct future, which is complete once all individual tasks are canceled return CompletableFuture.allOf(futures); }
@Override public ExecutionState getAggregateState() { int[] num = new int[ExecutionState.values().length]; for (ExecutionVertex vertex : this.taskVertices) { num[vertex.getExecutionState().ordinal()]++; } return getAggregateJobVertexState(num, parallelism); }
@Override public void notifyNewVertices(List<ExecutionJobVertex> newJobVerticesTopological) { // we validate here that the vertices are in fact not connected to // any other vertices for (ExecutionJobVertex ejv : newJobVerticesTopological) { List<IntermediateResult> inputs = ejv.getInputs(); IntermediateResult[] outputs = ejv.getProducedDataSets(); if ((inputs != null && inputs.size() > 0) || (outputs != null && outputs.length > 0)) { throw new FlinkRuntimeException("Incompatible failover strategy - strategy '" + getStrategyName() + "' can only handle jobs with only disconnected tasks."); } } }
List<ArchivedExecutionJobVertex> archivedVerticesInCreationOrder = new ArrayList<>(); for (ExecutionJobVertex task : verticesInCreationOrder) { ArchivedExecutionJobVertex archivedTask = task.archive(); archivedVerticesInCreationOrder.add(archivedTask); archivedTasks.put(task.getJobVertexId(), archivedTask);
int parallelism = vertex.getMaxParallelism(); location = new KvStateLocation(jobId, jobVertexId, parallelism, registrationName); lookupTable.put(registrationName, location); vertex.fail(new SuppressRestartsException(duplicate));
public ExecutionGraph getExecutionGraph() { return this.jobVertex.getGraph(); }
final InputSplitAssigner splitAssigner = vertex.getSplitAssigner(); if (splitAssigner == null) { log.error("No InputSplitAssigner for vertex ID {}.", vertexID); IOException reason = new IOException("Could not serialize the next input split of class " + nextInputSplit.getClass() + ".", ex); vertex.fail(reason); throw reason;
executionJobVertexTerminationFutures.add(ejv.cancelWithFuture());
public void attachJobGraph(List<JobVertex> topologicallySorted) throws JobException { LOG.debug("Attaching {} topologically sorted vertices to existing job graph with {} " + "vertices and {} intermediate results.", topologicallySorted.size(), tasks.size(), intermediateResults.size()); final ArrayList<ExecutionJobVertex> newExecJobVertices = new ArrayList<>(topologicallySorted.size()); createExecutionJobVertex(topologicallySorted); for (JobVertex jobVertex : topologicallySorted) { if (jobVertex.isInputVertex() && !jobVertex.isStoppable()) { this.isStoppable = false; } ExecutionJobVertex ejv = tasks.get(jobVertex.getID()); ejv.connectToPredecessors(this.intermediateResults); for (IntermediateResult res : ejv.getProducedDataSets()) { IntermediateResult previousDataSet = this.intermediateResults.putIfAbsent(res.getId(), res); if (previousDataSet != null) { throw new JobException(String.format("Encountered two intermediate data set with ID %s : previous=[%s] / new=[%s]", res.getId(), res, previousDataSet)); } } this.verticesInCreationOrder.add(ejv); this.numVerticesTotal += ejv.getParallelism(); newExecJobVertices.add(ejv); } terminationFuture = new CompletableFuture<>(); failoverStrategy.notifyNewVertices(newExecJobVertices); }
public int getTotalNumberOfParallelSubtasks() { return this.jobVertex.getParallelism(); }
getGraph().getGraphManager().notifyInputSplitsCreated(getJobVertexId(), inputSplitsMap); } finally { currentThread.setContextClassLoader(oldContextClassLoader);
for (IntermediateResult intermediateResult : ev.getJobVertex().getInputs()) { IntermediateResultPartition resultPartition = intermediateResult.getPartitionOrNullById( predecessorResultPartition.getPartitionId());
futures.add(CompletableFuture.supplyAsync(() -> { try { ExecutionJobVertex ejv = new ExecutionJobVertex( this, jobVertex,