assertEquals(4, execMapVertex.getParallelism()); ExecutionVertex[] mapTaskVertices = execMapVertex.getTaskVertices(); assertEquals(2, execSinkVertex.getParallelism()); ExecutionVertex[] sinkTaskVertices = execSinkVertex.getTaskVertices();
public int getTotalNumberOfParallelSubtasks() { return this.jobVertex.getParallelism(); }
public int getTotalNumberOfParallelSubtasks() { return this.jobVertex.getParallelism(); }
public int getTotalNumberOfParallelSubtasks() { return this.jobVertex.getParallelism(); }
public int getTotalNumberOfParallelSubtasks() { return this.jobVertex.getParallelism(); }
private void makeAllOneRegion(List<ExecutionJobVertex> jobVertices) { LOG.warn("Cannot decompose ExecutionGraph into individual failover regions due to use of " + "Co-Location constraints (iterations). Job will fail over as one holistic unit."); final ArrayList<ExecutionVertex> allVertices = new ArrayList<>(); for (ExecutionJobVertex ejv : jobVertices) { // safe some incremental size growing allVertices.ensureCapacity(allVertices.size() + ejv.getParallelism()); for (ExecutionVertex ev : ejv.getTaskVertices()) { allVertices.add(ev); } } final FailoverRegion singleRegion = new FailoverRegion(executionGraph, executor, allVertices, regionFailLimit); for (ExecutionVertex ev : allVertices) { vertexToRegion.put(ev, singleRegion); } }
private void makeAllOneRegion(List<ExecutionJobVertex> jobVertices) { LOG.warn("Cannot decompose ExecutionGraph into individual failover regions due to use of " + "Co-Location constraints (iterations). Job will fail over as one holistic unit."); final ArrayList<ExecutionVertex> allVertices = new ArrayList<>(); for (ExecutionJobVertex ejv : jobVertices) { // safe some incremental size growing allVertices.ensureCapacity(allVertices.size() + ejv.getParallelism()); for (ExecutionVertex ev : ejv.getTaskVertices()) { allVertices.add(ev); } } final FailoverRegion singleRegion = new FailoverRegion(executionGraph, executor, allVertices); for (ExecutionVertex ev : allVertices) { vertexToRegion.put(ev, singleRegion); } }
private void makeAllOneRegion(List<ExecutionJobVertex> jobVertices) { LOG.warn("Cannot decompose ExecutionGraph into individual failover regions due to use of " + "Co-Location constraints (iterations). Job will fail over as one holistic unit."); final ArrayList<ExecutionVertex> allVertices = new ArrayList<>(); for (ExecutionJobVertex ejv : jobVertices) { // safe some incremental size growing allVertices.ensureCapacity(allVertices.size() + ejv.getParallelism()); for (ExecutionVertex ev : ejv.getTaskVertices()) { allVertices.add(ev); } } final FailoverRegion singleRegion = new FailoverRegion(executionGraph, executor, allVertices); for (ExecutionVertex ev : allVertices) { vertexToRegion.put(ev, singleRegion); } }
private void makeAllOneRegion(List<ExecutionJobVertex> jobVertices) { LOG.warn("Cannot decompose ExecutionGraph into individual failover regions due to use of " + "Co-Location constraints (iterations). Job will fail over as one holistic unit."); final ArrayList<ExecutionVertex> allVertices = new ArrayList<>(); for (ExecutionJobVertex ejv : jobVertices) { // safe some incremental size growing allVertices.ensureCapacity(allVertices.size() + ejv.getParallelism()); for (ExecutionVertex ev : ejv.getTaskVertices()) { allVertices.add(ev); } } final FailoverRegion singleRegion = new FailoverRegion(executionGraph, executor, allVertices); for (ExecutionVertex ev : allVertices) { vertexToRegion.put(ev, singleRegion); } }
/** * Creates an empty map with a {@link TaskStateStats} instance per task * that is involved in the checkpoint. * * @return An empty map with an {@link TaskStateStats} entry for each task that is involved in the checkpoint. */ private ConcurrentHashMap<JobVertexID, TaskStateStats> createEmptyTaskStateStatsMap() { ConcurrentHashMap<JobVertexID, TaskStateStats> taskStatsMap = new ConcurrentHashMap<>(jobVertices.size()); for (ExecutionJobVertex vertex : jobVertices) { TaskStateStats taskStats = new TaskStateStats(vertex.getJobVertexId(), vertex.getParallelism()); taskStatsMap.put(vertex.getJobVertexId(), taskStats); } return taskStatsMap; }
/** * Creates an empty map with a {@link TaskStateStats} instance per task * that is involved in the checkpoint. * * @return An empty map with an {@link TaskStateStats} entry for each task that is involved in the checkpoint. */ private ConcurrentHashMap<JobVertexID, TaskStateStats> createEmptyTaskStateStatsMap() { ConcurrentHashMap<JobVertexID, TaskStateStats> taskStatsMap = new ConcurrentHashMap<>(jobVertices.size()); for (ExecutionJobVertex vertex : jobVertices) { TaskStateStats taskStats = new TaskStateStats(vertex.getJobVertexId(), vertex.getParallelism()); taskStatsMap.put(vertex.getJobVertexId(), taskStats); } return taskStatsMap; }
/** * Optimize the new {@link ExecutionGraph}. Enrich it with execution prior locations to improve the scheduling. * * @param newExecutionGraph is the new {@link ExecutionGraph} to optimize by enriching * @param oldExecutionGraph is the old {@link ExecutionGraph} to help with the optimization */ private void optimizeNewExecutionGraph(ExecutionGraph newExecutionGraph, ExecutionGraph oldExecutionGraph) { Map<JobVertexID, ExecutionJobVertex> oldVertices = oldExecutionGraph.getAllVertices(); Map<JobVertexID, ExecutionJobVertex> newVertices = newExecutionGraph.getAllVertices(); for (JobVertexID jobVertexID : newVertices.keySet()) { ExecutionJobVertex oldEjv = oldVertices.get(jobVertexID); if (oldEjv != null) { ExecutionJobVertex newEjv = newVertices.get(jobVertexID); int minParallelism = oldEjv.getParallelism() < newEjv.getParallelism() ? oldEjv.getParallelism() : newEjv.getParallelism(); for (int i = 0; i < minParallelism; i++) { newEjv.getTaskVertices()[i].setLatestPriorLocation( oldEjv.getTaskVertices()[i].getCurrentAssignedResourceLocation()); } } } }
/** * Creates an empty map with a {@link TaskStateStats} instance per task * that is involved in the checkpoint. * * @return An empty map with an {@link TaskStateStats} entry for each task that is involved in the checkpoint. */ private ConcurrentHashMap<JobVertexID, TaskStateStats> createEmptyTaskStateStatsMap() { ConcurrentHashMap<JobVertexID, TaskStateStats> taskStatsMap = new ConcurrentHashMap<>(jobVertices.size()); for (ExecutionJobVertex vertex : jobVertices) { TaskStateStats taskStats = new TaskStateStats(vertex.getJobVertexId(), vertex.getParallelism()); taskStatsMap.put(vertex.getJobVertexId(), taskStats); } return taskStatsMap; }
/** * Creates an empty map with a {@link TaskStateStats} instance per task * that is involved in the checkpoint. * * @return An empty map with an {@link TaskStateStats} entry for each task that is involved in the checkpoint. */ private ConcurrentHashMap<JobVertexID, TaskStateStats> createEmptyTaskStateStatsMap() { ConcurrentHashMap<JobVertexID, TaskStateStats> taskStatsMap = new ConcurrentHashMap<>(jobVertices.size()); for (ExecutionJobVertex vertex : jobVertices) { TaskStateStats taskStats = new TaskStateStats(vertex.getJobVertexId(), vertex.getParallelism()); taskStatsMap.put(vertex.getJobVertexId(), taskStats); } return taskStatsMap; }
public void attachJobGraph(List<JobVertex> topologicallySorted) throws JobException { LOG.debug("Attaching {} topologically sorted vertices to existing job graph with {} " + "vertices and {} intermediate results.", topologicallySorted.size(), tasks.size(), intermediateResults.size()); final ArrayList<ExecutionJobVertex> newExecJobVertices = new ArrayList<>(topologicallySorted.size()); createExecutionJobVertex(topologicallySorted); for (JobVertex jobVertex : topologicallySorted) { if (jobVertex.isInputVertex() && !jobVertex.isStoppable()) { this.isStoppable = false; } ExecutionJobVertex ejv = tasks.get(jobVertex.getID()); ejv.connectToPredecessors(this.intermediateResults); for (IntermediateResult res : ejv.getProducedDataSets()) { IntermediateResult previousDataSet = this.intermediateResults.putIfAbsent(res.getId(), res); if (previousDataSet != null) { throw new JobException(String.format("Encountered two intermediate data set with ID %s : previous=[%s] / new=[%s]", res.getId(), res, previousDataSet)); } } this.verticesInCreationOrder.add(ejv); this.numVerticesTotal += ejv.getParallelism(); newExecJobVertices.add(ejv); } terminationFuture = new CompletableFuture<>(); failoverStrategy.notifyNewVertices(newExecJobVertices); }
public SerializedValue<TaskInformation> getSerializedTaskInformation() throws IOException { if (null == serializedTaskInformation) { int parallelism = getParallelism(); int maxParallelism = getMaxParallelism(); if (LOG.isDebugEnabled()) { LOG.debug("Creating task information for " + generateDebugString()); } serializedTaskInformation = new SerializedValue<>( new TaskInformation( jobVertex.getID(), jobVertex.getName(), parallelism, maxParallelism, jobVertex.getInvokableClassName(), jobVertex.getConfiguration())); } return serializedTaskInformation; }
public ArchivedExecutionJobVertex(ExecutionJobVertex jobVertex) { this.taskVertices = new ArchivedExecutionVertex[jobVertex.getTaskVertices().length]; for (int x = 0; x < taskVertices.length; x++) { taskVertices[x] = jobVertex.getTaskVertices()[x].archive(); } archivedUserAccumulators = jobVertex.getAggregatedUserAccumulatorsStringified(); this.id = jobVertex.getJobVertexId(); this.name = jobVertex.getJobVertex().getName(); this.parallelism = jobVertex.getParallelism(); this.maxParallelism = jobVertex.getMaxParallelism(); }
public ArchivedExecutionJobVertex(ExecutionJobVertex jobVertex) { this.taskVertices = new ArchivedExecutionVertex[jobVertex.getTaskVertices().length]; for (int x = 0; x < taskVertices.length; x++) { taskVertices[x] = jobVertex.getTaskVertices()[x].archive(); } archivedUserAccumulators = jobVertex.getAggregatedUserAccumulatorsStringified(); this.id = jobVertex.getJobVertexId(); this.name = jobVertex.getJobVertex().getName(); this.parallelism = jobVertex.getParallelism(); this.maxParallelism = jobVertex.getMaxParallelism(); }
public ArchivedExecutionJobVertex(ExecutionJobVertex jobVertex) { this.taskVertices = new ArchivedExecutionVertex[jobVertex.getTaskVertices().length]; for (int x = 0; x < taskVertices.length; x++) { taskVertices[x] = jobVertex.getTaskVertices()[x].archive(); } archivedUserAccumulators = jobVertex.getAggregatedUserAccumulatorsStringified(); this.id = jobVertex.getJobVertexId(); this.name = jobVertex.getJobVertex().getName(); this.parallelism = jobVertex.getParallelism(); this.maxParallelism = jobVertex.getMaxParallelism(); }
public ArchivedExecutionJobVertex(ExecutionJobVertex jobVertex) { this.taskVertices = new ArchivedExecutionVertex[jobVertex.getTaskVertices().length]; for (int x = 0; x < taskVertices.length; x++) { taskVertices[x] = jobVertex.getTaskVertices()[x].archive(); } archivedUserAccumulators = jobVertex.getAggregatedUserAccumulatorsStringified(); this.id = jobVertex.getJobVertexId(); this.name = jobVertex.getJobVertex().getName(); this.parallelism = jobVertex.getParallelism(); this.maxParallelism = jobVertex.getMaxParallelism(); this.operatorDescriptors = jobVertex.getOperatorDescriptors(); }