List<JobVertex> jobVertices = jobGraph.getVerticesSortedTopologicallyFromSources(); assertEquals(2, sourceVertex.getParallelism()); assertEquals(4, mapVertex.getParallelism()); assertEquals(2, sinkVertex.getParallelism()); final JobInformation jobInformation = new DummyJobInformation( jobId, jobName); ExecutionGraph eg = new ExecutionGraph( jobInformation, TestingUtils.defaultExecutor(), TestingUtils.defaultExecutor(), AkkaUtils.getDefaultTimeout(), new NoRestartStrategy(), new RestartAllStrategy.Factory(), new TestingSlotProvider(ignored -> new CompletableFuture<>()), ExecutionGraph.class.getClassLoader(), VoidBlobWriter.getInstance(), AkkaUtils.getDefaultTimeout()); try { eg.attachJobGraph(jobVertices); e.printStackTrace(); fail("Building ExecutionGraph failed: " + e.getMessage()); ExecutionJobVertex execSourceVertex = eg.getJobVertex(sourceVertex.getID()); ExecutionJobVertex execMapVertex = eg.getJobVertex(mapVertex.getID());
public void stop() throws StoppingException { if (isStoppable) { for (ExecutionVertex ev : this.getAllExecutionVertices()) { if (ev.getNumberOfInputs() == 0) { // send signal to sources only ev.stop(); } } } else { throw new StoppingException("This job is not stoppable."); } }
public void scheduleForExecution() throws JobException { if (transitionState(JobStatus.CREATED, JobStatus.RUNNING)) { switch (scheduleMode) { case LAZY_FROM_SOURCES: scheduleLazy(slotProvider); break; case EAGER: scheduleEager(slotProvider, scheduleAllocationTimeout); break; default: throw new JobException("Schedule mode is invalid."); } } else { throw new IllegalStateException("Job may only be scheduled from state " + JobStatus.CREATED); } }
public void attachJobGraph(List<JobVertex> topologicallySorted) throws JobException { LOG.debug("Attaching {} topologically sorted vertices to existing job graph with {} " + "vertices and {} intermediate results.", topologicallySorted.size(), tasks.size(), intermediateResults.size()); final ArrayList<ExecutionJobVertex> newExecJobVertices = new ArrayList<>(topologicallySorted.size()); createExecutionJobVertex(topologicallySorted); for (JobVertex jobVertex : topologicallySorted) { if (jobVertex.isInputVertex() && !jobVertex.isStoppable()) { this.isStoppable = false; } ExecutionJobVertex ejv = tasks.get(jobVertex.getID()); ejv.connectToPredecessors(this.intermediateResults); for (IntermediateResult res : ejv.getProducedDataSets()) { IntermediateResult previousDataSet = this.intermediateResults.putIfAbsent(res.getId(), res); if (previousDataSet != null) { throw new JobException(String.format("Encountered two intermediate data set with ID %s : previous=[%s] / new=[%s]", res.getId(), res, previousDataSet)); } } this.verticesInCreationOrder.add(ejv); this.numVerticesTotal += ejv.getParallelism(); newExecJobVertices.add(ejv); } terminationFuture = new CompletableFuture<>(); failoverStrategy.notifyNewVertices(newExecJobVertices); }
public void connectToPredecessors(Map<IntermediateDataSetID, IntermediateResult> intermediateDataSets) throws JobException { List<JobEdge> inputs = jobVertex.getInputs(); LOG.debug(String.format("Connecting ExecutionJobVertex %s (%s) to %d predecessors.", jobVertex.getID(), jobVertex.getName(), inputs.size())); if (edge.getSource() == null) { LOG.debug(String.format("Connecting input %d of vertex %s (%s) to intermediate result referenced via ID %s.", num, jobVertex.getID(), jobVertex.getName(), edge.getSourceId())); } else { LOG.debug(String.format("Connecting input %d of vertex %s (%s) to intermediate result referenced via predecessor %s (%s).", num, jobVertex.getID(), jobVertex.getName(), edge.getSource().getProducer().getID(), edge.getSource().getProducer().getName())); IntermediateResult ires = intermediateDataSets.get(edge.getSourceId()); if (ires == null) { throw new JobException("Cannot connect this job graph to the previous graph. No previous intermediate result found for ID " + edge.getSourceId()); int consumerIndex = ires.registerConsumer(); ev.connectSource(num, ires, edge, consumerIndex);
List<JobEdge> inputs = jobVertex.getInputs(); LOG.debug(String.format("Connecting ExecutionJobVertex %s (%s) to %d predecessors.", jobVertex.getID(), jobVertex.getName(), inputs.size())); if (edge.getSource() == null) { LOG.debug(String.format("Connecting input %d of vertex %s (%s) to intermediate result referenced via ID %s, %s.", num, jobVertex.getID(), jobVertex.getName(), edge.getSourceId(), edge.getDistributionPattern())); } else { LOG.debug(String.format("Connecting input %d of vertex %s (%s) to intermediate result referenced via predecessor %s (%s), %s, %s", num, jobVertex.getID(), jobVertex.getName(), edge.getSource().getProducer().getID(), edge.getSource().getProducer().getName(), edge.getDistributionPattern(), edge.getSource().getResultType())); IntermediateResult ires = intermediateDataSets.get(edge.getSourceId()); if (ires == null) { throw new JobException("Cannot connect this job graph to the previous graph. No previous intermediate result found for ID " + edge.getSourceId()); int consumerIndex = ires.registerConsumer(); for (int i = 0; i < ires.getPartitions().length; i++) { IntermediateResultPartition partition = ires.getPartitions()[i]; Collection<ExecutionVertexID> consumerExecutionVertices = edge.getConsumerExecutionVertices(i); ExecutionVertex consumerVertex = taskVertices[executionVertexID.getSubTaskIndex()]; ExecutionEdge ee = new ExecutionEdge(partition, consumerVertex, num); partition.addConsumer(ee, consumerIndex); executionEdges.get(executionVertexID.getSubTaskIndex()).add(ee); ev.setInputExecutionEdges(executionEdges.get(i).toArray(new ExecutionEdge[]{}), num);
if (!slot.isAlive()) { throw new JobException("Target slot (TaskManager) for deployment is no longer alive."); if (!transitionState(previous, DEPLOYING)) { if (this != slot.getPayload()) { throw new IllegalStateException( String.format("The execution %s has not been assigned to the assigned slot.", this)); slot.releaseSlot(new FlinkException("Actual state of execution " + this + " (" + state + ") does not match expected state DEPLOYING.")); return; LOG.info(String.format("Deploying %s (attempt #%d) to %s", vertex.getTaskNameWithSubtaskIndex(), attemptNumber, getAssignedResourceLocation().getHostname())); final TaskDeploymentDescriptor deployment = vertex.createDeploymentDescriptor( attemptId, slot, final TaskManagerGateway taskManagerGateway = slot.getTaskManagerGateway(); final CompletableFuture<Acknowledge> submitResultFuture = taskManagerGateway.submitTask(deployment, rpcTimeout); String taskname = vertex.getTaskNameWithSubtaskIndex() + " (" + attemptId + ')'; markFailed(new Exception( "Cannot deploy task " + taskname + " - TaskManager (" + getAssignedResourceLocation() + ") not responding after a rpcTimeout of " + rpcTimeout, failure)); } else {
if (!slot.isAlive()) { throw new JobException("Target slot (TaskManager) for deployment is no longer alive."); if (!transitionState(previous, DEPLOYING)) { if (!slot.setExecutedVertex(this)) { throw new JobException("Could not assign the ExecutionVertex to the slot " + slot); slot.releaseSlot(); return; LOG.info(String.format("Deploying %s (attempt #%d) to %s", vertex.getTaskNameWithSubtaskIndex(), attemptNumber, getAssignedResourceLocation().getHostname())); final TaskDeploymentDescriptor deployment = vertex.createDeploymentDescriptor( attemptId, slot, final TaskManagerGateway taskManagerGateway = slot.getTaskManagerGateway(); final Future<Acknowledge> submitResultFuture = taskManagerGateway.submitTask(deployment, timeout); submitResultFuture.exceptionallyAsync(new ApplyFunction<Throwable, Void>() { @Override public Void apply(Throwable failure) { markFailed(t); ExceptionUtils.rethrow(t);
futures.add(CompletableFuture.supplyAsync(() -> { try { ExecutionJobVertex ejv = new ExecutionJobVertex( this, jobVertex, getJobConfiguration().getInteger(CoreOptions.DEFAULT_PARALLELISM), rpcTimeout, globalModVersion, createTimestamp); ExecutionJobVertex previousTask = tasks.putIfAbsent(jobVertex.getID(), ejv); if (previousTask != null) { throw new JobException( String.format( "Encountered two job vertices with ID %s : previous=[%s] / new=[%s]", jobVertex.getID(), ejv, previousTask)); Collection<JobException> exceptions = FutureUtils.combineAll(futures).get(); throw new JobException("Could not create execution job vertex.", e);
Map<OperatorID, InputSplitSource<?>> splitSourceMap = jobVertex.getInputSplitSources(); currentThread.setContextClassLoader(graph.getUserClassLoader()); getGraph().getGraphManager().notifyInputSplitsCreated(getJobVertexId(), inputSplitsMap); } finally { currentThread.setContextClassLoader(oldContextClassLoader); throw new JobException("Creating the input splits caused an error: " + t.getMessage(), t);
if (jobVertex.isInputVertex() && !jobVertex.isStoppable()) { this.isStoppable = false; new ExecutionJobVertex(this, jobVertex, 1, rpcCallTimeout, globalModVersion, createTimestamp); ejv.connectToPredecessors(this.intermediateResults); ExecutionJobVertex previousTask = this.tasks.putIfAbsent(jobVertex.getID(), ejv); if (previousTask != null) { throw new JobException(String.format("Encountered two job vertices with ID %s : previous=[%s] / new=[%s]", jobVertex.getID(), ejv, previousTask)); for (IntermediateResult res : ejv.getProducedDataSets()) { IntermediateResult previousDataSet = this.intermediateResults.putIfAbsent(res.getId(), res); if (previousDataSet != null) { throw new JobException(String.format("Encountered two intermediate data set with ID %s : previous=[%s] / new=[%s]", res.getId(), res, previousDataSet)); this.numVerticesTotal += ejv.getParallelism(); newExecJobVertices.add(ejv); terminationFuture = new FlinkCompletableFuture<>(); failoverStrategy.notifyNewVertices(newExecJobVertices);
public void connectToPredecessors(Map<IntermediateDataSetID, IntermediateResult> intermediateDataSets) throws JobException { List<JobEdge> inputs = jobVertex.getInputs(); LOG.debug(String.format("Connecting ExecutionJobVertex %s (%s) to %d predecessors.", jobVertex.getID(), jobVertex.getName(), inputs.size())); if (edge.getSource() == null) { LOG.debug(String.format("Connecting input %d of vertex %s (%s) to intermediate result referenced via ID %s.", num, jobVertex.getID(), jobVertex.getName(), edge.getSourceId())); } else { LOG.debug(String.format("Connecting input %d of vertex %s (%s) to intermediate result referenced via predecessor %s (%s).", num, jobVertex.getID(), jobVertex.getName(), edge.getSource().getProducer().getID(), edge.getSource().getProducer().getName())); IntermediateResult ires = intermediateDataSets.get(edge.getSourceId()); if (ires == null) { throw new JobException("Cannot connect this job graph to the previous graph. No previous intermediate result found for ID " + edge.getSourceId()); int consumerIndex = ires.registerConsumer(); ev.connectSource(num, ires, edge, consumerIndex);
public void stop() throws StoppingException { if (isStoppable) { for (ExecutionVertex ev : this.getAllExecutionVertices()) { if (ev.getNumberOfInputs() == 0) { // send signal to sources only ev.stop(); } } } else { throw new StoppingException("This job is not stoppable."); } }
if (transitionState(JobStatus.CREATED, JobStatus.RUNNING)) { newSchedulingFuture = scheduleLazy(slotProvider); break; newSchedulingFuture = scheduleEager(slotProvider, allocationTimeout); break; throw new JobException("Schedule mode is invalid."); if (throwable != null && !(throwable instanceof CancellationException)) { failGlobal(ExceptionUtils.stripCompletionException(throwable));
if (jobVertex.isInputVertex() && !jobVertex.isStoppable()) { this.isStoppable = false; ExecutionJobVertex ejv = new ExecutionJobVertex( this, jobVertex, createTimestamp); ejv.connectToPredecessors(this.intermediateResults); ExecutionJobVertex previousTask = this.tasks.putIfAbsent(jobVertex.getID(), ejv); if (previousTask != null) { throw new JobException(String.format("Encountered two job vertices with ID %s : previous=[%s] / new=[%s]", jobVertex.getID(), ejv, previousTask)); for (IntermediateResult res : ejv.getProducedDataSets()) { IntermediateResult previousDataSet = this.intermediateResults.putIfAbsent(res.getId(), res); if (previousDataSet != null) { throw new JobException(String.format("Encountered two intermediate data set with ID %s : previous=[%s] / new=[%s]", res.getId(), res, previousDataSet)); this.numVerticesTotal += ejv.getParallelism(); newExecJobVertices.add(ejv); failoverStrategy.notifyNewVertices(newExecJobVertices);
public void connectToPredecessors(Map<IntermediateDataSetID, IntermediateResult> intermediateDataSets) throws JobException { List<JobEdge> inputs = jobVertex.getInputs(); LOG.debug(String.format("Connecting ExecutionJobVertex %s (%s) to %d predecessors.", jobVertex.getID(), jobVertex.getName(), inputs.size())); if (edge.getSource() == null) { LOG.debug(String.format("Connecting input %d of vertex %s (%s) to intermediate result referenced via ID %s.", num, jobVertex.getID(), jobVertex.getName(), edge.getSourceId())); } else { LOG.debug(String.format("Connecting input %d of vertex %s (%s) to intermediate result referenced via predecessor %s (%s).", num, jobVertex.getID(), jobVertex.getName(), edge.getSource().getProducer().getID(), edge.getSource().getProducer().getName())); IntermediateResult ires = intermediateDataSets.get(edge.getSourceId()); if (ires == null) { throw new JobException("Cannot connect this job graph to the previous graph. No previous intermediate result found for ID " + edge.getSourceId()); int consumerIndex = ires.registerConsumer(); ev.connectSource(num, ires, edge, consumerIndex);
public void stop() throws StoppingException { if (isStoppable) { for (ExecutionVertex ev : this.getAllExecutionVertices()) { if (ev.getNumberOfInputs() == 0) { // send signal to sources only ev.stop(); } } } else { throw new StoppingException("This job is not stoppable."); } }
if (transitionState(JobStatus.CREATED, JobStatus.RUNNING)) { newSchedulingFuture = scheduleLazy(slotProvider); break; newSchedulingFuture = scheduleEager(slotProvider, allocationTimeout); break; throw new JobException("Schedule mode is invalid."); if (throwable != null && !(throwable instanceof CancellationException)) { failGlobal(ExceptionUtils.stripCompletionException(throwable));
if (jobVertex.isInputVertex() && !jobVertex.isStoppable()) { this.isStoppable = false; ExecutionJobVertex ejv = new ExecutionJobVertex( this, jobVertex, createTimestamp); ejv.connectToPredecessors(this.intermediateResults); ExecutionJobVertex previousTask = this.tasks.putIfAbsent(jobVertex.getID(), ejv); if (previousTask != null) { throw new JobException(String.format("Encountered two job vertices with ID %s : previous=[%s] / new=[%s]", jobVertex.getID(), ejv, previousTask)); for (IntermediateResult res : ejv.getProducedDataSets()) { IntermediateResult previousDataSet = this.intermediateResults.putIfAbsent(res.getId(), res); if (previousDataSet != null) { throw new JobException(String.format("Encountered two intermediate data set with ID %s : previous=[%s] / new=[%s]", res.getId(), res, previousDataSet)); this.numVerticesTotal += ejv.getParallelism(); newExecJobVertices.add(ejv); failoverStrategy.notifyNewVertices(newExecJobVertices);
public void stop() throws StoppingException { if (isStoppable) { for (ExecutionVertex ev : this.getAllExecutionVertices()) { if (ev.getNumberOfInputs() == 0) { // send signal to sources only ev.stop(); } } } else { throw new StoppingException("This job is not stoppable."); } }