public void setGraphManager(GraphManager graphManager) { this.graphManager = checkNotNull(graphManager); this.allowLazyDeployment = graphManager.allowLazyDeployment(); }
private void setupGraphManager() { if (graphManager != null) { graphManager.dispose(); } graphManager = createGraphManager(); this.operationLogManager.registerLogHandler(OperationLogType.GRAPH_MANAGER, this.graphManager); executionGraph.setGraphManager(graphManager); executionGraph.registerExecutionListener(graphManager); }
@Override public CompletableFuture<Acknowledge> scheduleOrUpdateConsumers( final ResultPartitionID partitionID, final Time timeout) { if (graphManager.isReconciling()) { return FutureUtils.completedExceptionally( new JobMasterException("The job master is reconciling with task executors.")); } try { executionGraph.scheduleOrUpdateConsumers(partitionID); return CompletableFuture.completedFuture(Acknowledge.get()); } catch (Exception e) { return FutureUtils.completedExceptionally(e); } }
private GraphManager createGraphManager() { Configuration schedulingConfig = jobGraph.getSchedulingConfiguration(); GraphManagerPlugin graphManagerPlugin = GraphManagerPluginFactory.createGraphManagerPlugin( schedulingConfig, userCodeLoader); Configuration jobConfig = jobGraph.getJobConfiguration(); Configuration configuration = new Configuration(); if (jobConfig != null) { configuration.addAll(jobConfig); } if (schedulingConfig != null) { configuration.addAll(schedulingConfig); } GraphManager graphManager = new GraphManager( graphManagerPlugin, getSelfGateway(JobMasterGateway.class), operationLogManager, executionGraph); graphManager.open(jobGraph, new SchedulingConfig(configuration, userCodeLoader)); return graphManager; }
if (JobStatus.CREATED.equals(executionGraph.getState()) && !graphManager.isReconciling()) { scheduleExecutionGraph(); } else { executionGraph.getReconcileFuture().thenApplyAsync( (Collection<ExecutionAttemptID> reconcileFailedExecutions) -> { graphManager.leaveReconcile(); if (JobStatus.RUNNING.equals(executionGraph.getState())) { for (ExecutionAttemptID executionAttemptId : reconcileFailedExecutions) {
void sendPartitionInfos() { if (vertex.getExecutionGraph().getGraphManager().isReplaying()) { return; } synchronized (updatePartitionLock) { updatePartitionFuture = null; } // check if the ExecutionVertex has already been archived and thus cleared the // partial partition infos queue if (partialInputChannelDeploymentDescriptors != null && !partialInputChannelDeploymentDescriptors.isEmpty()) { PartialInputChannelDeploymentDescriptor partialInputChannelDeploymentDescriptor; List<PartitionInfo> partitionInfos = new ArrayList<>(partialInputChannelDeploymentDescriptors.size()); while ((partialInputChannelDeploymentDescriptor = partialInputChannelDeploymentDescriptors.poll()) != null) { partitionInfos.add( new PartitionInfo( partialInputChannelDeploymentDescriptor.getResultId(), partialInputChannelDeploymentDescriptor.createInputChannelDeploymentDescriptor( this, getVertex().getExecutionGraph().getResultPartitionLocationTrackerProxy()))); } sendUpdatePartitionInfoRpcCall(partitionInfos); } }
public void resetExecutionVerticesAndNotify(long modVersion, List<ExecutionVertex> executionVertices) throws Exception { final long resetTimestamp = System.currentTimeMillis(); List<ExecutionVertexID> evIds = new ArrayList<>(executionVertices.size()); for (ExecutionVertex ev : executionVertices) { ev.resetForNewExecution(resetTimestamp, modVersion); evIds.add(ev.getExecutionVertexID()); } // if we have checkpointed state, reload it into the executions // we restart scheduler to ensure EXACTLY_ONCE mechanism and // to trigger new checkpoint without having to wait for old checkpoint expired if (checkpointCoordinator != null) { checkpointCoordinator.stopCheckpointScheduler(); checkpointCoordinator.restoreLatestCheckpointedState(executionVertices, false, true); checkpointCoordinator.startCheckpointScheduler(); } graphManager.notifyExecutionVertexFailover(evIds); }
getGraph().getGraphManager().notifyInputSplitsCreated(getJobVertexId(), inputSplitsMap); } finally { currentThread.setContextClassLoader(oldContextClassLoader);
protected void notifyAndUpdateConsumers(IntermediateResultPartition partition) { getExecutionGraph().getGraphManager().notifyResultPartitionConsumable( getExecutionVertexID(), partition.getIntermediateResult().getId(), partition.getPartitionNumber(), getCurrentAssignedResourceLocation()); getExecutionGraph().getFutureExecutor().execute(() -> { currentExecution.updateConsumers(partition.getConsumers()); }); }
graphManager.enterReconcile();
final ExecutionState oldState = oldExecution.getState(); if (oldState.isTerminal() || getExecutionGraph().getGraphManager().isReplaying()) { priorExecutions.add(oldExecution); latestPriorLocation = oldExecution.getAssignedResourceLocation();
graphManager.dispose();
/** * Updates the task execution state for a given task. * * @param taskExecutionState New task execution state for a given task * @return Acknowledge the task execution state update */ @Override public CompletableFuture<Acknowledge> updateTaskExecutionState( final TaskExecutionState taskExecutionState) { checkNotNull(taskExecutionState, "taskExecutionState"); if (graphManager.isReconciling()) { return FutureUtils.completedExceptionally( new JobMasterException("The job master is reconciling with task executors.")); } if (executionGraph.updateState(taskExecutionState)) { return CompletableFuture.completedFuture(Acknowledge.get()); } else { return FutureUtils.completedExceptionally( new ExecutionGraphException("The execution attempt " + taskExecutionState.getID() + " was not found.")); } }