/** * Restores the latest checkpointed state. * * <p>The recovery of checkpoints might block. Make sure that calls to this method don't * block the job manager actor and run asynchronously. * * @param errorIfNoCheckpoint Fail if there is no checkpoint available * @param allowNonRestoredState Allow to skip checkpoint state that cannot be mapped * to the ExecutionGraph vertices (if the checkpoint contains state for a * job vertex that is not part of this ExecutionGraph). */ public void restoreLatestCheckpointedState(boolean errorIfNoCheckpoint, boolean allowNonRestoredState) throws Exception { synchronized (progressLock) { if (checkpointCoordinator != null) { checkpointCoordinator.restoreLatestCheckpointedState(getAllVertices(), errorIfNoCheckpoint, allowNonRestoredState); } } }
/** * Restores the latest checkpointed state. * * <p>The recovery of checkpoints might block. Make sure that calls to this method don't * block the job manager actor and run asynchronously. * * @param errorIfNoCheckpoint Fail if there is no checkpoint available * @param allowNonRestoredState Allow to skip checkpoint state that cannot be mapped * to the the ExecutionGraph vertices (if the checkpoint contains state for a * job vertex that is not part of this ExecutionGraph). */ public void restoreLatestCheckpointedState(boolean errorIfNoCheckpoint, boolean allowNonRestoredState) throws Exception { synchronized (progressLock) { if (checkpointCoordinator != null) { checkpointCoordinator.restoreLatestCheckpointedState(getAllVertices(), errorIfNoCheckpoint, allowNonRestoredState); } } }
/** * Restores the latest checkpointed state. * * <p>The recovery of checkpoints might block. Make sure that calls to this method don't * block the job manager actor and run asynchronously. * * @param errorIfNoCheckpoint Fail if there is no checkpoint available * @param allowNonRestoredState Allow to skip checkpoint state that cannot be mapped * to the ExecutionGraph vertices (if the checkpoint contains state for a * job vertex that is not part of this ExecutionGraph). */ public void restoreLatestCheckpointedState(boolean errorIfNoCheckpoint, boolean allowNonRestoredState) throws Exception { synchronized (progressLock) { if (checkpointCoordinator != null) { checkpointCoordinator.restoreLatestCheckpointedState(getAllVertices(), errorIfNoCheckpoint, allowNonRestoredState); } } }
/** * Restores the latest checkpointed state. * * <p>The recovery of checkpoints might block. Make sure that calls to this method don't * block the job manager actor and run asynchronously. * * @param errorIfNoCheckpoint Fail if there is no checkpoint available * @param allowNonRestoredState Allow to skip checkpoint state that cannot be mapped * to the ExecutionGraph vertices (if the checkpoint contains state for a * job vertex that is not part of this ExecutionGraph). */ public void restoreLatestCheckpointedState(boolean errorIfNoCheckpoint, boolean allowNonRestoredState) throws Exception { synchronized (progressLock) { if (checkpointCoordinator != null) { checkpointCoordinator.restoreLatestCheckpointedState(getAllVertices(), errorIfNoCheckpoint, allowNonRestoredState); } } }
public void scheduleForExecution() throws JobException { for (ExecutionJobVertex ejv : getAllVertices().values()) { ejv.setUpInputSplits(null); } final boolean enableSharedSlot = jobManagerConfiguration.getBoolean(JobManagerOptions.SLOT_ENABLE_SHARED_SLOT); if (enableSharedSlot) { updateSharedSlotResources(); } if (transitionState(JobStatus.CREATED, JobStatus.RUNNING)) { graphManager.startScheduling(); } else { throw new IllegalStateException("Job may only be scheduled from state " + JobStatus.CREATED); } }
/** * Optimize the new {@link ExecutionGraph}. Enrich it with execution prior locations to improve the scheduling. * * @param newExecutionGraph is the new {@link ExecutionGraph} to optimize by enriching * @param oldExecutionGraph is the old {@link ExecutionGraph} to help with the optimization */ private void optimizeNewExecutionGraph(ExecutionGraph newExecutionGraph, ExecutionGraph oldExecutionGraph) { Map<JobVertexID, ExecutionJobVertex> oldVertices = oldExecutionGraph.getAllVertices(); Map<JobVertexID, ExecutionJobVertex> newVertices = newExecutionGraph.getAllVertices(); for (JobVertexID jobVertexID : newVertices.keySet()) { ExecutionJobVertex oldEjv = oldVertices.get(jobVertexID); if (oldEjv != null) { ExecutionJobVertex newEjv = newVertices.get(jobVertexID); int minParallelism = oldEjv.getParallelism() < newEjv.getParallelism() ? oldEjv.getParallelism() : newEjv.getParallelism(); for (int i = 0; i < minParallelism; i++) { newEjv.getTaskVertices()[i].setLatestPriorLocation( oldEjv.getTaskVertices()[i].getCurrentAssignedResourceLocation()); } } } }
checkpointCoordinator.restoreLatestCheckpointedState(getAllVertices(), false, false);
checkpointCoordinator.restoreLatestCheckpointedState(getAllVertices(), false, false);
checkpointCoordinator.restoreLatestCheckpointedState(getAllVertices(), false, false);
checkpointCoordinator.restoreLatestCheckpointedState(getAllVertices(), false, false);
private ExecutionGraph createAndRestoreExecutionGraph(JobManagerJobMetricGroup currentJobManagerJobMetricGroup) throws Exception { ExecutionGraph newExecutionGraph = createExecutionGraph(currentJobManagerJobMetricGroup); final CheckpointCoordinator checkpointCoordinator = newExecutionGraph.getCheckpointCoordinator(); if (checkpointCoordinator != null) { // check whether we find a valid checkpoint if (!checkpointCoordinator.restoreLatestCheckpointedState( newExecutionGraph.getAllVertices(), false, false)) { // check whether we can restore from a savepoint tryRestoreExecutionGraphFromSavepoint(newExecutionGraph, jobGraph.getSavepointRestoreSettings()); } } return newExecutionGraph; }
private ExecutionGraph createAndRestoreExecutionGraph(JobManagerJobMetricGroup currentJobManagerJobMetricGroup) throws Exception { ExecutionGraph newExecutionGraph = createExecutionGraph(currentJobManagerJobMetricGroup); final CheckpointCoordinator checkpointCoordinator = newExecutionGraph.getCheckpointCoordinator(); if (checkpointCoordinator != null) { // check whether we find a valid checkpoint if (!checkpointCoordinator.restoreLatestCheckpointedState( newExecutionGraph.getAllVertices(), false, false)) { // check whether we can restore from a savepoint tryRestoreExecutionGraphFromSavepoint(newExecutionGraph, jobGraph.getSavepointRestoreSettings()); } } return newExecutionGraph; }
private ExecutionGraph createAndRestoreExecutionGraph( JobGraph jobGraph, JobManagerJobMetricGroup currentJobManagerJobMetricGroup) throws Exception { ExecutionGraph newExecutionGraph = createExecutionGraph(jobGraph, currentJobManagerJobMetricGroup); final CheckpointCoordinator checkpointCoordinator = newExecutionGraph.getCheckpointCoordinator(); if (checkpointCoordinator != null) { // check whether we find a valid checkpoint if (!checkpointCoordinator.restoreLatestCheckpointedState( newExecutionGraph.getAllVertices(), false, false)) { // check whether we can restore from a savepoint tryRestoreExecutionGraphFromSavepoint(newExecutionGraph, jobGraph.getSavepointRestoreSettings()); } } return newExecutionGraph; }
/** * Tries to restore the given {@link ExecutionGraph} from the provided {@link SavepointRestoreSettings}. * * @param executionGraphToRestore {@link ExecutionGraph} which is supposed to be restored * @param savepointRestoreSettings {@link SavepointRestoreSettings} containing information about the savepoint to restore from * @throws Exception if the {@link ExecutionGraph} could not be restored */ private void tryRestoreExecutionGraphFromSavepoint(ExecutionGraph executionGraphToRestore, SavepointRestoreSettings savepointRestoreSettings) throws Exception { if (savepointRestoreSettings.restoreSavepoint()) { final CheckpointCoordinator checkpointCoordinator = executionGraphToRestore.getCheckpointCoordinator(); if (checkpointCoordinator != null) { checkpointCoordinator.restoreSavepoint( savepointRestoreSettings.getRestorePath(), savepointRestoreSettings.allowNonRestoredState(), executionGraphToRestore.getAllVertices(), userCodeLoader); } } }
/** * Tries to restore the given {@link ExecutionGraph} from the provided {@link SavepointRestoreSettings}. * * @param executionGraphToRestore {@link ExecutionGraph} which is supposed to be restored * @param savepointRestoreSettings {@link SavepointRestoreSettings} containing information about the savepoint to restore from * @throws Exception if the {@link ExecutionGraph} could not be restored */ private void tryRestoreExecutionGraphFromSavepoint(ExecutionGraph executionGraphToRestore, SavepointRestoreSettings savepointRestoreSettings) throws Exception { if (savepointRestoreSettings.restoreSavepoint()) { final CheckpointCoordinator checkpointCoordinator = executionGraphToRestore.getCheckpointCoordinator(); if (checkpointCoordinator != null) { checkpointCoordinator.restoreSavepoint( savepointRestoreSettings.getRestorePath(), savepointRestoreSettings.allowNonRestoredState(), executionGraphToRestore.getAllVertices(), userCodeLoader); } } }
/** * Tries to restore the given {@link ExecutionGraph} from the provided {@link SavepointRestoreSettings}. * * @param executionGraphToRestore {@link ExecutionGraph} which is supposed to be restored * @param savepointRestoreSettings {@link SavepointRestoreSettings} containing information about the savepoint to restore from * @throws Exception if the {@link ExecutionGraph} could not be restored */ private void tryRestoreExecutionGraphFromSavepoint(ExecutionGraph executionGraphToRestore, SavepointRestoreSettings savepointRestoreSettings) throws Exception { if (savepointRestoreSettings.restoreSavepoint()) { final CheckpointCoordinator checkpointCoordinator = executionGraphToRestore.getCheckpointCoordinator(); if (checkpointCoordinator != null) { checkpointCoordinator.restoreSavepoint( savepointRestoreSettings.getRestorePath(), savepointRestoreSettings.allowNonRestoredState(), savepointRestoreSettings.resumeFromLatestCheckpoint(), executionGraphToRestore.getAllVertices(), userCodeLoader); } } }
this.kvStateLocationRegistry = new KvStateLocationRegistry(jobId, getAllVertices());
this.kvStateLocationRegistry = new KvStateLocationRegistry(jobInformation.getJobId(), getAllVertices());
this.kvStateLocationRegistry = new KvStateLocationRegistry(jobInformation.getJobId(), getAllVertices());
InputSplitsOperationLog splitsOperationLog = (InputSplitsOperationLog) opLog; try { executionGraph.getAllVertices().get(splitsOperationLog.getJobVertexID()) .setUpInputSplits(splitsOperationLog.getInputSplitsMap()); } catch (Exception e) {