@Test public void testRemoteExecutionWithSavepoint() throws Exception { SavepointRestoreSettings restoreSettings = SavepointRestoreSettings.forPath("fakePath"); RemoteStreamEnvironment env = new RemoteStreamEnvironment("fakeHost", 1, null, new String[]{}, null, restoreSettings); env.fromElements(1).map(x -> x * 2); RestClusterClient mockedClient = Mockito.mock(RestClusterClient.class); JobExecutionResult expectedResult = new JobExecutionResult(null, 0, null); PowerMockito.whenNew(RestClusterClient.class).withAnyArguments().thenReturn(mockedClient); when(mockedClient.run(Mockito.any(), Mockito.any(), Mockito.any(), Mockito.any(), Mockito.eq(restoreSettings))) .thenReturn(expectedResult); JobExecutionResult actualResult = env.execute("fakeJobName"); Assert.assertEquals(expectedResult, actualResult); } }
public static SavepointRestoreSettings createSavepointRestoreSettings(CommandLine commandLine) { if (commandLine.hasOption(SAVEPOINT_PATH_OPTION.getOpt())) { String savepointPath = commandLine.getOptionValue(SAVEPOINT_PATH_OPTION.getOpt()); boolean allowNonRestoredState = commandLine.hasOption(SAVEPOINT_ALLOW_NON_RESTORED_OPTION.getOpt()); return SavepointRestoreSettings.forPath(savepointPath, allowNonRestoredState); } else { return SavepointRestoreSettings.none(); } }
@Override JobGraph validateGraph() { JobGraph jobGraph = super.validateGraph(); final SavepointRestoreSettings savepointRestoreSettings = jobGraph.getSavepointRestoreSettings(); Assert.assertTrue(savepointRestoreSettings.allowNonRestoredState()); Assert.assertEquals(RESTORE_PATH, savepointRestoreSettings.getRestorePath()); return jobGraph; } }
jobGraph.setSavepointRestoreSettings(SavepointRestoreSettings.forPath("unknown path")); assertEquals("unknown path", jobGraph.getSavepointRestoreSettings().getRestorePath());
String savepointPath = line.getOptionValue(SAVEPOINT_PATH_OPTION.getOpt()); boolean allowNonRestoredState = line.hasOption(SAVEPOINT_ALLOW_NON_RESTORED_OPTION.getOpt()); this.savepointSettings = SavepointRestoreSettings.forPath(savepointPath, allowNonRestoredState); } else if (line.hasOption(RESUME_PATH_OPTION.getOpt())){ String checkpointPath = line.getOptionValue(RESUME_PATH_OPTION.getOpt()); boolean allowNonRestoredState = line.hasOption(SAVEPOINT_ALLOW_NON_RESTORED_OPTION.getOpt()); this.savepointSettings = SavepointRestoreSettings.forResumePath(checkpointPath, allowNonRestoredState); } else { this.savepointSettings = SavepointRestoreSettings.none();
RunOptions options = CliFrontendParser.parseRunCommand(parameters); SavepointRestoreSettings savepointSettings = options.getSavepointRestoreSettings(); assertTrue(savepointSettings.restoreSavepoint()); assertEquals("expectedSavepointPath", savepointSettings.getRestorePath()); assertFalse(savepointSettings.allowNonRestoredState()); RunOptions options = CliFrontendParser.parseRunCommand(parameters); SavepointRestoreSettings savepointSettings = options.getSavepointRestoreSettings(); assertTrue(savepointSettings.restoreSavepoint()); assertEquals("expectedSavepointPath", savepointSettings.getRestorePath()); assertTrue(savepointSettings.allowNonRestoredState());
/** * Tries to restore the given {@link ExecutionGraph} from the provided {@link SavepointRestoreSettings}. * * @param executionGraphToRestore {@link ExecutionGraph} which is supposed to be restored * @param savepointRestoreSettings {@link SavepointRestoreSettings} containing information about the savepoint to restore from * @throws Exception if the {@link ExecutionGraph} could not be restored */ private void tryRestoreExecutionGraphFromSavepoint(ExecutionGraph executionGraphToRestore, SavepointRestoreSettings savepointRestoreSettings) throws Exception { if (savepointRestoreSettings.restoreSavepoint()) { final CheckpointCoordinator checkpointCoordinator = executionGraphToRestore.getCheckpointCoordinator(); if (checkpointCoordinator != null) { checkpointCoordinator.restoreSavepoint( savepointRestoreSettings.getRestorePath(), savepointRestoreSettings.allowNonRestoredState(), savepointRestoreSettings.resumeFromLatestCheckpoint(), executionGraphToRestore.getAllVertices(), userCodeLoader); } } }
public JobSubmissionResult run( FlinkPlan compiledPlan, List<URL> libraries, List<URL> classpaths, ClassLoader classLoader) throws ProgramInvocationException { return run(compiledPlan, libraries, classpaths, classLoader, SavepointRestoreSettings.none()); }
@Override public String toString() { if (restoreSavepoint()) { return "SavepointRestoreSettings.forPath(" + "restorePath='" + restorePath + '\'' + ", allowNonRestoredState=" + allowNonRestoredState + ')'; } else { return "SavepointRestoreSettings.none()"; } }
public static SavepointRestoreSettings forPath(String savepointPath, boolean allowNonRestoredState) { checkNotNull(savepointPath, "Savepoint restore path."); return new SavepointRestoreSettings(savepointPath, allowNonRestoredState, false); }
if (savepointPath != null) { try { tryRestoreExecutionGraphFromSavepoint(newExecutionGraph, SavepointRestoreSettings.forPath(savepointPath, false)); } catch (Exception e) { final String message = String.format("Could not restore from temporary rescaling savepoint. This might indicate " + } catch (Exception e) { final String message = String.format("Could not restore from initial savepoint. This might indicate " + "that the savepoint %s got corrupted.", jobGraph.getSavepointRestoreSettings().getRestorePath());
/** * Tries to restore the given {@link ExecutionGraph} from the provided {@link SavepointRestoreSettings}. * * @param executionGraphToRestore {@link ExecutionGraph} which is supposed to be restored * @param savepointRestoreSettings {@link SavepointRestoreSettings} containing information about the savepoint to restore from * @throws Exception if the {@link ExecutionGraph} could not be restored */ private void tryRestoreExecutionGraphFromSavepoint(ExecutionGraph executionGraphToRestore, SavepointRestoreSettings savepointRestoreSettings) throws Exception { if (savepointRestoreSettings.restoreSavepoint()) { final CheckpointCoordinator checkpointCoordinator = executionGraphToRestore.getCheckpointCoordinator(); if (checkpointCoordinator != null) { checkpointCoordinator.restoreSavepoint( savepointRestoreSettings.getRestorePath(), savepointRestoreSettings.allowNonRestoredState(), executionGraphToRestore.getAllVertices(), userCodeLoader); } } }
public JobSubmissionResult run(JobWithJars program, int parallelism) throws ProgramInvocationException { return run(program, parallelism, SavepointRestoreSettings.none()); }
@Override public String toString() { if (restoreSavepoint()) { return "SavepointRestoreSettings.forPath(" + "restorePath='" + restorePath + '\'' + ", allowNonRestoredState=" + allowNonRestoredState + ')'; } else { return "SavepointRestoreSettings.none()"; } }
public static SavepointRestoreSettings forResumePath(String checkpointPath, boolean allowNonRestoredState) { checkNotNull(checkpointPath, "Checkpoint resume path."); return new SavepointRestoreSettings(checkpointPath, allowNonRestoredState, true); }
jobGraph.setSavepointRestoreSettings(SavepointRestoreSettings.forPath(savepointPath));
private SavepointRestoreSettings getSavepointRestoreSettings( final @Nonnull HandlerRequest<JarRunRequestBody, JarRunMessageParameters> request) throws RestHandlerException { final JarRunRequestBody requestBody = request.getRequestBody(); final boolean allowNonRestoredState = fromRequestBodyOrQueryParameter( requestBody.getAllowNonRestoredState(), () -> getQueryParameter(request, AllowNonRestoredStateQueryParameter.class), false, log); final String savepointPath = fromRequestBodyOrQueryParameter( emptyToNull(requestBody.getSavepointPath()), () -> emptyToNull(getQueryParameter(request, SavepointPathQueryParameter.class)), null, log); final SavepointRestoreSettings savepointRestoreSettings; if (savepointPath != null) { savepointRestoreSettings = SavepointRestoreSettings.forPath( savepointPath, allowNonRestoredState); } else { savepointRestoreSettings = SavepointRestoreSettings.none(); } return savepointRestoreSettings; }
if (savepointPath != null) { try { tryRestoreExecutionGraphFromSavepoint(newExecutionGraph, SavepointRestoreSettings.forPath(savepointPath, false)); } catch (Exception e) { final String message = String.format("Could not restore from temporary rescaling savepoint. This might indicate " + } catch (Exception e) { final String message = String.format("Could not restore from initial savepoint. This might indicate " + "that the savepoint %s got corrupted.", jobGraph.getSavepointRestoreSettings().getRestorePath());
/** * Tries to restore the given {@link ExecutionGraph} from the provided {@link SavepointRestoreSettings}. * * @param executionGraphToRestore {@link ExecutionGraph} which is supposed to be restored * @param savepointRestoreSettings {@link SavepointRestoreSettings} containing information about the savepoint to restore from * @throws Exception if the {@link ExecutionGraph} could not be restored */ private void tryRestoreExecutionGraphFromSavepoint(ExecutionGraph executionGraphToRestore, SavepointRestoreSettings savepointRestoreSettings) throws Exception { if (savepointRestoreSettings.restoreSavepoint()) { final CheckpointCoordinator checkpointCoordinator = executionGraphToRestore.getCheckpointCoordinator(); if (checkpointCoordinator != null) { checkpointCoordinator.restoreSavepoint( savepointRestoreSettings.getRestorePath(), savepointRestoreSettings.allowNonRestoredState(), executionGraphToRestore.getAllVertices(), userCodeLoader); } } }
savepointRestoreSettings = SavepointRestoreSettings.none();