@Override public JobManagerMessages.CancellationSuccess process(JobManagerMessages.CancelJob message) { Assert.assertEquals(expectedJobID, message.jobID()); return new JobManagerMessages.CancellationSuccess(message.jobID(), null); } }
@Override public JobManagerMessages.CancellationSuccess process(JobManagerMessages.CancelJobWithSavepoint message) { Assert.assertEquals(expectedJobID, message.jobID()); Assert.assertEquals(expectedTargetDirectory, message.savepointDirectory()); return new JobManagerMessages.CancellationSuccess(message.jobID(), savepointPathToReturn); } }
/** * Cancels a job identified by the job id and triggers a savepoint. * @param jobId the job id * @param savepointDirectory directory the savepoint should be written to * @return path where the savepoint is located * @throws Exception In case an error occurred. */ public String cancelWithSavepoint(JobID jobId, @Nullable String savepointDirectory) throws Exception { final ActorGateway jobManager = getJobManagerGateway(); Object cancelMsg = new JobManagerMessages.CancelJobWithSavepoint(jobId, savepointDirectory); Future<Object> response = jobManager.ask(cancelMsg, timeout); final Object rc = Await.result(response, timeout); if (rc instanceof JobManagerMessages.CancellationSuccess) { JobManagerMessages.CancellationSuccess success = (JobManagerMessages.CancellationSuccess) rc; return success.savepointPath(); } else if (rc instanceof JobManagerMessages.CancellationFailure) { throw new Exception("Cancel & savepoint for the job with ID " + jobId + " failed.", ((JobManagerMessages.CancellationFailure) rc).cause()); } else { throw new IllegalStateException("Unexpected response: " + rc); } }
@Override public CompletableFuture<Acknowledge> rescaleJob(JobID jobId, int newParallelism) { rescaleJobFuture.complete(Tuple2.of(jobId, newParallelism)); return CompletableFuture.completedFuture(Acknowledge.get()); } }
/** * Cancels a job identified by the job id. * @param jobId the job id * @throws Exception In case an error occurred. */ public void cancel(JobID jobId) throws Exception { final ActorGateway jobManager = getJobManagerGateway(); Object cancelMsg = new JobManagerMessages.CancelJob(jobId); Future<Object> response = jobManager.ask(cancelMsg, timeout); final Object rc = Await.result(response, timeout); if (rc instanceof JobManagerMessages.CancellationSuccess) { // no further action required } else if (rc instanceof JobManagerMessages.CancellationFailure) { throw new Exception("Canceling the job with ID " + jobId + " failed.", ((JobManagerMessages.CancellationFailure) rc).cause()); } else { throw new IllegalStateException("Unexpected response: " + rc); } }
public CompletableFuture<Acknowledge> disposeSavepoint(String savepointPath) throws FlinkException { final ActorGateway jobManager = getJobManagerGateway(); Object msg = new JobManagerMessages.DisposeSavepoint(savepointPath); CompletableFuture<Object> responseFuture = FutureUtils.toJava( jobManager.ask( msg, timeout)); return responseFuture.thenApply( (Object response) -> { if (response instanceof JobManagerMessages.DisposeSavepointSuccess$) { return Acknowledge.get(); } else if (response instanceof JobManagerMessages.DisposeSavepointFailure) { JobManagerMessages.DisposeSavepointFailure failureResponse = (JobManagerMessages.DisposeSavepointFailure) response; if (failureResponse.cause() instanceof ClassNotFoundException) { throw new CompletionException( new ClassNotFoundException("Savepoint disposal failed, because of a " + "missing class. This is most likely caused by a custom state " + "instance, which cannot be disposed without the user code class " + "loader. Please provide the program jar with which you have created " + "the savepoint via -j <JAR> for disposal.", failureResponse.cause().getCause())); } else { throw new CompletionException(failureResponse.cause()); } } else { throw new CompletionException(new FlinkRuntimeException("Unknown response type " + response.getClass().getSimpleName() + '.')); } }); }
/** * Stops a program on Flink cluster whose job-manager is configured in this client's configuration. * Stopping works only for streaming programs. Be aware, that the program might continue to run for * a while after sending the stop command, because after sources stopped to emit data all operators * need to finish processing. * * @param jobId * the job ID of the streaming program to stop * @throws Exception * If the job ID is invalid (ie, is unknown or refers to a batch job) or if sending the stop signal * failed. That might be due to an I/O problem, ie, the job-manager is unreachable. */ public void stop(final JobID jobId) throws Exception { final ActorGateway jobManager = getJobManagerGateway(); Future<Object> response = jobManager.ask(new JobManagerMessages.StopJob(jobId), timeout); final Object rc = Await.result(response, timeout); if (rc instanceof JobManagerMessages.StoppingSuccess) { // no further action required } else if (rc instanceof JobManagerMessages.StoppingFailure) { throw new Exception("Stopping the job with ID " + jobId + " failed.", ((JobManagerMessages.StoppingFailure) rc).cause()); } else { throw new IllegalStateException("Unexpected response: " + rc); } }
/** * Triggers a savepoint for the job identified by the job id. The savepoint will be written to the given savepoint * directory, or {@link org.apache.flink.configuration.CheckpointingOptions#SAVEPOINT_DIRECTORY} if it is null. * * @param jobId job id * @param savepointDirectory directory the savepoint should be written to * @return path future where the savepoint is located * @throws FlinkException if no connection to the cluster could be established */ public CompletableFuture<String> triggerSavepoint(JobID jobId, @Nullable String savepointDirectory) throws FlinkException { final ActorGateway jobManager = getJobManagerGateway(); Future<Object> response = jobManager.ask(new JobManagerMessages.TriggerSavepoint(jobId, Option.<String>apply(savepointDirectory)), new FiniteDuration(1, TimeUnit.HOURS)); CompletableFuture<Object> responseFuture = FutureUtils.toJava(response); return responseFuture.thenApply((responseMessage) -> { if (responseMessage instanceof JobManagerMessages.TriggerSavepointSuccess) { JobManagerMessages.TriggerSavepointSuccess success = (JobManagerMessages.TriggerSavepointSuccess) responseMessage; return success.savepointPath(); } else if (responseMessage instanceof JobManagerMessages.TriggerSavepointFailure) { JobManagerMessages.TriggerSavepointFailure failure = (JobManagerMessages.TriggerSavepointFailure) responseMessage; throw new CompletionException(failure.cause()); } else { throw new CompletionException( new IllegalStateException("Unknown JobManager response of type " + responseMessage.getClass())); } }); }
@Override public void handleMessage(Object message) { getSender().tell( decorateMessage(new JobManagerMessages.JobResultFailure( new SerializedThrowable(new Exception("test")))), getSelf()); }
/** * Tells the JobManager to finish the sessions (jobs) defined by the given IDs. * * @param jobIds The IDs that identify the sessions. */ public void endSessions(List<JobID> jobIds) throws Exception { if (jobIds == null) { throw new IllegalArgumentException("The JobIDs must not be null"); } ActorGateway jobManagerGateway = getJobManagerGateway(); for (JobID jid : jobIds) { if (jid != null) { log.info("Telling job manager to end the session {}.", jid); jobManagerGateway.tell(new JobManagerMessages.RemoveCachedJob(jid)); } } }
/** * Requests the {@link JobStatus} of the job with the given {@link JobID}. */ public CompletableFuture<JobStatus> getJobStatus(JobID jobId) { final ActorGateway jobManager; try { jobManager = getJobManagerGateway(); } catch (FlinkException e) { throw new RuntimeException("Could not retrieve JobManage gateway.", e); } Future<Object> response = jobManager.ask(JobManagerMessages.getRequestJobStatus(jobId), timeout); CompletableFuture<Object> javaFuture = FutureUtils.toJava(response); return javaFuture.thenApply((responseMessage) -> { if (responseMessage instanceof JobManagerMessages.CurrentJobStatus) { return ((JobManagerMessages.CurrentJobStatus) responseMessage).status(); } else if (responseMessage instanceof JobManagerMessages.JobNotFound) { throw new CompletionException( new IllegalStateException("Could not find job with JobId " + jobId)); } else { throw new CompletionException( new IllegalStateException("Unknown JobManager response of type " + responseMessage.getClass())); } }); }
@Override public void handleMessage(Object message) { if (message instanceof JobManagerMessages.SubmitJob) { JobID jid = ((JobManagerMessages.SubmitJob) message).jobGraph().getJobID(); getSender().tell( decorateMessage(new JobManagerMessages.JobSubmitSuccess(jid)), getSelf()); } else if (message.getClass() == JobManagerMessages.getRequestLeaderSessionID().getClass()) { getSender().tell( decorateMessage(new JobManagerMessages.ResponseLeaderSessionID(leaderSessionID)), getSelf()); } else if (message instanceof JobManagerMessages.RequestBlobManagerPort$) { getSender().tell(1337, getSelf()); } else { getSender().tell( decorateMessage(new Status.Failure(new Exception("Unknown message " + message))), getSelf()); } }
@Override public JobManagerMessages.TriggerSavepointSuccess process(JobManagerMessages.TriggerSavepoint message) { Assert.assertEquals(expectedJobID, message.jobId()); if (expectedTargetDirectory == null) { Assert.assertTrue(message.savepointDirectory().isEmpty()); } else { Assert.assertEquals(expectedTargetDirectory, message.savepointDirectory().get()); } return new JobManagerMessages.TriggerSavepointSuccess(message.jobId(), 0, savepointPathToReturn, 0); } }
@Override public Future<Object> ask(Object message, FiniteDuration timeout) { messageArrived = true; if (message instanceof JobManagerMessages.StopJob) { JobManagerMessages.StopJob stopJob = (JobManagerMessages.StopJob) message; Assert.assertEquals(expectedJobID, stopJob.jobID()); return Future$.MODULE$.successful(new JobManagerMessages.StoppingSuccess(stopJob.jobID())); } fail("Expected StopJob message, got: " + message.getClass()); return null; } }
@Override public JobManagerMessages.DisposeSavepointFailure process(JobManagerMessages.DisposeSavepoint message) { return new JobManagerMessages.DisposeSavepointFailure(new ClassNotFoundException("Test Exception")); } }
static void init() throws Exception { jarDir = TMP.newFolder().toPath(); // properties are set property by surefire plugin final String parameterProgramJarName = System.getProperty("parameterJarName") + ".jar"; final String parameterProgramWithoutManifestJarName = System.getProperty("parameterJarWithoutManifestName") + ".jar"; final Path jarLocation = Paths.get(System.getProperty("targetDir")); jarWithManifest = Files.copy( jarLocation.resolve(parameterProgramJarName), jarDir.resolve("program-with-manifest.jar")); jarWithoutManifest = Files.copy( jarLocation.resolve(parameterProgramWithoutManifestJarName), jarDir.resolve("program-without-manifest.jar")); restfulGateway = new TestingDispatcherGateway.Builder() .setBlobServerPort(BLOB_SERVER_RESOURCE.getBlobServerPort()) .setSubmitFunction(jobGraph -> { LAST_SUBMITTED_JOB_GRAPH_REFERENCE.set(jobGraph); return CompletableFuture.completedFuture(Acknowledge.get()); }) .build(); gatewayRetriever = () -> CompletableFuture.completedFuture(restfulGateway); localAddressFuture = CompletableFuture.completedFuture("shazam://localhost:12345"); timeout = Time.seconds(10); responseHeaders = Collections.emptyMap(); executor = TestingUtils.defaultExecutor(); }
@Override public CompletableFuture<Acknowledge> disposeSavepoint(String savepointPath) { final SavepointDisposalRequest savepointDisposalRequest = new SavepointDisposalRequest(savepointPath); final CompletableFuture<TriggerResponse> savepointDisposalTriggerFuture = sendRequest( SavepointDisposalTriggerHeaders.getInstance(), savepointDisposalRequest); final CompletableFuture<AsynchronousOperationInfo> savepointDisposalFuture = savepointDisposalTriggerFuture.thenCompose( (TriggerResponse triggerResponse) -> { final TriggerId triggerId = triggerResponse.getTriggerId(); final SavepointDisposalStatusHeaders savepointDisposalStatusHeaders = SavepointDisposalStatusHeaders.getInstance(); final SavepointDisposalStatusMessageParameters savepointDisposalStatusMessageParameters = savepointDisposalStatusHeaders.getUnresolvedMessageParameters(); savepointDisposalStatusMessageParameters.triggerIdPathParameter.resolve(triggerId); return pollResourceAsync( () -> sendRequest( savepointDisposalStatusHeaders, savepointDisposalStatusMessageParameters)); }); return savepointDisposalFuture.thenApply( (AsynchronousOperationInfo asynchronousOperationInfo) -> { if (asynchronousOperationInfo.getFailureCause() == null) { return Acknowledge.get(); } else { throw new CompletionException(asynchronousOperationInfo.getFailureCause()); } }); }
/** * Tests disposal with a JAR file. */ @Test public void testDisposeWithJar() throws Exception { replaceStdOutAndStdErr(); final CompletableFuture<String> disposeSavepointFuture = new CompletableFuture<>(); final DisposeSavepointClusterClient clusterClient = new DisposeSavepointClusterClient( (String savepointPath) -> { disposeSavepointFuture.complete(savepointPath); return CompletableFuture.completedFuture(Acknowledge.get()); }, getConfiguration()); try { CliFrontend frontend = new MockedCliFrontend(clusterClient); // Fake JAR file File f = tmp.newFile(); ZipOutputStream out = new ZipOutputStream(new FileOutputStream(f)); out.close(); final String disposePath = "any-path"; String[] parameters = { "-d", disposePath, "-j", f.getAbsolutePath() }; frontend.savepoint(parameters); final String actualSavepointPath = disposeSavepointFuture.get(); assertEquals(disposePath, actualSavepointPath); } finally { clusterClient.shutdown(); restoreStdOutAndStdErr(); } }
@Override public CompletableFuture<Acknowledge> rescaleJob(JobID jobId, int newParallelism) { final RescalingTriggerHeaders rescalingTriggerHeaders = RescalingTriggerHeaders.getInstance(); final RescalingTriggerMessageParameters rescalingTriggerMessageParameters = rescalingTriggerHeaders.getUnresolvedMessageParameters(); rescalingTriggerMessageParameters.jobPathParameter.resolve(jobId); rescalingTriggerMessageParameters.rescalingParallelismQueryParameter.resolve(Collections.singletonList(newParallelism)); final CompletableFuture<TriggerResponse> rescalingTriggerResponseFuture = sendRequest( rescalingTriggerHeaders, rescalingTriggerMessageParameters); final CompletableFuture<AsynchronousOperationInfo> rescalingOperationFuture = rescalingTriggerResponseFuture.thenCompose( (TriggerResponse triggerResponse) -> { final TriggerId triggerId = triggerResponse.getTriggerId(); final RescalingStatusHeaders rescalingStatusHeaders = RescalingStatusHeaders.getInstance(); final RescalingStatusMessageParameters rescalingStatusMessageParameters = rescalingStatusHeaders.getUnresolvedMessageParameters(); rescalingStatusMessageParameters.jobPathParameter.resolve(jobId); rescalingStatusMessageParameters.triggerIdPathParameter.resolve(triggerId); return pollResourceAsync( () -> sendRequest( rescalingStatusHeaders, rescalingStatusMessageParameters)); }); return rescalingOperationFuture.thenApply( (AsynchronousOperationInfo asynchronousOperationInfo) -> { if (asynchronousOperationInfo.getFailureCause() == null) { return Acknowledge.get(); } else { throw new CompletionException(asynchronousOperationInfo.getFailureCause()); } }); }
@Test public void testDisposeSavepointSuccess() throws Exception { replaceStdOutAndStdErr(); String savepointPath = "expectedSavepointPath"; ClusterClient clusterClient = new DisposeSavepointClusterClient( (String path) -> CompletableFuture.completedFuture(Acknowledge.get()), getConfiguration()); try { CliFrontend frontend = new MockedCliFrontend(clusterClient); String[] parameters = { "-d", savepointPath }; frontend.savepoint(parameters); String outMsg = buffer.toString(); assertTrue(outMsg.contains(savepointPath)); assertTrue(outMsg.contains("disposed")); } finally { clusterClient.shutdown(); restoreStdOutAndStdErr(); } }