@Override public CompletableFuture<Acknowledge> rescaleJob(JobID jobId, int newParallelism) { rescaleJobFuture.complete(Tuple2.of(jobId, newParallelism)); return CompletableFuture.completedFuture(Acknowledge.get()); } }
public CompletableFuture<Acknowledge> disposeSavepoint(String savepointPath) throws FlinkException { final ActorGateway jobManager = getJobManagerGateway(); Object msg = new JobManagerMessages.DisposeSavepoint(savepointPath); CompletableFuture<Object> responseFuture = FutureUtils.toJava( jobManager.ask( msg, timeout)); return responseFuture.thenApply( (Object response) -> { if (response instanceof JobManagerMessages.DisposeSavepointSuccess$) { return Acknowledge.get(); } else if (response instanceof JobManagerMessages.DisposeSavepointFailure) { JobManagerMessages.DisposeSavepointFailure failureResponse = (JobManagerMessages.DisposeSavepointFailure) response; if (failureResponse.cause() instanceof ClassNotFoundException) { throw new CompletionException( new ClassNotFoundException("Savepoint disposal failed, because of a " + "missing class. This is most likely caused by a custom state " + "instance, which cannot be disposed without the user code class " + "loader. Please provide the program jar with which you have created " + "the savepoint via -j <JAR> for disposal.", failureResponse.cause().getCause())); } else { throw new CompletionException(failureResponse.cause()); } } else { throw new CompletionException(new FlinkRuntimeException("Unknown response type " + response.getClass().getSimpleName() + '.')); } }); }
static void init() throws Exception { jarDir = TMP.newFolder().toPath(); // properties are set property by surefire plugin final String parameterProgramJarName = System.getProperty("parameterJarName") + ".jar"; final String parameterProgramWithoutManifestJarName = System.getProperty("parameterJarWithoutManifestName") + ".jar"; final Path jarLocation = Paths.get(System.getProperty("targetDir")); jarWithManifest = Files.copy( jarLocation.resolve(parameterProgramJarName), jarDir.resolve("program-with-manifest.jar")); jarWithoutManifest = Files.copy( jarLocation.resolve(parameterProgramWithoutManifestJarName), jarDir.resolve("program-without-manifest.jar")); restfulGateway = new TestingDispatcherGateway.Builder() .setBlobServerPort(BLOB_SERVER_RESOURCE.getBlobServerPort()) .setSubmitFunction(jobGraph -> { LAST_SUBMITTED_JOB_GRAPH_REFERENCE.set(jobGraph); return CompletableFuture.completedFuture(Acknowledge.get()); }) .build(); gatewayRetriever = () -> CompletableFuture.completedFuture(restfulGateway); localAddressFuture = CompletableFuture.completedFuture("shazam://localhost:12345"); timeout = Time.seconds(10); responseHeaders = Collections.emptyMap(); executor = TestingUtils.defaultExecutor(); }
@Override public CompletableFuture<Acknowledge> disposeSavepoint(String savepointPath) { final SavepointDisposalRequest savepointDisposalRequest = new SavepointDisposalRequest(savepointPath); final CompletableFuture<TriggerResponse> savepointDisposalTriggerFuture = sendRequest( SavepointDisposalTriggerHeaders.getInstance(), savepointDisposalRequest); final CompletableFuture<AsynchronousOperationInfo> savepointDisposalFuture = savepointDisposalTriggerFuture.thenCompose( (TriggerResponse triggerResponse) -> { final TriggerId triggerId = triggerResponse.getTriggerId(); final SavepointDisposalStatusHeaders savepointDisposalStatusHeaders = SavepointDisposalStatusHeaders.getInstance(); final SavepointDisposalStatusMessageParameters savepointDisposalStatusMessageParameters = savepointDisposalStatusHeaders.getUnresolvedMessageParameters(); savepointDisposalStatusMessageParameters.triggerIdPathParameter.resolve(triggerId); return pollResourceAsync( () -> sendRequest( savepointDisposalStatusHeaders, savepointDisposalStatusMessageParameters)); }); return savepointDisposalFuture.thenApply( (AsynchronousOperationInfo asynchronousOperationInfo) -> { if (asynchronousOperationInfo.getFailureCause() == null) { return Acknowledge.get(); } else { throw new CompletionException(asynchronousOperationInfo.getFailureCause()); } }); }
/** * Tests disposal with a JAR file. */ @Test public void testDisposeWithJar() throws Exception { replaceStdOutAndStdErr(); final CompletableFuture<String> disposeSavepointFuture = new CompletableFuture<>(); final DisposeSavepointClusterClient clusterClient = new DisposeSavepointClusterClient( (String savepointPath) -> { disposeSavepointFuture.complete(savepointPath); return CompletableFuture.completedFuture(Acknowledge.get()); }, getConfiguration()); try { CliFrontend frontend = new MockedCliFrontend(clusterClient); // Fake JAR file File f = tmp.newFile(); ZipOutputStream out = new ZipOutputStream(new FileOutputStream(f)); out.close(); final String disposePath = "any-path"; String[] parameters = { "-d", disposePath, "-j", f.getAbsolutePath() }; frontend.savepoint(parameters); final String actualSavepointPath = disposeSavepointFuture.get(); assertEquals(disposePath, actualSavepointPath); } finally { clusterClient.shutdown(); restoreStdOutAndStdErr(); } }
@Override public CompletableFuture<Acknowledge> rescaleJob(JobID jobId, int newParallelism) { final RescalingTriggerHeaders rescalingTriggerHeaders = RescalingTriggerHeaders.getInstance(); final RescalingTriggerMessageParameters rescalingTriggerMessageParameters = rescalingTriggerHeaders.getUnresolvedMessageParameters(); rescalingTriggerMessageParameters.jobPathParameter.resolve(jobId); rescalingTriggerMessageParameters.rescalingParallelismQueryParameter.resolve(Collections.singletonList(newParallelism)); final CompletableFuture<TriggerResponse> rescalingTriggerResponseFuture = sendRequest( rescalingTriggerHeaders, rescalingTriggerMessageParameters); final CompletableFuture<AsynchronousOperationInfo> rescalingOperationFuture = rescalingTriggerResponseFuture.thenCompose( (TriggerResponse triggerResponse) -> { final TriggerId triggerId = triggerResponse.getTriggerId(); final RescalingStatusHeaders rescalingStatusHeaders = RescalingStatusHeaders.getInstance(); final RescalingStatusMessageParameters rescalingStatusMessageParameters = rescalingStatusHeaders.getUnresolvedMessageParameters(); rescalingStatusMessageParameters.jobPathParameter.resolve(jobId); rescalingStatusMessageParameters.triggerIdPathParameter.resolve(triggerId); return pollResourceAsync( () -> sendRequest( rescalingStatusHeaders, rescalingStatusMessageParameters)); }); return rescalingOperationFuture.thenApply( (AsynchronousOperationInfo asynchronousOperationInfo) -> { if (asynchronousOperationInfo.getFailureCause() == null) { return Acknowledge.get(); } else { throw new CompletionException(asynchronousOperationInfo.getFailureCause()); } }); }
@Test public void testJarSubmission() throws Exception { final TestingDispatcherGateway restfulGateway = new TestingDispatcherGateway.Builder() .setBlobServerPort(blobServerResource.getBlobServerPort()) .setSubmitFunction(jobGraph -> CompletableFuture.completedFuture(Acknowledge.get())) .build(); final JarHandlers handlers = new JarHandlers(temporaryFolder.newFolder().toPath(), restfulGateway); final JarUploadHandler uploadHandler = handlers.uploadHandler; final JarListHandler listHandler = handlers.listHandler; final JarPlanHandler planHandler = handlers.planHandler; final JarRunHandler runHandler = handlers.runHandler; final JarDeleteHandler deleteHandler = handlers.deleteHandler; // targetDir property is set via surefire configuration final Path originalJar = Paths.get(System.getProperty("targetDir")).resolve("test-program.jar"); final Path jar = Files.copy(originalJar, temporaryFolder.getRoot().toPath().resolve("test-program.jar")); final String storedJarPath = uploadJar(uploadHandler, jar, restfulGateway); final String storedJarName = Paths.get(storedJarPath).getFileName().toString(); final JarListInfo postUploadListResponse = listJars(listHandler, restfulGateway); Assert.assertEquals(1, postUploadListResponse.jarFileList.size()); final JarListInfo.JarFileInfo listEntry = postUploadListResponse.jarFileList.iterator().next(); Assert.assertEquals(jar.getFileName().toString(), listEntry.name); Assert.assertEquals(storedJarName, listEntry.id); final JobPlanInfo planResponse = showPlan(planHandler, storedJarName, restfulGateway); // we're only interested in the core functionality so checking for a small detail is sufficient Assert.assertThat(planResponse.getJsonPlan(), containsString("TestProgram.java:29")); runJar(runHandler, storedJarName, restfulGateway); deleteJar(deleteHandler, storedJarName, restfulGateway); final JarListInfo postDeleteListResponse = listJars(listHandler, restfulGateway); Assert.assertEquals(0, postDeleteListResponse.jarFileList.size()); }
@Test public void testDisposeSavepointSuccess() throws Exception { replaceStdOutAndStdErr(); String savepointPath = "expectedSavepointPath"; ClusterClient clusterClient = new DisposeSavepointClusterClient( (String path) -> CompletableFuture.completedFuture(Acknowledge.get()), getConfiguration()); try { CliFrontend frontend = new MockedCliFrontend(clusterClient); String[] parameters = { "-d", savepointPath }; frontend.savepoint(parameters); String outMsg = buffer.toString(); assertTrue(outMsg.contains(savepointPath)); assertTrue(outMsg.contains("disposed")); } finally { clusterClient.shutdown(); restoreStdOutAndStdErr(); } }
assertThat(disposeSavepointFuture.get(), is(Acknowledge.get()));
@Override public CompletableFuture<Acknowledge> archiveExecutionGraph(AccessExecutionGraph executionGraph) { return CompletableFuture.completedFuture(Acknowledge.get()); } }
@Override public CompletableFuture<Acknowledge> freeSlot(AllocationID allocationId, Throwable cause, Time timeout) { freeSlotInternal(allocationId, cause); return CompletableFuture.completedFuture(Acknowledge.get()); }
@Override public CompletableFuture<Acknowledge> cancel(Time timeout) { executionGraph.cancel(); return CompletableFuture.completedFuture(Acknowledge.get()); }
@Override public CompletableFuture<Acknowledge> scheduleOrUpdateConsumers( final ResultPartitionID partitionID, final Time timeout) { try { executionGraph.scheduleOrUpdateConsumers(partitionID); return CompletableFuture.completedFuture(Acknowledge.get()); } catch (Exception e) { return FutureUtils.completedExceptionally(e); } }
@Override public CompletableFuture<Acknowledge> stop(Time timeout) { try { executionGraph.stop(); } catch (StoppingException e) { return FutureUtils.completedExceptionally(e); } return CompletableFuture.completedFuture(Acknowledge.get()); }
@Override public CompletableFuture<Acknowledge> setPlacementConstraints( JobID jobId, List<PlacementConstraint> constraints, @RpcTimeout Time timeout) { slotManager.setJobConstraints(jobId, constraints); return CompletableFuture.completedFuture(Acknowledge.get()); }
@Override public CompletableFuture<Acknowledge> scheduleOrUpdateConsumers( final ResultPartitionID partitionID, final Time timeout) { try { executionGraph.scheduleOrUpdateConsumers(partitionID); return CompletableFuture.completedFuture(Acknowledge.get()); } catch (Exception e) { return FutureUtils.completedExceptionally(e); } }
@Override public CompletableFuture<Acknowledge> sendSlotReport(ResourceID taskManagerResourceId, InstanceID taskManagerRegistrationId, SlotReport slotReport, Time timeout) { final WorkerRegistration<WorkerType> workerTypeWorkerRegistration = taskExecutors.get(taskManagerResourceId); if (workerTypeWorkerRegistration.getInstanceID().equals(taskManagerRegistrationId)) { slotManager.registerTaskManager(workerTypeWorkerRegistration, slotReport); return CompletableFuture.completedFuture(Acknowledge.get()); } else { return FutureUtils.completedExceptionally(new ResourceManagerException(String.format("Unknown TaskManager registration id %s.", taskManagerRegistrationId))); } }
@Override public CompletableFuture<Acknowledge> archiveExecutionGraph(AccessExecutionGraph executionGraph) { try { FsJobArchivist.archiveJob(archivePath, executionGraph.getJobID(), jsonArchivist.archiveJsonWithPath(executionGraph)); return CompletableFuture.completedFuture(Acknowledge.get()); } catch (IOException e) { return FutureUtils.completedExceptionally(e); } } }
@RpcMethod public Acknowledge scheduleOrUpdateConsumers( final UUID leaderSessionID, final ResultPartitionID partitionID) throws Exception { validateLeaderSessionId(leaderSessionID); executionGraph.scheduleOrUpdateConsumers(partitionID); return Acknowledge.get(); }
@Override public CompletableFuture<Acknowledge> archiveExecutionGraph(AccessExecutionGraph executionGraph) { try { FsJobArchivist.archiveJob(archivePath, executionGraph.getJobID(), jsonArchivist.archiveJsonWithPath(executionGraph)); return CompletableFuture.completedFuture(Acknowledge.get()); } catch (IOException e) { return FutureUtils.completedExceptionally(e); } } }