private <T> void deployJobOnNewCluster( ClusterDescriptor<T> clusterDescriptor, JobGraph jobGraph, Result<T> result, ClassLoader classLoader) throws Exception { ClusterClient<T> clusterClient = null; try { // deploy job cluster with job attached clusterClient = clusterDescriptor.deployJobCluster(context.getClusterSpec(), jobGraph, false); // save information about the new cluster result.setClusterInformation(clusterClient.getClusterId(), clusterClient.getWebInterfaceURL()); // get result if (awaitJobResult) { // we need to hard cast for now final JobExecutionResult jobResult = ((RestClusterClient<T>) clusterClient) .requestJobResult(jobGraph.getJobID()) .get() .toJobExecutionResult(context.getClassLoader()); // throws exception if job fails executionResultBucket.add(jobResult); } } finally { try { if (clusterClient != null) { clusterClient.shutdown(); } } catch (Exception e) { // ignore } } }
private void executeSchedulingTest(Configuration configuration) throws Exception { configuration.setInteger(RestOptions.PORT, 0); final long slotIdleTimeout = 50L; configuration.setLong(JobManagerOptions.SLOT_IDLE_TIMEOUT, slotIdleTimeout); final int parallelism = 4; final MiniClusterConfiguration miniClusterConfiguration = new MiniClusterConfiguration.Builder() .setConfiguration(configuration) .setNumTaskManagers(parallelism) .setNumSlotsPerTaskManager(1) .build(); try (MiniCluster miniCluster = new MiniCluster(miniClusterConfiguration)) { miniCluster.start(); MiniClusterClient miniClusterClient = new MiniClusterClient(configuration, miniCluster); JobGraph jobGraph = createJobGraph(slotIdleTimeout << 1, parallelism); CompletableFuture<JobSubmissionResult> submissionFuture = miniClusterClient.submitJob(jobGraph); // wait for the submission to succeed JobSubmissionResult jobSubmissionResult = submissionFuture.get(); CompletableFuture<JobResult> resultFuture = miniClusterClient.requestJobResult(jobSubmissionResult.getJobID()); JobResult jobResult = resultFuture.get(); assertThat(jobResult.getSerializedThrowable().isPresent(), is(false)); } }
jobIdSerializer.serialize(result.getJobId(), gen, provider); final Map<String, SerializedValue<OptionalFailure<Object>>> accumulatorResults = result.getAccumulatorResults(); for (final Map.Entry<String, SerializedValue<OptionalFailure<Object>>> nameValue : accumulatorResults.entrySet()) { final String name = nameValue.getKey(); gen.writeNumberField(FIELD_NAME_NET_RUNTIME, result.getNetRuntime()); if (result.getSerializedThrowable().isPresent()) { gen.writeFieldName(FIELD_NAME_FAILURE_CAUSE); final SerializedThrowable serializedThrowable = result.getSerializedThrowable().get(); serializedThrowableSerializer.serialize(serializedThrowable, gen, provider);
public JobResult build() { return new JobResult( jobId, applicationStatus, accumulatorResults == null ? Collections.emptyMap() : accumulatorResults, netRuntime, serializedThrowable); } }
@Override public CompletableFuture<JobResult> requestJobResult(JobID jobId, Time timeout) { final CompletableFuture<JobManagerRunner> jobManagerRunnerFuture = jobManagerRunnerFutures.get(jobId); if (jobManagerRunnerFuture == null) { final ArchivedExecutionGraph archivedExecutionGraph = archivedExecutionGraphStore.get(jobId); if (archivedExecutionGraph == null) { return FutureUtils.completedExceptionally(new FlinkJobNotFoundException(jobId)); } else { return CompletableFuture.completedFuture(JobResult.createFrom(archivedExecutionGraph)); } } else { return jobManagerRunnerFuture.thenCompose(JobManagerRunner::getResultFuture).thenApply(JobResult::createFrom); } }
jobIdSerializer.serialize(result.getJobId(), gen, provider); gen.writeString(result.getApplicationStatus().name()); final Map<String, SerializedValue<OptionalFailure<Object>>> accumulatorResults = result.getAccumulatorResults(); for (final Map.Entry<String, SerializedValue<OptionalFailure<Object>>> nameValue : accumulatorResults.entrySet()) { final String name = nameValue.getKey(); gen.writeNumberField(FIELD_NAME_NET_RUNTIME, result.getNetRuntime()); if (result.getSerializedThrowable().isPresent()) { gen.writeFieldName(FIELD_NAME_FAILURE_CAUSE); final SerializedThrowable serializedThrowable = result.getSerializedThrowable().get(); serializedThrowableSerializer.serialize(serializedThrowable, gen, provider);
public JobResult build() { return new JobResult( jobId, accumulatorResults == null ? Collections.emptyMap() : accumulatorResults, netRuntime, serializedThrowable); } }
@Override public CompletableFuture<JobResult> requestJobResult(JobID jobId, Time timeout) { final CompletableFuture<JobManagerRunner> jobManagerRunnerFuture = jobManagerRunnerFutures.get(jobId); if (jobManagerRunnerFuture == null) { final ArchivedExecutionGraph archivedExecutionGraph = archivedExecutionGraphStore.get(jobId); if (archivedExecutionGraph == null) { return FutureUtils.completedExceptionally(new FlinkJobNotFoundException(jobId)); } else { return CompletableFuture.completedFuture(JobResult.createFrom(archivedExecutionGraph)); } } else { return jobManagerRunnerFuture.thenCompose(JobManagerRunner::getResultFuture).thenApply(JobResult::createFrom); } }
jobIdSerializer.serialize(result.getJobId(), gen, provider); gen.writeString(result.getApplicationStatus().name()); final Map<String, SerializedValue<OptionalFailure<Object>>> accumulatorResults = result.getAccumulatorResults(); for (final Map.Entry<String, SerializedValue<OptionalFailure<Object>>> nameValue : accumulatorResults.entrySet()) { final String name = nameValue.getKey(); gen.writeNumberField(FIELD_NAME_NET_RUNTIME, result.getNetRuntime()); if (result.getSerializedThrowable().isPresent()) { gen.writeFieldName(FIELD_NAME_FAILURE_CAUSE); final SerializedThrowable serializedThrowable = result.getSerializedThrowable().get(); serializedThrowableSerializer.serialize(serializedThrowable, gen, provider);
@Override public JobSubmissionResult submitJob(JobGraph jobGraph, ClassLoader classLoader) throws ProgramInvocationException { final CompletableFuture<JobSubmissionResult> jobSubmissionResultFuture = submitJob(jobGraph); if (isDetached()) { try { return jobSubmissionResultFuture.get(); } catch (InterruptedException | ExecutionException e) { ExceptionUtils.checkInterrupted(e); throw new ProgramInvocationException("Could not run job in detached mode.", jobGraph.getJobID(), e); } } else { final CompletableFuture<JobResult> jobResultFuture = jobSubmissionResultFuture.thenCompose( (JobSubmissionResult ignored) -> requestJobResult(jobGraph.getJobID())); final JobResult jobResult; try { jobResult = jobResultFuture.get(); } catch (InterruptedException | ExecutionException e) { ExceptionUtils.checkInterrupted(e); throw new ProgramInvocationException("Could not run job", jobGraph.getJobID(), e); } try { return jobResult.toJobExecutionResult(classLoader); } catch (JobExecutionException e) { throw new ProgramInvocationException("Job failed", jobGraph.getJobID(), e); } catch (IOException | ClassNotFoundException e) { throw new ProgramInvocationException("Job failed", jobGraph.getJobID(), e); } } }
assertThat(jobResult.getSerializedThrowable().isPresent(), is(false)); } finally { if (clusterClient != null) {
public JobResult build() { return new JobResult( jobId, applicationStatus, accumulatorResults == null ? Collections.emptyMap() : accumulatorResults, netRuntime, serializedThrowable); } }
@Override public CompletableFuture<JobResult> requestJobResult(JobID jobId, Time timeout) { final JobManagerRunner jobManagerRunner = jobManagerRunners.get(jobId); if (jobManagerRunner == null) { final ArchivedExecutionGraph archivedExecutionGraph = archivedExecutionGraphStore.get(jobId); if (archivedExecutionGraph == null) { return FutureUtils.completedExceptionally(new FlinkJobNotFoundException(jobId)); } else { return CompletableFuture.completedFuture(JobResult.createFrom(archivedExecutionGraph)); } } else { return jobManagerRunner.getResultFuture().thenApply(JobResult::createFrom); } }
this.lastJobExecutionResult = jobResult.toJobExecutionResult(classLoader); return lastJobExecutionResult; } catch (JobExecutionException e) {
@Override public CompletableFuture<JobResult> requestJobResult(JobID jobId, Time timeout) { final CompletableFuture<JobResult> jobResultFuture = super.requestJobResult(jobId, timeout); if (executionMode == ClusterEntrypoint.ExecutionMode.NORMAL) { // terminate the MiniDispatcher once we served the first JobResult successfully jobResultFuture.thenAccept((JobResult result) -> { ApplicationStatus status = result.getSerializedThrowable().isPresent() ? ApplicationStatus.FAILED : ApplicationStatus.SUCCEEDED; jobTerminationFuture.complete(status); }); } return jobResultFuture; }
/** * This method runs a job in blocking mode. The method returns only after the job * completed successfully, or after it failed terminally. * * @param job The Flink job to execute * @return The result of the job execution * * @throws JobExecutionException Thrown if anything went amiss during initial job launch, * or if the job terminally failed. */ @Override public JobExecutionResult executeJobBlocking(JobGraph job) throws JobExecutionException, InterruptedException { checkNotNull(job, "job is null"); final CompletableFuture<JobSubmissionResult> submissionFuture = submitJob(job); final CompletableFuture<JobResult> jobResultFuture = submissionFuture.thenCompose( (JobSubmissionResult ignored) -> requestJobResult(job.getJobID())); final JobResult jobResult; try { jobResult = jobResultFuture.get(); } catch (ExecutionException e) { throw new JobExecutionException(job.getJobID(), "Could not retrieve JobResult.", ExceptionUtils.stripExecutionException(e)); } try { return jobResult.toJobExecutionResult(Thread.currentThread().getContextClassLoader()); } catch (IOException | ClassNotFoundException e) { throw new JobExecutionException(job.getJobID(), e); } }
@Override public CompletableFuture<JobResult> requestJobResult(JobID jobId, Time timeout) { final CompletableFuture<JobResult> jobResultFuture = super.requestJobResult(jobId, timeout); if (executionMode == ClusterEntrypoint.ExecutionMode.NORMAL) { // terminate the MiniDispatcher once we served the first JobResult successfully jobResultFuture.thenAccept((JobResult result) -> { ApplicationStatus status = result.getSerializedThrowable().isPresent() ? ApplicationStatus.FAILED : ApplicationStatus.SUCCEEDED; jobTerminationFuture.complete(status); }); } return jobResultFuture; }
/** * This method runs a job in blocking mode. The method returns only after the job * completed successfully, or after it failed terminally. * * @param job The Flink job to execute * @return The result of the job execution * * @throws JobExecutionException Thrown if anything went amiss during initial job launch, * or if the job terminally failed. */ @Override public JobExecutionResult executeJobBlocking(JobGraph job) throws JobExecutionException, InterruptedException { checkNotNull(job, "job is null"); final CompletableFuture<JobSubmissionResult> submissionFuture = submitJob(job); final CompletableFuture<JobResult> jobResultFuture = submissionFuture.thenCompose( (JobSubmissionResult ignored) -> requestJobResult(job.getJobID())); final JobResult jobResult; try { jobResult = jobResultFuture.get(); } catch (ExecutionException e) { throw new JobExecutionException(job.getJobID(), "Could not retrieve JobResult.", ExceptionUtils.stripExecutionException(e)); } try { return jobResult.toJobExecutionResult(Thread.currentThread().getContextClassLoader()); } catch (IOException | ClassNotFoundException e) { throw new JobExecutionException(job.getJobID(), e); } }
@Override public CompletableFuture<JobResult> requestJobResult(JobID jobId, Time timeout) { final CompletableFuture<JobResult> jobResultFuture = super.requestJobResult(jobId, timeout); if (executionMode == ClusterEntrypoint.ExecutionMode.NORMAL) { // terminate the MiniDispatcher once we served the first JobResult successfully jobResultFuture.thenAccept((JobResult result) -> { ApplicationStatus status = result.getSerializedThrowable().isPresent() ? ApplicationStatus.FAILED : ApplicationStatus.SUCCEEDED; jobTerminationFuture.complete(status); }); } return jobResultFuture; }
return jobResult.toJobExecutionResult(classLoader); } catch (JobResult.WrappedJobException e) { throw new ProgramInvocationException(e.getCause());