@Override public CompletableFuture<Collection<JobStatusMessage>> listJobs() { return sendRequest(JobsOverviewHeaders.getInstance()) .thenApply( (multipleJobsDetails) -> multipleJobsDetails .getJobs() .stream() .map(detail -> new JobStatusMessage( detail.getJobId(), detail.getJobName(), detail.getStatus(), detail.getStartTime())) .collect(Collectors.toList())); }
/** * This method replicates the JSON response that would be given by the JobsOverviewHandler when * listing both running and finished jobs. * * <p>Every job archive contains a joboverview.json file containing the same structure. Since jobs are archived on * their own however the list of finished jobs only contains a single job. * * <p>For the display in the HistoryServer WebFrontend we have to combine these overviews. */ private static void updateJobOverview(File webOverviewDir, File webDir) { try (JsonGenerator gen = jacksonFactory.createGenerator(HistoryServer.createOrGetFile(webDir, JobsOverviewHeaders.URL))) { File[] overviews = new File(webOverviewDir.getPath()).listFiles(); if (overviews != null) { Collection<JobDetails> allJobs = new ArrayList<>(overviews.length); for (File overview : overviews) { MultipleJobsDetails subJobs = mapper.readValue(overview, MultipleJobsDetails.class); allJobs.addAll(subJobs.getJobs()); } mapper.writeValue(gen, new MultipleJobsDetails(allJobs)); } } catch (IOException ioe) { LOG.error("Failed to update job overview.", ioe); } } }
@Override protected CompletableFuture<MultipleJobsDetails> handleRequest(@Nonnull HandlerRequest<EmptyRequestBody, EmptyMessageParameters> request, @Nonnull DispatcherGateway gateway) throws RestHandlerException { JobDetails running = new JobDetails(new JobID(), "job1", 0, 0, 0, JobStatus.RUNNING, 0, new int[9], 0); JobDetails finished = new JobDetails(new JobID(), "job2", 0, 0, 0, JobStatus.FINISHED, 0, new int[9], 0); return CompletableFuture.completedFuture(new MultipleJobsDetails(Arrays.asList(running, finished))); } }
@Override public void onSuccess(Object result) throws Throwable { MultipleJobsDetails details = (MultipleJobsDetails) result; ArrayList<String> toRetain = new ArrayList<>(); for (JobDetails job : details.getRunningJobs()) { toRetain.add(job.getJobId().toString()); } for (JobDetails job : details.getFinishedJobs()) { toRetain.add(job.getJobId().toString()); } synchronized (metrics) { metrics.jobs.keySet().retainAll(toRetain); } } }, ctx);
@Override public MultipleJobsDetails process(RequestJobDetails message) { JobDetails running = new JobDetails(new JobID(), "job1", 0, 0, 0, JobStatus.RUNNING, 0, new int[9], 0); JobDetails finished = new JobDetails(new JobID(), "job2", 0, 0, 0, JobStatus.FINISHED, 0, new int[9], 0); return new MultipleJobsDetails(Arrays.asList(running, finished)); } }
for (JobDetails detail : result.getRunningJobs()) { writeJobDetailOverviewAsJson(detail, gen, now); for (JobDetails detail : result.getFinishedJobs()) { writeJobDetailOverviewAsJson(detail, gen, now); for (JobDetails detail : includeRunningJobs ? result.getRunningJobs() : result.getFinishedJobs()) { writeJobDetailOverviewAsJson(detail, gen, now);
/** * Lists the currently running and finished jobs on the cluster. * * @return future collection of running and finished jobs * @throws Exception if no connection to the cluster could be established */ public CompletableFuture<Collection<JobStatusMessage>> listJobs() throws Exception { final ActorGateway jobManager = getJobManagerGateway(); Future<Object> response = jobManager.ask(new RequestJobDetails(true, false), timeout); CompletableFuture<Object> responseFuture = FutureUtils.toJava(response); return responseFuture.thenApply((responseMessage) -> { if (responseMessage instanceof MultipleJobsDetails) { MultipleJobsDetails details = (MultipleJobsDetails) responseMessage; final Collection<JobDetails> jobDetails = details.getJobs(); Collection<JobStatusMessage> flattenedDetails = new ArrayList<>(jobDetails.size()); jobDetails.forEach(detail -> flattenedDetails.add(new JobStatusMessage(detail.getJobId(), detail.getJobName(), detail.getStatus(), detail.getStartTime()))); return flattenedDetails; } else { throw new CompletionException( new IllegalStateException("Unknown JobManager response of type " + responseMessage.getClass())); } }); }
/** * This method replicates the JSON response that would be given by the JobsOverviewHandler when * listing both running and finished jobs. * * <p>Every job archive contains a joboverview.json file containing the same structure. Since jobs are archived on * their own however the list of finished jobs only contains a single job. * * <p>For the display in the HistoryServer WebFrontend we have to combine these overviews. */ private static void updateJobOverview(File webOverviewDir, File webDir) { try (JsonGenerator gen = jacksonFactory.createGenerator(HistoryServer.createOrGetFile(webDir, JobsOverviewHeaders.URL))) { File[] overviews = new File(webOverviewDir.getPath()).listFiles(); if (overviews != null) { Collection<JobDetails> allJobs = new ArrayList<>(overviews.length); for (File overview : overviews) { MultipleJobsDetails subJobs = mapper.readValue(overview, MultipleJobsDetails.class); allJobs.addAll(subJobs.getJobs()); } mapper.writeValue(gen, new MultipleJobsDetails(allJobs)); } } catch (IOException ioe) { LOG.error("Failed to update job overview.", ioe); } } }
MultipleJobsDetails multipleJobsDetails = new MultipleJobsDetails(Collections.singleton(jobDetails));
MultipleJobsDetails overview = mapper.readValue(response, MultipleJobsDetails.class); Assert.assertEquals(numJobs + 1, overview.getJobs().size()); } finally { hs.stop();
/** * This method replicates the JSON response that would be given by the JobsOverviewHandler when * listing both running and finished jobs. * * <p>Every job archive contains a joboverview.json file containing the same structure. Since jobs are archived on * their own however the list of finished jobs only contains a single job. * * <p>For the display in the HistoryServer WebFrontend we have to combine these overviews. */ private static void updateJobOverview(File webOverviewDir, File webDir) { try (JsonGenerator gen = jacksonFactory.createGenerator(HistoryServer.createOrGetFile(webDir, JobsOverviewHeaders.URL))) { File[] overviews = new File(webOverviewDir.getPath()).listFiles(); if (overviews != null) { Collection<JobDetails> allJobs = new ArrayList<>(overviews.length); for (File overview : overviews) { MultipleJobsDetails subJobs = mapper.readValue(overview, MultipleJobsDetails.class); allJobs.addAll(subJobs.getJobs()); } mapper.writeValue(gen, new MultipleJobsDetails(allJobs)); } } catch (IOException ioe) { LOG.error("Failed to update job overview.", ioe); } } }
@Override public CompletableFuture<MultipleJobsDetails> requestMultipleJobDetails(Time timeout) { List<CompletableFuture<Optional<JobDetails>>> individualOptionalJobDetails = queryJobMastersForInformation( (JobMasterGateway jobMasterGateway) -> jobMasterGateway.requestJobDetails(timeout)); CompletableFuture<Collection<Optional<JobDetails>>> optionalCombinedJobDetails = FutureUtils.combineAll( individualOptionalJobDetails); CompletableFuture<Collection<JobDetails>> combinedJobDetails = optionalCombinedJobDetails.thenApply(this::flattenOptionalCollection); final Collection<JobDetails> completedJobDetails = archivedExecutionGraphStore.getAvailableJobDetails(); return combinedJobDetails.thenApply( (Collection<JobDetails> runningJobDetails) -> { final Collection<JobDetails> allJobDetails = new ArrayList<>(completedJobDetails.size() + runningJobDetails.size()); allJobDetails.addAll(runningJobDetails); allJobDetails.addAll(completedJobDetails); return new MultipleJobsDetails(allJobDetails); }); }
@Override protected CompletableFuture<JobIdsWithStatusOverview> handleRequest( @Nonnull HandlerRequest<EmptyRequestBody, EmptyMessageParameters> request, @Nonnull RestfulGateway gateway) throws RestHandlerException { return gateway.requestMultipleJobDetails(timeout).thenApply( multipleJobDetails -> new JobIdsWithStatusOverview( multipleJobDetails .getJobs() .stream() .map( jobDetails -> new JobIdsWithStatusOverview.JobIdWithStatus( jobDetails.getJobId(), jobDetails.getStatus())) .collect(Collectors.toList()) ) ); } }
@Override public CompletableFuture<MultipleJobsDetails> requestMultipleJobDetails(Time timeout) { List<CompletableFuture<Optional<JobDetails>>> individualOptionalJobDetails = queryJobMastersForInformation( (JobMasterGateway jobMasterGateway) -> jobMasterGateway.requestJobDetails(timeout)); CompletableFuture<Collection<Optional<JobDetails>>> optionalCombinedJobDetails = FutureUtils.combineAll( individualOptionalJobDetails); CompletableFuture<Collection<JobDetails>> combinedJobDetails = optionalCombinedJobDetails.thenApply(this::flattenOptionalCollection); final Collection<JobDetails> completedJobDetails = archivedExecutionGraphStore.getAvailableJobDetails(); return combinedJobDetails.thenApply( (Collection<JobDetails> runningJobDetails) -> { final Collection<JobDetails> allJobDetails = new ArrayList<>(completedJobDetails.size() + runningJobDetails.size()); allJobDetails.addAll(runningJobDetails); allJobDetails.addAll(completedJobDetails); return new MultipleJobsDetails(allJobDetails); }); }
@Override protected CompletableFuture<JobIdsWithStatusOverview> handleRequest( @Nonnull HandlerRequest<EmptyRequestBody, EmptyMessageParameters> request, @Nonnull RestfulGateway gateway) throws RestHandlerException { return gateway.requestMultipleJobDetails(timeout).thenApply( multipleJobDetails -> new JobIdsWithStatusOverview( multipleJobDetails .getJobs() .stream() .map( jobDetails -> new JobIdsWithStatusOverview.JobIdWithStatus( jobDetails.getJobId(), jobDetails.getStatus())) .collect(Collectors.toList()) ) ); } }
@Override public CompletableFuture<MultipleJobsDetails> requestMultipleJobDetails(Time timeout) { List<CompletableFuture<Optional<JobDetails>>> individualOptionalJobDetails = queryJobMastersForInformation( (JobMasterGateway jobMasterGateway) -> jobMasterGateway.requestJobDetails(timeout)); CompletableFuture<Collection<Optional<JobDetails>>> optionalCombinedJobDetails = FutureUtils.combineAll( individualOptionalJobDetails); CompletableFuture<Collection<JobDetails>> combinedJobDetails = optionalCombinedJobDetails.thenApply(this::flattenOptionalCollection); final Collection<JobDetails> completedJobDetails = archivedExecutionGraphStore.getAvailableJobDetails(); return combinedJobDetails.thenApply( (Collection<JobDetails> runningJobDetails) -> { final Collection<JobDetails> allJobDetails = new ArrayList<>(completedJobDetails.size() + runningJobDetails.size()); allJobDetails.addAll(runningJobDetails); allJobDetails.addAll(completedJobDetails); return new MultipleJobsDetails(allJobDetails); }); }
@Override public CompletableFuture<Collection<JobStatusMessage>> listJobs() { return sendRequest(JobsOverviewHeaders.getInstance()) .thenApply( (multipleJobsDetails) -> multipleJobsDetails .getJobs() .stream() .map(detail -> new JobStatusMessage( detail.getJobId(), detail.getJobName(), detail.getStatus(), detail.getStartTime())) .collect(Collectors.toList())); }
@Override public Collection<ArchivedJson> archiveJsonWithPath(AccessExecutionGraph graph) throws IOException { ResponseBody json = new MultipleJobsDetails(Collections.singleton(WebMonitorUtils.createDetailsForJob(graph))); String path = getMessageHeaders().getTargetRestEndpointURL() .replace(':' + JobIDPathParameter.KEY, graph.getJobID().toString()); return Collections.singletonList(new ArchivedJson(path, json)); } }
@Override protected CompletableFuture<JobIdsWithStatusOverview> handleRequest( @Nonnull HandlerRequest<EmptyRequestBody, EmptyMessageParameters> request, @Nonnull RestfulGateway gateway) throws RestHandlerException { return gateway.requestMultipleJobDetails(timeout).thenApply( multipleJobDetails -> new JobIdsWithStatusOverview( multipleJobDetails .getJobs() .stream() .map( jobDetails -> new JobIdsWithStatusOverview.JobIdWithStatus( jobDetails.getJobId(), jobDetails.getStatus())) .collect(Collectors.toList()) ) ); } }
@Override public Collection<ArchivedJson> archiveJsonWithPath(AccessExecutionGraph graph) throws IOException { ResponseBody json = new MultipleJobsDetails(Collections.singleton(WebMonitorUtils.createDetailsForJob(graph))); String path = getMessageHeaders().getTargetRestEndpointURL() .replace(':' + JobIDPathParameter.KEY, graph.getJobID().toString()); return Collections.singletonList(new ArchivedJson(path, json)); } }