@Override public Iterator<ArchivedExecutionVertex> iterator() { return new AllVerticesIterator(getVerticesTopologically().iterator()); } };
@Override public CompletableFuture<ArchivedExecutionGraph> requestJob(Time timeout) { return CompletableFuture.completedFuture(ArchivedExecutionGraph.createFrom(executionGraph)); }
private int calculateSize(JobID jobId, ArchivedExecutionGraph serializableExecutionGraph) { final File archivedExecutionGraphFile = getExecutionGraphFile(jobId); if (archivedExecutionGraphFile.exists()) { return Math.toIntExact(archivedExecutionGraphFile.length()); } else { LOG.debug("Could not find archived execution graph file for {}. Estimating the size instead.", jobId); return serializableExecutionGraph.getAllVertices().size() * 1000 + serializableExecutionGraph.getAccumulatorsSerialized().size() * 1000; } }
protected void jobReachedGloballyTerminalState(ArchivedExecutionGraph archivedExecutionGraph) { Preconditions.checkArgument( archivedExecutionGraph.getState().isGloballyTerminalState(), "Job %s is in state %s which is not globally terminal.", archivedExecutionGraph.getJobID(), archivedExecutionGraph.getState()); log.info("Job {} reached globally terminal state {}.", archivedExecutionGraph.getJobID(), archivedExecutionGraph.getState()); archiveExecutionGraph(archivedExecutionGraph); final JobID jobId = archivedExecutionGraph.getJobID(); removeJobAndRegisterTerminationFuture(jobId, true); }
private void archiveExecutionGraph(ArchivedExecutionGraph archivedExecutionGraph) { try { archivedExecutionGraphStore.put(archivedExecutionGraph); } catch (IOException e) { log.info( "Could not store completed job {}({}).", archivedExecutionGraph.getJobName(), archivedExecutionGraph.getJobID(), e); } final CompletableFuture<Acknowledge> executionGraphFuture = historyServerArchivist.archiveExecutionGraph(archivedExecutionGraph); executionGraphFuture.whenComplete( (Acknowledge ignored, Throwable throwable) -> { if (throwable != null) { log.info( "Could not archive completed job {}({}) to the history server.", archivedExecutionGraph.getJobName(), archivedExecutionGraph.getJobID(), throwable); } }); }
@Override public void put(ArchivedExecutionGraph archivedExecutionGraph) throws IOException { final JobStatus jobStatus = archivedExecutionGraph.getState(); final JobID jobId = archivedExecutionGraph.getJobID(); final String jobName = archivedExecutionGraph.getJobName(); Preconditions.checkArgument( jobStatus.isGloballyTerminalState(), "The job " + jobName + '(' + jobId + ") is not in a globally terminal state. Instead it is in state " + jobStatus + '.'); switch (jobStatus) { case FINISHED: numFinishedJobs++; break; case CANCELED: numCanceledJobs++; break; case FAILED: numFailedJobs++; break; default: throw new IllegalStateException("The job " + jobName + '(' + jobId + ") should have been in a globally terminal state. " + "Instead it was in state " + jobStatus + '.'); } // write the ArchivedExecutionGraph to disk storeArchivedExecutionGraph(archivedExecutionGraph); final JobDetails detailsForJob = WebMonitorUtils.createDetailsForJob(archivedExecutionGraph); jobDetailsCache.put(jobId, detailsForJob); archivedExecutionGraphCache.put(jobId, archivedExecutionGraph); }
return new ArchivedExecutionGraph( getJobID(), getJobName(),
@Override public void put(ArchivedExecutionGraph serializableExecutionGraph) throws IOException { serializableExecutionGraphs.put(serializableExecutionGraph.getJobID(), serializableExecutionGraph); }
@Override protected void jobReachedGloballyTerminalState(ArchivedExecutionGraph archivedExecutionGraph) { super.jobReachedGloballyTerminalState(archivedExecutionGraph); if (executionMode == ClusterEntrypoint.ExecutionMode.DETACHED) { // shut down since we don't have to wait for the execution result retrieval jobTerminationFuture.complete(ApplicationStatus.fromJobStatus(archivedExecutionGraph.getState())); } }
@Override public void put(ArchivedExecutionGraph archivedExecutionGraph) throws IOException { final JobStatus jobStatus = archivedExecutionGraph.getState(); final JobID jobId = archivedExecutionGraph.getJobID(); final String jobName = archivedExecutionGraph.getJobName(); Preconditions.checkArgument( jobStatus.isGloballyTerminalState(), "The job " + jobName + '(' + jobId + ") is not in a globally terminal state. Instead it is in state " + jobStatus + '.'); switch (jobStatus) { case FINISHED: numFinishedJobs++; break; case CANCELED: numCanceledJobs++; break; case FAILED: numFailedJobs++; break; default: throw new IllegalStateException("The job " + jobName + '(' + jobId + ") should have been in a globally terminal state. " + "Instead it was in state " + jobStatus + '.'); } // write the ArchivedExecutionGraph to disk storeArchivedExecutionGraph(archivedExecutionGraph); final JobDetails detailsForJob = WebMonitorUtils.createDetailsForJob(archivedExecutionGraph); jobDetailsCache.put(jobId, detailsForJob); archivedExecutionGraphCache.put(jobId, archivedExecutionGraph); }
protected void jobReachedGloballyTerminalState(ArchivedExecutionGraph archivedExecutionGraph) { Preconditions.checkArgument( archivedExecutionGraph.getState().isGloballyTerminalState(), "Job %s is in state %s which is not globally terminal.", archivedExecutionGraph.getJobID(), archivedExecutionGraph.getState()); log.info("Job {} reached globally terminal state {}.", archivedExecutionGraph.getJobID(), archivedExecutionGraph.getState()); archiveExecutionGraph(archivedExecutionGraph); final JobID jobId = archivedExecutionGraph.getJobID(); removeJobAndRegisterTerminationFuture(jobId, true); }
private void archiveExecutionGraph(ArchivedExecutionGraph archivedExecutionGraph) { try { archivedExecutionGraphStore.put(archivedExecutionGraph); } catch (IOException e) { log.info( "Could not store completed job {}({}).", archivedExecutionGraph.getJobName(), archivedExecutionGraph.getJobID(), e); } final CompletableFuture<Acknowledge> executionGraphFuture = historyServerArchivist.archiveExecutionGraph(archivedExecutionGraph); executionGraphFuture.whenComplete( (Acknowledge ignored, Throwable throwable) -> { if (throwable != null) { log.info( "Could not archive completed job {}({}) to the history server.", archivedExecutionGraph.getJobName(), archivedExecutionGraph.getJobID(), throwable); } }); }
return new ArchivedExecutionGraph( executionGraph.getJobID(), executionGraph.getJobName(),
@Override public void put(ArchivedExecutionGraph serializableExecutionGraph) throws IOException { serializableExecutionGraphs.put(serializableExecutionGraph.getJobID(), serializableExecutionGraph); }
@Override protected void jobReachedGloballyTerminalState(ArchivedExecutionGraph archivedExecutionGraph) { super.jobReachedGloballyTerminalState(archivedExecutionGraph); if (executionMode == ClusterEntrypoint.ExecutionMode.DETACHED) { // shut down since we don't have to wait for the execution result retrieval jobTerminationFuture.complete(ApplicationStatus.fromJobStatus(archivedExecutionGraph.getState())); } }
@Override public void put(ArchivedExecutionGraph archivedExecutionGraph) throws IOException { final JobStatus jobStatus = archivedExecutionGraph.getState(); final JobID jobId = archivedExecutionGraph.getJobID(); final String jobName = archivedExecutionGraph.getJobName(); Preconditions.checkArgument( jobStatus.isGloballyTerminalState(), "The job " + jobName + '(' + jobId + ") is not in a globally terminal state. Instead it is in state " + jobStatus + '.'); switch (jobStatus) { case FINISHED: numFinishedJobs++; break; case CANCELED: numCanceledJobs++; break; case FAILED: numFailedJobs++; break; default: throw new IllegalStateException("The job " + jobName + '(' + jobId + ") should have been in a globally terminal state. " + "Instead it was in state " + jobStatus + '.'); } // write the ArchivedExecutionGraph to disk storeArchivedExecutionGraph(archivedExecutionGraph); final JobDetails detailsForJob = WebMonitorUtils.createDetailsForJob(archivedExecutionGraph); jobDetailsCache.put(jobId, detailsForJob); archivedExecutionGraphCache.put(jobId, archivedExecutionGraph); }
private int calculateSize(JobID jobId, ArchivedExecutionGraph serializableExecutionGraph) { final File archivedExecutionGraphFile = getExecutionGraphFile(jobId); if (archivedExecutionGraphFile.exists()) { return Math.toIntExact(archivedExecutionGraphFile.length()); } else { LOG.debug("Could not find archived execution graph file for {}. Estimating the size instead.", jobId); return serializableExecutionGraph.getAllVertices().size() * 1000 + serializableExecutionGraph.getAccumulatorsSerialized().size() * 1000; } }
protected void jobReachedGloballyTerminalState(ArchivedExecutionGraph archivedExecutionGraph) { Preconditions.checkArgument( archivedExecutionGraph.getState().isGloballyTerminalState(), "Job %s is in state %s which is not globally terminal.", archivedExecutionGraph.getJobID(), archivedExecutionGraph.getState()); log.info("Job {} reached globally terminal state {}.", archivedExecutionGraph.getJobID(), archivedExecutionGraph.getState()); archiveExecutionGraph(archivedExecutionGraph); final JobID jobId = archivedExecutionGraph.getJobID(); removeJobAndRegisterTerminationFuture(jobId, true); }
private void archiveExecutionGraph(ArchivedExecutionGraph archivedExecutionGraph) { try { archivedExecutionGraphStore.put(archivedExecutionGraph); } catch (IOException e) { log.info( "Could not store completed job {}({}).", archivedExecutionGraph.getJobName(), archivedExecutionGraph.getJobID(), e); } final CompletableFuture<Acknowledge> executionGraphFuture = historyServerArchivist.archiveExecutionGraph(archivedExecutionGraph); executionGraphFuture.whenComplete( (Acknowledge ignored, Throwable throwable) -> { if (throwable != null) { log.info( "Could not archive completed job {}({}) to the history server.", archivedExecutionGraph.getJobName(), archivedExecutionGraph.getJobID(), throwable); } }); }
@Override public Iterator<ArchivedExecutionVertex> iterator() { return new AllVerticesIterator(getVerticesTopologically().iterator()); } };