@Override public Map<String, OptionalFailure<Object>> getAccumulators(JobID jobID, ClassLoader loader) throws Exception { AccessExecutionGraph executionGraph = miniCluster.getExecutionGraph(jobID).get(); Map<String, SerializedValue<OptionalFailure<Object>>> accumulatorsSerialized = executionGraph.getAccumulatorsSerialized(); Map<String, OptionalFailure<Object>> result = new HashMap<>(accumulatorsSerialized.size()); for (Map.Entry<String, SerializedValue<OptionalFailure<Object>>> acc : accumulatorsSerialized.entrySet()) { result.put(acc.getKey(), acc.getValue().deserializeValue(loader)); } return result; }
private static JobDetailsInfo createJobDetailsInfo(AccessExecutionGraph executionGraph, @Nullable MetricFetcher<?> metricFetcher) { final long now = System.currentTimeMillis(); final long startTime = executionGraph.getStatusTimestamp(JobStatus.CREATED); final long endTime = executionGraph.getState().isGloballyTerminalState() ? executionGraph.getStatusTimestamp(executionGraph.getState()) : -1L; final long duration = (endTime > 0L ? endTime : now) - startTime; timestamps.put(jobStatus, executionGraph.getStatusTimestamp(jobStatus)); Collection<JobDetailsInfo.JobVertexDetailsInfo> jobVertexInfos = new ArrayList<>(executionGraph.getAllVertices().size()); int[] jobVerticesPerState = new int[ExecutionState.values().length]; for (AccessExecutionJobVertex accessExecutionJobVertex : executionGraph.getVerticesTopologically()) { final JobDetailsInfo.JobVertexDetailsInfo vertexDetailsInfo = createJobVertexDetailsInfo( accessExecutionJobVertex, now, executionGraph.getJobID(), metricFetcher); executionGraph.getJobID(), executionGraph.getJobName(), executionGraph.isStoppable(), executionGraph.getState(), startTime, endTime, jobVertexInfos, jobVerticesPerStateMap, executionGraph.getJsonPlan());
public static JobDetails createDetailsForJob(AccessExecutionGraph job) { JobStatus status = job.getState(); long started = job.getStatusTimestamp(JobStatus.CREATED); long finished = status.isGloballyTerminalState() ? job.getStatusTimestamp(status) : -1L; int[] countsPerStatus = new int[ExecutionState.values().length]; long lastChanged = 0; int numTotalTasks = 0; for (AccessExecutionJobVertex ejv : job.getVerticesTopologically()) { AccessExecutionVertex[] vertices = ejv.getTaskVertices(); numTotalTasks += vertices.length; for (AccessExecutionVertex vertex : vertices) { ExecutionState state = vertex.getExecutionState(); countsPerStatus[state.ordinal()]++; lastChanged = Math.max(lastChanged, vertex.getStateTimestamp(state)); } } lastChanged = Math.max(lastChanged, finished); return new JobDetails(job.getJobID(), job.getJobName(), started, finished, status, lastChanged, countsPerStatus, numTotalTasks); }
@Override public Collection<ArchivedJson> archiveJsonWithPath(AccessExecutionGraph graph) throws IOException { List<ArchivedJson> archive = new ArrayList<>(); for (AccessExecutionJobVertex task : graph.getAllVertices().values()) { String json = createVertexDetailsByTaskManagerJson(task, graph.getJobID().toString(), null); String path = JOB_VERTEX_TASKMANAGERS_REST_PATH .replace(":jobid", graph.getJobID().toString()) .replace(":vertexid", task.getJobVertexId().toString()); archive.add(new ArchivedJson(path, json)); } return archive; } }
/** * Creates the {@link JobResult} from the given {@link AccessExecutionGraph} which * must be in a globally terminal state. * * @param accessExecutionGraph to create the JobResult from * @return JobResult of the given AccessExecutionGraph */ public static JobResult createFrom(AccessExecutionGraph accessExecutionGraph) { final JobID jobId = accessExecutionGraph.getJobID(); final JobStatus jobStatus = accessExecutionGraph.getState(); checkArgument( jobStatus.isGloballyTerminalState(), "The job " + accessExecutionGraph.getJobName() + '(' + jobId + ") is not in a globally " + "terminal state. It is in state " + jobStatus + '.'); final JobResult.Builder builder = new JobResult.Builder(); builder.jobId(jobId); final long netRuntime = accessExecutionGraph.getStatusTimestamp(jobStatus) - accessExecutionGraph.getStatusTimestamp(JobStatus.CREATED); // guard against clock changes final long guardedNetRuntime = Math.max(netRuntime, 0L); builder.netRuntime(guardedNetRuntime); builder.accumulatorResults(accessExecutionGraph.getAccumulatorsSerialized()); if (jobStatus != JobStatus.FINISHED) { final ErrorInfo errorInfo = accessExecutionGraph.getFailureInfo(); if (errorInfo != null) { builder.serializedThrowable(errorInfo.getException()); } } return builder.build(); }
gen.writeStringField("jid", graph.getJobID().toString()); gen.writeStringField("name", graph.getJobName()); final ArchivedExecutionConfig summary = graph.getArchivedExecutionConfig();
@Override public Collection<ArchivedJson> archiveJsonWithPath(AccessExecutionGraph graph) throws IOException { String json = createCheckpointStatsJson(graph); String path = CHECKPOINT_STATS_REST_PATH .replace(":jobid", graph.getJobID().toString()); return Collections.singletonList(new ArchivedJson(path, json)); } }
private static JobVerticesInfo createJobVerticesInfo(AccessExecutionGraph executionGraph, @Nullable MetricFetcher<?> metricFetcher) { Collection<JobVerticesInfo.JobVertex> jobVertices = new ArrayList<>(executionGraph.getAllVertices().size()); Collection<JobVerticesInfo.JobOperator> jobOperators = new ArrayList<>(); metricFetcher.update(); for (AccessExecutionJobVertex accessExecutionJobVertex : executionGraph.getVerticesTopologically()) { final JobVerticesInfo.JobVertex jobVertex = createJobVertex( accessExecutionJobVertex, executionGraph.getJobID(), metricFetcher, jobOperators); jobVertices.add(jobVertex); } return new JobVerticesInfo(jobVertices, jobOperators); }
private static JobAccumulatorsInfo createJobAccumulatorsInfo(AccessExecutionGraph graph, boolean includeSerializedValue) { StringifiedAccumulatorResult[] stringifiedAccs = graph.getAccumulatorResultsStringified(); List<JobAccumulatorsInfo.UserTaskAccumulator> userTaskAccumulators = new ArrayList<>(stringifiedAccs.length); for (StringifiedAccumulatorResult acc : stringifiedAccs) { userTaskAccumulators.add( new JobAccumulatorsInfo.UserTaskAccumulator( acc.getName(), acc.getType(), acc.getValue())); } JobAccumulatorsInfo accumulatorsInfo; if (includeSerializedValue) { Map<String, SerializedValue<OptionalFailure<Object>>> serializedUserTaskAccumulators = graph.getAccumulatorsSerialized(); accumulatorsInfo = new JobAccumulatorsInfo(Collections.emptyList(), userTaskAccumulators, serializedUserTaskAccumulators); } else { accumulatorsInfo = new JobAccumulatorsInfo(Collections.emptyList(), userTaskAccumulators, Collections.emptyMap()); } return accumulatorsInfo; } }
@Override public Collection<ArchivedJson> archiveJsonWithPath(AccessExecutionGraph graph) throws IOException { String path = JOB_PLAN_REST_PATH .replace(":jobid", graph.getJobID().toString()); String json = graph.getJsonPlan(); return Collections.singletonList(new ArchivedJson(path, json)); } }
private static JobExceptionsInfo createJobExceptionsInfo(AccessExecutionGraph executionGraph) { ErrorInfo rootException = executionGraph.getFailureInfo(); String rootExceptionMessage = null; Long rootTimestamp = null; for (AccessExecutionVertex task : executionGraph.getAllExecutionVertices()) { String t = task.getFailureCauseAsString(); if (t != null && !t.equals(ExceptionUtils.STRINGIFIED_NULL_EXCEPTION)) {
AccessExecutionGraph cached = cache.get(jid); if (cached != null) { if (cached.getState() == JobStatus.SUSPENDED) { cache.remove(jid); } else {
private static JobExceptionsInfo createJobExceptionsInfo(AccessExecutionGraph executionGraph) { ErrorInfo rootException = executionGraph.getFailureInfo(); String rootExceptionMessage = null; Long rootTimestamp = null; int numExceptionsSofar = 0; boolean truncated = false; for (AccessExecutionJobVertex jobVertex : executionGraph.getVerticesTopologically()) { for (AccessExecutionVertex executionVertex : jobVertex.getTaskVertices()) {
String rootException = graph.getFailureCauseAsString(); if (rootException != null && !rootException.equals(ExceptionUtils.STRINGIFIED_NULL_EXCEPTION)) { gen.writeStringField("root-exception", rootException); boolean truncated = false; for (AccessExecutionVertex task : graph.getAllExecutionVertices()) { String t = task.getFailureCauseAsString(); if (t != null && !t.equals(ExceptionUtils.STRINGIFIED_NULL_EXCEPTION)) {
resourceId2taskManagerInfo.put(t.getResourceId().toString(), t); for (AccessExecutionJobVertex vertex : executionGraph.getVerticesTopologically()) { for (AccessExecutionVertex task : vertex.getTaskVertices()) { TaskManagerLocation taskManagerLocation = task.getCurrentAssignedResourceLocation();
public static String createJobAccumulatorsJson(AccessExecutionGraph graph) throws IOException { StringWriter writer = new StringWriter(); JsonGenerator gen = JsonFactory.jacksonFactory.createGenerator(writer); StringifiedAccumulatorResult[] allAccumulators = graph.getAccumulatorResultsStringified(); gen.writeStartObject(); gen.writeArrayFieldStart("job-accumulators"); // empty for now gen.writeEndArray(); gen.writeArrayFieldStart("user-task-accumulators"); for (StringifiedAccumulatorResult acc : allAccumulators) { gen.writeStartObject(); gen.writeStringField("name", acc.getName()); gen.writeStringField("type", acc.getType()); gen.writeStringField("value", acc.getValue()); gen.writeEndObject(); } gen.writeEndArray(); gen.writeEndObject(); gen.close(); return writer.toString(); } }
private static JobDetailsInfo createJobDetailsInfo(AccessExecutionGraph executionGraph, @Nullable MetricFetcher<?> metricFetcher) { final long now = System.currentTimeMillis(); final long startTime = executionGraph.getStatusTimestamp(JobStatus.CREATED); final long endTime = executionGraph.getState().isGloballyTerminalState() ? executionGraph.getStatusTimestamp(executionGraph.getState()) : -1L; final long duration = (endTime > 0L ? endTime : now) - startTime; timestamps.put(jobStatus, executionGraph.getStatusTimestamp(jobStatus)); Collection<JobDetailsInfo.JobVertexDetailsInfo> jobVertexInfos = new ArrayList<>(executionGraph.getAllVertices().size()); int[] jobVerticesPerState = new int[ExecutionState.values().length]; for (AccessExecutionJobVertex accessExecutionJobVertex : executionGraph.getVerticesTopologically()) { final JobDetailsInfo.JobVertexDetailsInfo vertexDetailsInfo = createJobVertexDetailsInfo( accessExecutionJobVertex, now, executionGraph.getJobID(), metricFetcher); executionGraph.getJobID(), executionGraph.getJobName(), executionGraph.isStoppable(), executionGraph.getState(), startTime, endTime, jobVertexInfos, jobVerticesPerStateMap, executionGraph.getJsonPlan());
public static JobDetails createDetailsForJob(AccessExecutionGraph job) { JobStatus status = job.getState(); long started = job.getStatusTimestamp(JobStatus.CREATED); long finished = status.isGloballyTerminalState() ? job.getStatusTimestamp(status) : -1L; long duration = (finished >= 0L ? finished : System.currentTimeMillis()) - started; int[] countsPerStatus = new int[ExecutionState.values().length]; long lastChanged = 0; int numTotalTasks = 0; for (AccessExecutionJobVertex ejv : job.getVerticesTopologically()) { AccessExecutionVertex[] vertices = ejv.getTaskVertices(); numTotalTasks += vertices.length; for (AccessExecutionVertex vertex : vertices) { ExecutionState state = vertex.getExecutionState(); countsPerStatus[state.ordinal()]++; lastChanged = Math.max(lastChanged, vertex.getStateTimestamp(state)); } } lastChanged = Math.max(lastChanged, finished); return new JobDetails( job.getJobID(), job.getJobName(), started, finished, duration, status, lastChanged, countsPerStatus, numTotalTasks); }
@Override public Collection<ArchivedJson> archiveJsonWithPath(AccessExecutionGraph graph) throws IOException { Collection<? extends AccessExecutionJobVertex> vertices = graph.getAllVertices().values(); List<ArchivedJson> archive = new ArrayList<>(vertices.size()); for (AccessExecutionJobVertex task : vertices) { ResponseBody json = createJobVertexDetailsInfo(task, graph.getJobID(), null); String path = getMessageHeaders().getTargetRestEndpointURL() .replace(':' + JobIDPathParameter.KEY, graph.getJobID().toString()) .replace(':' + JobVertexIdPathParameter.KEY, task.getJobVertexId().toString()); archive.add(new ArchivedJson(path, json)); } return archive; }
final JobID jobId = accessExecutionGraph.getJobID(); final JobStatus jobStatus = accessExecutionGraph.getState(); "The job " + accessExecutionGraph.getJobName() + '(' + jobId + ") is not in a globally " + "terminal state. It is in state " + jobStatus + '.'); builder.jobId(jobId); builder.applicationStatus(ApplicationStatus.fromJobStatus(accessExecutionGraph.getState())); final long netRuntime = accessExecutionGraph.getStatusTimestamp(jobStatus) - accessExecutionGraph.getStatusTimestamp(JobStatus.CREATED); builder.accumulatorResults(accessExecutionGraph.getAccumulatorsSerialized()); final ErrorInfo errorInfo = accessExecutionGraph.getFailureInfo();