int num = 0; for (AccessExecutionVertex vertex : jobVertex.getTaskVertices()) { final ExecutionState status = vertex.getExecutionState(); TaskManagerLocation location = vertex.getCurrentAssignedResourceLocation(); String locationString = location == null ? "(unassigned)" : location.getHostname() + ":" + location.dataPort(); long startTime = vertex.getStateTimestamp(ExecutionState.DEPLOYING); if (startTime == 0) { startTime = -1; long endTime = status.isTerminal() ? vertex.getStateTimestamp(status) : -1; long duration = startTime > 0 ? ((endTime > 0 ? endTime : now) - startTime) : -1; gen.writeNumberField("subtask", num); gen.writeStringField("status", status.name()); gen.writeNumberField("attempt", vertex.getCurrentExecutionAttempt().getAttemptNumber()); gen.writeStringField("host", locationString); gen.writeNumberField("start-time", startTime); vertex.getCurrentExecutionAttempt(), fetcher, jobID,
boolean truncated = false; for (AccessExecutionVertex task : executionGraph.getAllExecutionVertices()) { String t = task.getFailureCauseAsString(); if (t != null && !t.equals(ExceptionUtils.STRINGIFIED_NULL_EXCEPTION)) { if (taskExceptionList.size() >= MAX_NUMBER_EXCEPTION_TO_REPORT) { TaskManagerLocation location = task.getCurrentAssignedResourceLocation(); String locationString = location != null ? location.getFQDNHostname() + ':' + location.dataPort() : "(unassigned)"; long timestamp = task.getStateTimestamp(ExecutionState.FAILED); taskExceptionList.add(new JobExceptionsInfo.ExecutionExceptionInfo( t, task.getTaskNameWithSubtaskIndex(), locationString, timestamp == 0 ? -1 : timestamp));
@Override public Collection<ArchivedJson> archiveJsonWithPath(AccessExecutionGraph graph) throws IOException { List<ArchivedJson> archive = new ArrayList<>(); for (AccessExecutionJobVertex task : graph.getAllVertices().values()) { for (AccessExecutionVertex subtask : task.getTaskVertices()) { String curAttemptJson = createAttemptAccumulatorsJson(subtask.getCurrentExecutionAttempt()); String curAttemptPath = SUBTASK_ATTEMPT_ACCUMULATORS_REST_PATH .replace(":jobid", graph.getJobID().toString()) .replace(":vertexid", task.getJobVertexId().toString()) .replace(":subtasknum", String.valueOf(subtask.getParallelSubtaskIndex())) .replace(":attempt", String.valueOf(subtask.getCurrentExecutionAttempt().getAttemptNumber())); archive.add(new ArchivedJson(curAttemptPath, curAttemptJson)); for (int x = 0; x < subtask.getCurrentExecutionAttempt().getAttemptNumber(); x++) { AccessExecution attempt = subtask.getPriorExecutionAttempt(x); String json = createAttemptAccumulatorsJson(attempt); String path = SUBTASK_ATTEMPT_ACCUMULATORS_REST_PATH .replace(":jobid", graph.getJobID().toString()) .replace(":vertexid", task.getJobVertexId().toString()) .replace(":subtasknum", String.valueOf(subtask.getParallelSubtaskIndex())) .replace(":attempt", String.valueOf(attempt.getAttemptNumber())); archive.add(new ArchivedJson(path, json)); } } } return archive; } }
@Override protected SubtaskExecutionAllAttemptsInfo handleRequest( HandlerRequest<EmptyRequestBody, SubtaskMessageParameters> request, AccessExecutionVertex executionVertex) throws RestHandlerException { final AccessExecution execution = executionVertex.getCurrentExecutionAttempt(); final int currentAttemptNum = execution.getAttemptNumber(); final List<SubtaskExecutionAttemptInfo> allAttempts = new ArrayList<>(); allAttempts.add(SubtaskExecutionAttemptInfo.create(execution)); if (currentAttemptNum > 0) { for (int i = currentAttemptNum - 1; i >= 0; i--) { final AccessExecution currentExecution = executionVertex.getPriorExecutionAttempt(i); if (currentExecution != null) { allAttempts.add(SubtaskExecutionAttemptInfo.create(currentExecution)); } } } return new SubtaskExecutionAllAttemptsInfo(allAttempts); } }
String t = task.getFailureCauseAsString(); if (t != null && !t.equals(ExceptionUtils.STRINGIFIED_NULL_EXCEPTION)) { if (numExceptionsSoFar >= MAX_NUMBER_EXCEPTION_TO_REPORT) { TaskManagerLocation location = task.getCurrentAssignedResourceLocation(); String locationString = location != null ? location.getFQDNHostname() + ':' + location.dataPort() : "(unassigned)"; gen.writeStringField("task", task.getTaskNameWithSubtaskIndex()); gen.writeStringField("location", locationString); gen.writeEndObject();
for (AccessExecutionVertex vertex : jobVertex.getTaskVertices()) { TaskManagerLocation location = vertex.getCurrentAssignedResourceLocation(); String locationString = location == null ? "(unassigned)" : location.getHostname(); gen.writeNumberField("attempt", vertex.getCurrentExecutionAttempt().getAttemptNumber()); gen.writeStringField("host", locationString); StringifiedAccumulatorResult[] accs = vertex.getCurrentExecutionAttempt().getUserAccumulatorsStringified(); gen.writeArrayFieldStart("user-accumulators"); for (StringifiedAccumulatorResult acc : accs) {
final ExecutionState state = vertex.getExecutionState(); tasksPerState[state.ordinal()]++; long started = vertex.getStateTimestamp(ExecutionState.DEPLOYING); if (started > 0) { startTime = Math.min(startTime, started); endTime = Math.max(endTime, vertex.getStateTimestamp(state)); vertex.getCurrentExecutionAttempt(), fetcher, graph.getJobID().toString(),
for (AccessExecutionVertex vertex : jobVertex.getTaskVertices()) { long[] timestamps = vertex.getCurrentExecutionAttempt().getStateTimestamps(); ExecutionState status = vertex.getExecutionState(); gen.writeNumberField("subtask", num++); TaskManagerLocation location = vertex.getCurrentAssignedResourceLocation(); String locationString = location == null ? "(unassigned)" : location.getHostname(); gen.writeStringField("host", locationString);
@Override protected R handleRequest(HandlerRequest<EmptyRequestBody, M> request, AccessExecutionVertex executionVertex) throws RestHandlerException { final Integer attemptNumber = request.getPathParameter(SubtaskAttemptPathParameter.class); final AccessExecution currentAttempt = executionVertex.getCurrentExecutionAttempt(); if (attemptNumber == currentAttempt.getAttemptNumber()) { return handleRequest(request, currentAttempt); } else if (attemptNumber >= 0 && attemptNumber < currentAttempt.getAttemptNumber()) { final AccessExecution execution = executionVertex.getPriorExecutionAttempt(attemptNumber); if (execution != null) { return handleRequest(request, execution); } else { throw new RestHandlerException("Attempt " + attemptNumber + " not found in subtask " + executionVertex.getTaskNameWithSubtaskIndex(), HttpResponseStatus.NOT_FOUND); } } else { throw new RestHandlerException("Invalid attempt num " + attemptNumber, HttpResponseStatus.NOT_FOUND); } }
public static JobDetails createDetailsForJob(AccessExecutionGraph job) { JobStatus status = job.getState(); long started = job.getStatusTimestamp(JobStatus.CREATED); long finished = status.isGloballyTerminalState() ? job.getStatusTimestamp(status) : -1L; long duration = (finished >= 0L ? finished : System.currentTimeMillis()) - started; int[] countsPerStatus = new int[ExecutionState.values().length]; long lastChanged = 0; int numTotalTasks = 0; for (AccessExecutionJobVertex ejv : job.getVerticesTopologically()) { AccessExecutionVertex[] vertices = ejv.getTaskVertices(); numTotalTasks += vertices.length; for (AccessExecutionVertex vertex : vertices) { ExecutionState state = vertex.getExecutionState(); countsPerStatus[state.ordinal()]++; lastChanged = Math.max(lastChanged, vertex.getStateTimestamp(state)); } } lastChanged = Math.max(lastChanged, finished); return new JobDetails( job.getJobID(), job.getJobName(), started, finished, duration, status, lastChanged, countsPerStatus, numTotalTasks); }
@Override public CompletableFuture<String> handleRequest(AccessExecutionVertex vertex, Map<String, String> params) { return handleRequest(vertex.getCurrentExecutionAttempt(), params); } }
TaskManagerLocation taskManagerLocation = task.getCurrentAssignedResourceLocation(); ResourceID tmId = taskManagerLocation.getResourceID(); TaskManagerInfo taskManagerInfo = resourceId2taskManagerInfo.get(tmId.toString());
private static JobExceptionsInfo.ExecutionExceptionInfo generateExecutionExceptionInfo(AccessExecutionJobVertex jobVertex, AccessExecutionVertex executionVertex, AccessExecution task) { String t = task != null ? task.getFailureCauseAsString() : null; if (t != null && !t.equals(ExceptionUtils.STRINGIFIED_NULL_EXCEPTION)) { TaskManagerLocation location = task.getAssignedResourceLocation(); String locationString = location != null ? location.getFQDNHostname() + ':' + location.dataPort() : "(unassigned)"; long timestamp = task.getStateTimestamp(ExecutionState.FAILED); return new JobExceptionsInfo.ExecutionExceptionInfo( t, executionVertex.getTaskNameWithSubtaskIndex(), locationString, timestamp == 0 ? -1 : timestamp, jobVertex.getJobVertexId().toString(), task.getParallelSubtaskIndex(), task.getAttemptNumber()); } else { return null; } } }
final ExecutionState state = vertex.getExecutionState(); tasksPerState[state.ordinal()]++; long started = vertex.getStateTimestamp(ExecutionState.DEPLOYING); if (started > 0L) { startTime = Math.min(startTime, started); endTime = Math.max(endTime, vertex.getStateTimestamp(state)); vertex.getCurrentExecutionAttempt(), metricFetcher, jobId.toString(),
for (AccessExecutionVertex vertex : jobVertex.getTaskVertices()) { long[] timestamps = vertex.getCurrentExecutionAttempt().getStateTimestamps(); ExecutionState status = vertex.getExecutionState(); long duration = start >= 0 ? end - start : -1L; TaskManagerLocation location = vertex.getCurrentAssignedResourceLocation(); String locationString = location == null ? "(unassigned)" : location.getHostname();
@Override protected R handleRequest(HandlerRequest<EmptyRequestBody, M> request, AccessExecutionVertex executionVertex) throws RestHandlerException { final Integer attemptNumber = request.getPathParameter(SubtaskAttemptPathParameter.class); final AccessExecution currentAttempt = executionVertex.getCurrentExecutionAttempt(); if (attemptNumber == currentAttempt.getAttemptNumber()) { return handleRequest(request, currentAttempt); } else if (attemptNumber >= 0 && attemptNumber < currentAttempt.getAttemptNumber()) { final AccessExecution execution = executionVertex.getPriorExecutionAttempt(attemptNumber); if (execution != null) { return handleRequest(request, execution); } else { throw new RestHandlerException("Attempt " + attemptNumber + " not found in subtask " + executionVertex.getTaskNameWithSubtaskIndex(), HttpResponseStatus.NOT_FOUND); } } else { throw new RestHandlerException("Invalid attempt num " + attemptNumber, HttpResponseStatus.NOT_FOUND); } }
public static JobDetails createDetailsForJob(AccessExecutionGraph job) { JobStatus status = job.getState(); long started = job.getStatusTimestamp(JobStatus.CREATED); long finished = status.isGloballyTerminalState() ? job.getStatusTimestamp(status) : -1L; int[] countsPerStatus = new int[ExecutionState.values().length]; long lastChanged = 0; int numTotalTasks = 0; for (AccessExecutionJobVertex ejv : job.getVerticesTopologically()) { AccessExecutionVertex[] vertices = ejv.getTaskVertices(); numTotalTasks += vertices.length; for (AccessExecutionVertex vertex : vertices) { ExecutionState state = vertex.getExecutionState(); countsPerStatus[state.ordinal()]++; lastChanged = Math.max(lastChanged, vertex.getStateTimestamp(state)); } } lastChanged = Math.max(lastChanged, finished); return new JobDetails(job.getJobID(), job.getJobName(), started, finished, status, lastChanged, countsPerStatus, numTotalTasks); }
@Override protected SubtasksAllAccumulatorsInfo handleRequest(HandlerRequest<EmptyRequestBody, JobVertexMessageParameters> request, AccessExecutionJobVertex jobVertex) throws RestHandlerException { JobVertexID jobVertexId = jobVertex.getJobVertexId(); int parallelism = jobVertex.getParallelism(); final List<SubtasksAllAccumulatorsInfo.SubtaskAccumulatorsInfo> subtaskAccumulatorsInfos = new ArrayList<>(); for (AccessExecutionVertex vertex : jobVertex.getTaskVertices()) { TaskManagerLocation location = vertex.getCurrentAssignedResourceLocation(); String locationString = location == null ? "(unassigned)" : location.getHostname(); StringifiedAccumulatorResult[] accs = vertex.getCurrentExecutionAttempt().getUserAccumulatorsStringified(); List<UserAccumulator> userAccumulators = new ArrayList<>(accs.length); for (StringifiedAccumulatorResult acc : accs) { userAccumulators.add(new UserAccumulator(acc.getName(), acc.getType(), acc.getValue())); } subtaskAccumulatorsInfos.add( new SubtasksAllAccumulatorsInfo.SubtaskAccumulatorsInfo( vertex.getCurrentExecutionAttempt().getParallelSubtaskIndex(), vertex.getCurrentExecutionAttempt().getAttemptNumber(), locationString, userAccumulators )); } return new SubtasksAllAccumulatorsInfo(jobVertexId, parallelism, subtaskAccumulatorsInfos); }
final AccessExecution currentAttempt = vertex.getCurrentExecutionAttempt(); if (attempt == currentAttempt.getAttemptNumber()) { return handleRequest(currentAttempt, params); AccessExecution exec = vertex.getPriorExecutionAttempt(attempt);
@Override public String handleRequest(AccessExecutionVertex vertex, Map<String, String> params) throws Exception { return handleRequest(vertex.getCurrentExecutionAttempt(), params); } }