public static SubtaskExecutionAttemptDetailsInfo create(AccessExecution execution, MutableIOMetrics ioMetrics) { final ExecutionState status = execution.getState(); final long now = System.currentTimeMillis(); final TaskManagerLocation location = execution.getAssignedResourceLocation(); final String locationString = location == null ? "(unassigned)" : location.getHostname() + ":" + location.dataPort(); long startTime = execution.getStateTimestamp(ExecutionState.DEPLOYING); if (startTime == 0) { startTime = -1; final long endTime = status.isTerminal() ? execution.getStateTimestamp(status) : -1; final long duration = startTime > 0 ? ((endTime > 0 ? endTime : now) - startTime) : -1; long[] stateTimestamps = execution.getStateTimestamps(); Map<ExecutionState, Long> stateTransitionTime = new HashMap<>(); execution.getParallelSubtaskIndex(), status, execution.getAttemptNumber(), locationString, startTime,
private static SubtaskExecutionAttemptAccumulatorsInfo createAccumulatorInfo(AccessExecution execution) { final StringifiedAccumulatorResult[] accs = execution.getUserAccumulatorsStringified(); final ArrayList<UserAccumulator> userAccumulatorList = new ArrayList<>(accs.length); for (StringifiedAccumulatorResult acc : accs) { userAccumulatorList.add(new UserAccumulator(acc.getName(), acc.getType(), acc.getValue())); } return new SubtaskExecutionAttemptAccumulatorsInfo( execution.getParallelSubtaskIndex(), execution.getAttemptNumber(), execution.getAttemptId().toString(), userAccumulatorList); } }
public static SubtaskExecutionAttemptInfo create(AccessExecution execution) { final ExecutionState status = execution.getState(); final long now = System.currentTimeMillis(); final TaskManagerLocation location = execution.getAssignedResourceLocation(); final String locationString = location == null ? "(unassigned)" : location.getHostname() + ":" + location.dataPort(); final ResourceID resourceID = location == null ? new ResourceID("(unassigned)") : location.getResourceID(); long startTime = execution.getStateTimestamp(ExecutionState.DEPLOYING); if (startTime == 0) { startTime = -1; } final long endTime = status.isTerminal() ? execution.getStateTimestamp(status) : -1; final long duration = startTime > 0 ? ((endTime > 0 ? endTime : now) - startTime) : -1; final String failureCause = execution.getFailureCauseAsString(); return new SubtaskExecutionAttemptInfo( execution.getAttemptId(), status, execution.getAttemptNumber(), locationString, startTime, endTime, duration, failureCause, resourceID ); } }
if (attempt.getState().isTerminal()) { IOMetrics ioMetrics = attempt.getIOMetrics(); if (ioMetrics != null) { // execAttempt is already finished, use final metrics stored in ExecutionGraph this.numBytesInLocal += ioMetrics.getNumBytesInLocal(); if (fetcher != null) { fetcher.update(); MetricStore.SubtaskMetricStore metrics = fetcher.getMetricStore().getSubtaskMetricStore(jobID, taskID, attempt.getParallelSubtaskIndex()); if (metrics != null) { this.numBytesInLocal += Long.valueOf(metrics.getMetric(MetricNames.IO_NUM_BYTES_IN_LOCAL, "0"));
private static JobExceptionsInfo.ExecutionExceptionInfo generateExecutionExceptionInfo(AccessExecutionJobVertex jobVertex, AccessExecutionVertex executionVertex, AccessExecution task) { String t = task != null ? task.getFailureCauseAsString() : null; if (t != null && !t.equals(ExceptionUtils.STRINGIFIED_NULL_EXCEPTION)) { TaskManagerLocation location = task.getAssignedResourceLocation(); String locationString = location != null ? location.getFQDNHostname() + ':' + location.dataPort() : "(unassigned)"; long timestamp = task.getStateTimestamp(ExecutionState.FAILED); return new JobExceptionsInfo.ExecutionExceptionInfo( t, executionVertex.getTaskNameWithSubtaskIndex(), locationString, timestamp == 0 ? -1 : timestamp, jobVertex.getJobVertexId().toString(), task.getParallelSubtaskIndex(), task.getAttemptNumber()); } else { return null; } } }
@Override protected SubtasksAllAccumulatorsInfo handleRequest(HandlerRequest<EmptyRequestBody, JobVertexMessageParameters> request, AccessExecutionJobVertex jobVertex) throws RestHandlerException { JobVertexID jobVertexId = jobVertex.getJobVertexId(); int parallelism = jobVertex.getParallelism(); final List<SubtasksAllAccumulatorsInfo.SubtaskAccumulatorsInfo> subtaskAccumulatorsInfos = new ArrayList<>(); for (AccessExecutionVertex vertex : jobVertex.getTaskVertices()) { TaskManagerLocation location = vertex.getCurrentAssignedResourceLocation(); String locationString = location == null ? "(unassigned)" : location.getHostname(); StringifiedAccumulatorResult[] accs = vertex.getCurrentExecutionAttempt().getUserAccumulatorsStringified(); List<UserAccumulator> userAccumulators = new ArrayList<>(accs.length); for (StringifiedAccumulatorResult acc : accs) { userAccumulators.add(new UserAccumulator(acc.getName(), acc.getType(), acc.getValue())); } subtaskAccumulatorsInfos.add( new SubtasksAllAccumulatorsInfo.SubtaskAccumulatorsInfo( vertex.getCurrentExecutionAttempt().getParallelSubtaskIndex(), vertex.getCurrentExecutionAttempt().getAttemptNumber(), locationString, userAccumulators )); } return new SubtasksAllAccumulatorsInfo(jobVertexId, parallelism, subtaskAccumulatorsInfos); }
if (attempt == currentAttempt.getAttemptNumber()) { return handleRequest(currentAttempt, params); else if (attempt >= 0 && attempt < currentAttempt.getAttemptNumber()) { AccessExecution exec = vertex.getPriorExecutionAttempt(attempt);
gen.writeNumberField("attempt", vertex.getCurrentExecutionAttempt().getAttemptNumber()); gen.writeStringField("host", locationString); StringifiedAccumulatorResult[] accs = vertex.getCurrentExecutionAttempt().getUserAccumulatorsStringified(); gen.writeArrayFieldStart("user-accumulators"); for (StringifiedAccumulatorResult acc : accs) {
for (AccessExecutionVertex vertex : jobVertex.getTaskVertices()) { long[] timestamps = vertex.getCurrentExecutionAttempt().getStateTimestamps(); ExecutionState status = vertex.getExecutionState();
MetricStore.ComponentMetricStore subTaskMetric = metricFetcher.getMetricStore() .getSubtaskMetricStore(jobId.toString(), ejv.getJobVertexId().toString(), vertex.getCurrentExecutionAttempt().getParallelSubtaskIndex()); counts.addIOMetrics( vertex.getCurrentExecutionAttempt(),
if (attempt.getState().isTerminal()) { IOMetrics ioMetrics = attempt.getIOMetrics(); if (ioMetrics != null) { // execAttempt is already finished, use final metrics stored in ExecutionGraph this.numBytesInLocal += ioMetrics.getNumBytesInLocal(); fetcher.update(); MetricStore.ComponentMetricStore metrics = fetcher.getMetricStore() .getSubtaskMetricStore(jobID, taskID, attempt.getParallelSubtaskIndex()); if (metrics != null) {
@Override protected SubtasksAllAccumulatorsInfo handleRequest(HandlerRequest<EmptyRequestBody, JobVertexMessageParameters> request, AccessExecutionJobVertex jobVertex) throws RestHandlerException { JobVertexID jobVertexId = jobVertex.getJobVertexId(); int parallelism = jobVertex.getParallelism(); final List<SubtasksAllAccumulatorsInfo.SubtaskAccumulatorsInfo> subtaskAccumulatorsInfos = new ArrayList<>(); for (AccessExecutionVertex vertex : jobVertex.getTaskVertices()) { TaskManagerLocation location = vertex.getCurrentAssignedResourceLocation(); String locationString = location == null ? "(unassigned)" : location.getHostname(); StringifiedAccumulatorResult[] accs = vertex.getCurrentExecutionAttempt().getUserAccumulatorsStringified(); List<UserAccumulator> userAccumulators = new ArrayList<>(accs.length); for (StringifiedAccumulatorResult acc : accs) { userAccumulators.add(new UserAccumulator(acc.getName(), acc.getType(), acc.getValue())); } subtaskAccumulatorsInfos.add( new SubtasksAllAccumulatorsInfo.SubtaskAccumulatorsInfo( vertex.getCurrentExecutionAttempt().getParallelSubtaskIndex(), vertex.getCurrentExecutionAttempt().getAttemptNumber(), locationString, userAccumulators )); } return new SubtasksAllAccumulatorsInfo(jobVertexId, parallelism, subtaskAccumulatorsInfos); }
@Override protected SubtaskExecutionAllAttemptsInfo handleRequest( HandlerRequest<EmptyRequestBody, SubtaskMessageParameters> request, AccessExecutionVertex executionVertex) throws RestHandlerException { final AccessExecution execution = executionVertex.getCurrentExecutionAttempt(); final int currentAttemptNum = execution.getAttemptNumber(); final List<SubtaskExecutionAttemptInfo> allAttempts = new ArrayList<>(); allAttempts.add(SubtaskExecutionAttemptInfo.create(execution)); if (currentAttemptNum > 0) { for (int i = currentAttemptNum - 1; i >= 0; i--) { final AccessExecution currentExecution = executionVertex.getPriorExecutionAttempt(i); if (currentExecution != null) { allAttempts.add(SubtaskExecutionAttemptInfo.create(currentExecution)); } } } return new SubtaskExecutionAllAttemptsInfo(allAttempts); } }
gen.writeNumberField("attempt", vertex.getCurrentExecutionAttempt().getAttemptNumber()); gen.writeStringField("host", locationString); StringifiedAccumulatorResult[] accs = vertex.getCurrentExecutionAttempt().getUserAccumulatorsStringified(); gen.writeArrayFieldStart("user-accumulators"); for (StringifiedAccumulatorResult acc : accs) {
for (AccessExecutionVertex vertex : jobVertex.getTaskVertices()) { long[] timestamps = vertex.getCurrentExecutionAttempt().getStateTimestamps(); ExecutionState status = vertex.getExecutionState();
JsonGenerator gen = JsonFactory.jacksonFactory.createGenerator(writer); final ExecutionState status = execAttempt.getState(); final long now = System.currentTimeMillis(); TaskManagerLocation location = execAttempt.getAssignedResourceLocation(); String locationString = location == null ? "(unassigned)" : location.getHostname(); long startTime = execAttempt.getStateTimestamp(ExecutionState.DEPLOYING); if (startTime == 0) { startTime = -1; long endTime = status.isTerminal() ? execAttempt.getStateTimestamp(status) : -1; long duration = startTime > 0 ? ((endTime > 0 ? endTime : now) - startTime) : -1; gen.writeNumberField("subtask", execAttempt.getParallelSubtaskIndex()); gen.writeStringField("status", status.name()); gen.writeNumberField("attempt", execAttempt.getAttemptNumber()); gen.writeStringField("host", locationString); gen.writeNumberField("start-time", startTime);
private static SubtaskExecutionAttemptAccumulatorsInfo createAccumulatorInfo(AccessExecution execution) { final StringifiedAccumulatorResult[] accs = execution.getUserAccumulatorsStringified(); final ArrayList<UserAccumulator> userAccumulatorList = new ArrayList<>(accs.length); for (StringifiedAccumulatorResult acc : accs) { userAccumulatorList.add(new UserAccumulator(acc.getName(), acc.getType(), acc.getValue())); } return new SubtaskExecutionAttemptAccumulatorsInfo( execution.getParallelSubtaskIndex(), execution.getAttemptNumber(), execution.getAttemptId().toString(), userAccumulatorList); } }
if (attempt.getState().isTerminal()) { IOMetrics ioMetrics = attempt.getIOMetrics(); if (ioMetrics != null) { // execAttempt is already finished, use final metrics stored in ExecutionGraph this.numBytesInLocal += ioMetrics.getNumBytesInLocal(); fetcher.update(); MetricStore.ComponentMetricStore metrics = fetcher.getMetricStore() .getSubtaskMetricStore(jobID, taskID, attempt.getParallelSubtaskIndex()); if (metrics != null) {
@Override protected SubtasksAllAccumulatorsInfo handleRequest(HandlerRequest<EmptyRequestBody, JobVertexMessageParameters> request, AccessExecutionJobVertex jobVertex) throws RestHandlerException { JobVertexID jobVertexId = jobVertex.getJobVertexId(); int parallelism = jobVertex.getParallelism(); final List<SubtasksAllAccumulatorsInfo.SubtaskAccumulatorsInfo> subtaskAccumulatorsInfos = new ArrayList<>(); for (AccessExecutionVertex vertex : jobVertex.getTaskVertices()) { TaskManagerLocation location = vertex.getCurrentAssignedResourceLocation(); String locationString = location == null ? "(unassigned)" : location.getHostname(); StringifiedAccumulatorResult[] accs = vertex.getCurrentExecutionAttempt().getUserAccumulatorsStringified(); List<UserAccumulator> userAccumulators = new ArrayList<>(accs.length); for (StringifiedAccumulatorResult acc : accs) { userAccumulators.add(new UserAccumulator(acc.getName(), acc.getType(), acc.getValue())); } subtaskAccumulatorsInfos.add( new SubtasksAllAccumulatorsInfo.SubtaskAccumulatorsInfo( vertex.getCurrentExecutionAttempt().getParallelSubtaskIndex(), vertex.getCurrentExecutionAttempt().getAttemptNumber(), locationString, userAccumulators )); } return new SubtasksAllAccumulatorsInfo(jobVertexId, parallelism, subtaskAccumulatorsInfos); }
@Override public Collection<ArchivedJson> archiveJsonWithPath(AccessExecutionGraph graph) throws IOException { List<ArchivedJson> archive = new ArrayList<>(); for (AccessExecutionJobVertex task : graph.getAllVertices().values()) { for (AccessExecutionVertex subtask : task.getTaskVertices()) { String curAttemptJson = createAttemptAccumulatorsJson(subtask.getCurrentExecutionAttempt()); String curAttemptPath = SUBTASK_ATTEMPT_ACCUMULATORS_REST_PATH .replace(":jobid", graph.getJobID().toString()) .replace(":vertexid", task.getJobVertexId().toString()) .replace(":subtasknum", String.valueOf(subtask.getParallelSubtaskIndex())) .replace(":attempt", String.valueOf(subtask.getCurrentExecutionAttempt().getAttemptNumber())); archive.add(new ArchivedJson(curAttemptPath, curAttemptJson)); for (int x = 0; x < subtask.getCurrentExecutionAttempt().getAttemptNumber(); x++) { AccessExecution attempt = subtask.getPriorExecutionAttempt(x); String json = createAttemptAccumulatorsJson(attempt); String path = SUBTASK_ATTEMPT_ACCUMULATORS_REST_PATH .replace(":jobid", graph.getJobID().toString()) .replace(":vertexid", task.getJobVertexId().toString()) .replace(":subtasknum", String.valueOf(subtask.getParallelSubtaskIndex())) .replace(":attempt", String.valueOf(attempt.getAttemptNumber())); archive.add(new ArchivedJson(path, json)); } } } return archive; } }