TaskAttemptID id = attempt.getAttemptId(); assertNotNull(id); long startTime = attempt.getStartTime(); assertTrue("Invalid Start time", startTime > 0); long finishTime = attempt.getFinishTime(); assertTrue("Task FINISH_TIME is < START_TIME in history file", startTime < finishTime); TaskType type = attempt.getTaskType(); assertTrue("Unknown Task type \"" + type + "\" is seen in " + "history file for task attempt " + id, String status = attempt.getTaskStatus(); assertTrue("Unexpected TASK_STATUS \"" + status + "\" is seen in" + " history file for task attempt " + id, long shuffleFinishTime = attempt.getShuffleFinishTime(); assertTrue(startTime < shuffleFinishTime); long sortFinishTime = attempt.getSortFinishTime(); assertTrue(shuffleFinishTime < sortFinishTime); long mapFinishTime = attempt.getMapFinishTime(); assertTrue(startTime < mapFinishTime); String hostname = attempt.getHostname(); Matcher m = hostNamePattern.matcher(hostname);
if (taskType.equals(task.getTaskType())){ taskList.setLength(0); taskList.append(attempt.getAttemptId()).append("\t"); taskList.append(StringUtils.getFormattedTimeWithDiff(dateFormat, attempt.getStartTime(), 0)).append("\t"); if (TaskType.REDUCE.equals(taskType)) { taskList.append(StringUtils.getFormattedTimeWithDiff(dateFormat, attempt.getShuffleFinishTime(), attempt.getStartTime())); taskList.append("\t"); taskList.append(StringUtils.getFormattedTimeWithDiff(dateFormat, attempt.getSortFinishTime(), attempt.getShuffleFinishTime())); attempt.getFinishTime(), attempt.getStartTime())); taskList.append("\t"); taskList.append(attempt.getHostname()).append("\t"); taskList.append(attempt.getError()); String taskLogsUrl = getTaskLogsUrl( WebAppUtils.getHttpSchemePrefix(fs.getConf()), attempt);
Integer.toString(attemptsMap.get(attempt).getHttpPort()), attempt.toString()) + "&filter=" + TaskLog.LogName.STDOUT; validateViewJob(stdoutURL, "GET"); Integer.toString(attemptsMap.get(attempt).getHttpPort()), attempt.toString()) + "&filter=" + TaskLog.LogName.STDERR; validateViewJob(stderrURL, "GET"); Integer.toString(attemptsMap.get(attempt).getHttpPort()), attempt.toString()) + "&filter=" + TaskLog.LogName.STDOUT;; Integer.toString(attemptsMap.get(attempt).getHttpPort()), attempt.toString()) + "&filter=" + TaskLog.LogName.STDERR;
long startTime = attempt.getStartTime(); long finishTime = attempt.getFinishTime(); if (attempt.getTaskType().equals(TaskType.MAP)) { if (mapStarted== 0 || mapStarted > startTime) { mapStarted = startTime; if (attempt.getTaskStatus().equals (TaskStatus.State.FAILED.toString())) { numFailedMaps++; } else if (attempt.getTaskStatus().equals (TaskStatus.State.KILLED.toString())) { numKilledMaps++; } else if (attempt.getTaskType().equals(TaskType.REDUCE)) { if (reduceStarted==0||reduceStarted > startTime) { reduceStarted = startTime; if (attempt.getTaskStatus().equals (TaskStatus.State.FAILED.toString())) { numFailedReduces++; } else if (attempt.getTaskStatus().equals (TaskStatus.State.KILLED.toString())) { numKilledReduces++; } else if (attempt.getTaskType().equals(TaskType.JOB_CLEANUP)) { if (cleanupStarted==0||cleanupStarted > startTime) { cleanupStarted = startTime; if (attempt.getTaskStatus().equals
long startTime = attempt.getStartTime(); long finishTime = attempt.getFinishTime(); if (attempt.getTaskType().equals(TaskType.MAP)) { if (mapStarted== 0 || mapStarted > startTime) { mapStarted = startTime; if (attempt.getTaskStatus().equals (TaskStatus.State.FAILED.toString())) { numFailedMaps++; } else if (attempt.getTaskStatus().equals (TaskStatus.State.KILLED.toString())) { numKilledMaps++; } else if (attempt.getTaskType().equals(TaskType.REDUCE)) { if (reduceStarted==0||reduceStarted > startTime) { reduceStarted = startTime; if (attempt.getTaskStatus().equals (TaskStatus.State.FAILED.toString())) { numFailedReduces++; } else if (attempt.getTaskStatus().equals (TaskStatus.State.KILLED.toString())) { numKilledReduces++; } else if (attempt.getTaskType().equals(TaskType.JOB_CLEANUP)) { if (cleanupStarted==0||cleanupStarted > startTime) { cleanupStarted = startTime; if (attempt.getTaskStatus().equals
long startTime = attempt.getStartTime(); long finishTime = attempt.getFinishTime(); if (attempt.getTaskType().equals(TaskType.MAP)) { if (mapStarted== 0 || mapStarted > startTime) { mapStarted = startTime; if (attempt.getTaskStatus().equals (TaskStatus.State.FAILED.toString())) { numFailedMaps++; } else if (attempt.getTaskStatus().equals (TaskStatus.State.KILLED.toString())) { numKilledMaps++; } else if (attempt.getTaskType().equals(TaskType.REDUCE)) { if (reduceStarted==0||reduceStarted > startTime) { reduceStarted = startTime; if (attempt.getTaskStatus().equals (TaskStatus.State.FAILED.toString())) { numFailedReduces++; } else if (attempt.getTaskStatus().equals (TaskStatus.State.KILLED.toString())) { numKilledReduces++; } else if (attempt.getTaskType().equals(TaskType.JOB_CLEANUP)) { if (cleanupStarted==0||cleanupStarted > startTime) { cleanupStarted = startTime; if (attempt.getTaskStatus().equals
@SuppressWarnings("unchecked") public TaskAttemptStateInternal recover(TaskAttemptInfo taInfo, OutputCommitter committer, boolean recoverOutput) { ContainerId containerId = taInfo.getContainerId(); NodeId containerNodeId = NodeId.fromString( taInfo.getHostname() + ":" + taInfo.getPort()); String nodeHttpAddress = StringInterner.weakIntern(taInfo.getHostname() + ":" + taInfo.getHttpPort()); nodeHttpAddress, null, null, null); computeRackAndLocality(); launchTime = taInfo.getStartTime(); finishTime = (taInfo.getFinishTime() != -1) ? taInfo.getFinishTime() : clock.getTime(); shufflePort = taInfo.getShufflePort(); trackerName = taInfo.getHostname(); httpPort = taInfo.getHttpPort(); sendLaunchedEvents(); reportedStatus.counters = taInfo.getCounters(); reportedStatus.stateString = taInfo.getState(); reportedStatus.phase = Phase.CLEANUP; reportedStatus.mapFinishTime = taInfo.getMapFinishTime(); reportedStatus.shuffleFinishTime = taInfo.getShuffleFinishTime(); reportedStatus.sortFinishTime = taInfo.getSortFinishTime(); addDiagnosticInfo(taInfo.getError()); String recoveredState = taInfo.getTaskStatus(); if (recoverOutput && TaskAttemptState.SUCCEEDED.toString().equals(recoveredState)) {
@SuppressWarnings("unchecked") public TaskAttemptStateInternal recover(TaskAttemptInfo taInfo, OutputCommitter committer, boolean recoverOutput) { ContainerId containerId = taInfo.getContainerId(); NodeId containerNodeId = NodeId.fromString( taInfo.getHostname() + ":" + taInfo.getPort()); String nodeHttpAddress = StringInterner.weakIntern(taInfo.getHostname() + ":" + taInfo.getHttpPort()); nodeHttpAddress, null, null, null); computeRackAndLocality(); launchTime = taInfo.getStartTime(); finishTime = (taInfo.getFinishTime() != -1) ? taInfo.getFinishTime() : clock.getTime(); shufflePort = taInfo.getShufflePort(); trackerName = taInfo.getHostname(); httpPort = taInfo.getHttpPort(); sendLaunchedEvents(); reportedStatus.counters = taInfo.getCounters(); reportedStatus.stateString = taInfo.getState(); reportedStatus.phase = Phase.CLEANUP; reportedStatus.mapFinishTime = taInfo.getMapFinishTime(); reportedStatus.shuffleFinishTime = taInfo.getShuffleFinishTime(); reportedStatus.sortFinishTime = taInfo.getSortFinishTime(); addDiagnosticInfo(taInfo.getError()); String recoveredState = taInfo.getTaskStatus(); if (recoverOutput && TaskAttemptState.SUCCEEDED.toString().equals(recoveredState)) {
long startTime = attempt.getStartTime(); long finishTime = attempt.getFinishTime(); if (attempt.getTaskType().equals(TaskType.MAP)) { if (mapStarted== 0 || mapStarted > startTime) { mapStarted = startTime; if (attempt.getTaskStatus().equals (TaskStatus.State.FAILED.toString())) { numFailedMaps++; } else if (attempt.getTaskStatus().equals (TaskStatus.State.KILLED.toString())) { numKilledMaps++; } else if (attempt.getTaskType().equals(TaskType.REDUCE)) { if (reduceStarted==0||reduceStarted > startTime) { reduceStarted = startTime; if (attempt.getTaskStatus().equals (TaskStatus.State.FAILED.toString())) { numFailedReduces++; } else if (attempt.getTaskStatus().equals (TaskStatus.State.KILLED.toString())) { numKilledReduces++; } else if (attempt.getTaskType().equals(TaskType.JOB_CLEANUP)) { if (cleanupStarted==0||cleanupStarted > startTime) { cleanupStarted = startTime; if (attempt.getTaskStatus().equals
@SuppressWarnings("unchecked") public TaskAttemptStateInternal recover(TaskAttemptInfo taInfo, OutputCommitter committer, boolean recoverOutput) { ContainerId containerId = taInfo.getContainerId(); NodeId containerNodeId = ConverterUtils.toNodeId(taInfo.getHostname() + ":" + taInfo.getPort()); String nodeHttpAddress = StringInterner.weakIntern(taInfo.getHostname() + ":" + taInfo.getHttpPort()); nodeHttpAddress, null, null, null); computeRackAndLocality(); launchTime = taInfo.getStartTime(); finishTime = (taInfo.getFinishTime() != -1) ? taInfo.getFinishTime() : clock.getTime(); shufflePort = taInfo.getShufflePort(); trackerName = taInfo.getHostname(); httpPort = taInfo.getHttpPort(); sendLaunchedEvents(); reportedStatus.counters = taInfo.getCounters(); reportedStatus.stateString = taInfo.getState(); reportedStatus.phase = Phase.CLEANUP; reportedStatus.mapFinishTime = taInfo.getMapFinishTime(); reportedStatus.shuffleFinishTime = taInfo.getShuffleFinishTime(); reportedStatus.sortFinishTime = taInfo.getSortFinishTime(); addDiagnosticInfo(taInfo.getError()); String recoveredState = taInfo.getTaskStatus(); if (recoverOutput && TaskAttemptState.SUCCEEDED.toString().equals(recoveredState)) {
TaskAttemptID attemptId = attempt.getAttemptId(); TaskID tid = attemptId.getTaskID(); " obtained from " + "history file did not match the expected value", ts.getStartTime() == attempt.getStartTime()); "history file " + ts.getFinishTime() + " did not match the expected value, " + attempt.getFinishTime(), ts.getFinishTime() == attempt.getFinishTime()); "history file did not match the expected value", ttStatus.getHttpPort() == attempt.getHttpPort()); if (attempt.getTaskStatus().equals("SUCCEEDED")) { String ttHostname = jt.getNode(ttStatus.getHost()).toString(); " obtained from" + " history file did not match the expected value", ttHostname.equals(attempt.getHostname())); if (attempt.getTaskStatus().equals("SUCCEEDED")) { if (attempt.getTaskType().equals("REDUCE")) { assertTrue("SHUFFLE_FINISHED time of task attempt " + attemptId + " obtained from history file did not match the expected" + " value", ts.getShuffleFinishTime() ==
task.getAllTaskAttempts(); for (JobHistoryParser.TaskAttemptInfo attempt : attempts.values()) { if (attempt.getTaskStatus(). equals(TaskStatus.State.SUCCEEDED.toString())) { long avgFinishTime = (attempt.getFinishTime() - attempt.getStartTime()); if (attempt.getTaskType().equals(TaskType.MAP)) { mapTasks[mapIndex++] = attempt; avgMapTime += avgFinishTime; } else if (attempt.getTaskType().equals(TaskType.REDUCE)) { reduceTasks[reduceIndex++] = attempt; avgShuffleTime += (attempt.getShuffleFinishTime() - attempt.getStartTime()); avgReduceTime += (attempt.getFinishTime() - attempt.getShuffleFinishTime());
task.getAllTaskAttempts(); for (JobHistoryParser.TaskAttemptInfo attempt : attempts.values()) { if (attempt.getTaskStatus(). equals(TaskStatus.State.SUCCEEDED.toString())) { long avgFinishTime = (attempt.getFinishTime() - attempt.getStartTime()); if (attempt.getTaskType().equals(TaskType.MAP)) { mapTasks[mapIndex++] = attempt; avgMapTime += avgFinishTime; } else if (attempt.getTaskType().equals(TaskType.REDUCE)) { reduceTasks[reduceIndex++] = attempt; avgShuffleTime += (attempt.getShuffleFinishTime() - attempt.getStartTime()); avgReduceTime += (attempt.getFinishTime() - attempt.getShuffleFinishTime());
throws IOException { out.print("<tr>"); out.print("<td>" + taskAttempt.getAttemptId() + "</td>"); out.print("<td>" + StringUtils.getFormattedTimeWithDiff(dateFormat, taskAttempt.getStartTime(), 0 ) + "</td>"); if (TaskType.REDUCE.equals(type)) { out.print("<td>" + StringUtils.getFormattedTimeWithDiff(dateFormat, taskAttempt.getShuffleFinishTime(), taskAttempt.getStartTime()) + "</td>"); out.print("<td>" + StringUtils.getFormattedTimeWithDiff(dateFormat, taskAttempt.getSortFinishTime(), taskAttempt.getShuffleFinishTime()) + "</td>"); taskAttempt.getFinishTime(), taskAttempt.getStartTime()) + "</td>"); out.print("<td>" + taskAttempt.getHostname() + "</td>"); out.print("<td>" + HtmlQuoting.quoteHtmlChars(taskAttempt.getError()) + "</td>"); Counters counters = taskAttempt.getCounters(); if (counters != null) { TaskAttemptID attemptId = taskAttempt.getAttemptId(); out.print("<td>" + "<a href=\"/taskstatshistory.jsp?attemptid=" + attemptId
task.getAllTaskAttempts(); for (JobHistoryParser.TaskAttemptInfo attempt : attempts.values()) { if (attempt.getTaskStatus(). equals(TaskStatus.State.SUCCEEDED.toString())) { long avgFinishTime = (attempt.getFinishTime() - attempt.getStartTime()); if (attempt.getTaskType().equals(TaskType.MAP)) { mapTasks[mapIndex++] = attempt; avgMapTime += avgFinishTime; } else if (attempt.getTaskType().equals(TaskType.REDUCE)) { reduceTasks[reduceIndex++] = attempt; avgShuffleTime += (attempt.getShuffleFinishTime() - attempt.getStartTime()); avgReduceTime += (attempt.getFinishTime() - attempt.getShuffleFinishTime());
task.getAllTaskAttempts(); for (JobHistoryParser.TaskAttemptInfo attempt : attempts.values()) { if (attempt.getTaskStatus(). equals(TaskStatus.State.SUCCEEDED.toString())) { long avgFinishTime = (attempt.getFinishTime() - attempt.getStartTime()); if (attempt.getTaskType().equals(TaskType.MAP)) { mapTasks[mapIndex++] = attempt; avgMapTime += avgFinishTime; } else if (attempt.getTaskType().equals(TaskType.REDUCE)) { reduceTasks[reduceIndex++] = attempt; avgShuffleTime += (attempt.getShuffleFinishTime() - attempt.getStartTime()); avgReduceTime += (attempt.getFinishTime() - attempt.getShuffleFinishTime());
details.append("\nTime taken by best performing "); details.append(taskType).append(" task "); details.append(min.getAttemptId().getTaskID().toString()).append(": "); if ("map".equals(taskType)) { details.append(StringUtils.formatTimeDiff( min.getFinishTime(), min.getStartTime())); } else if ("shuffle".equals(taskType)) { details.append(StringUtils.formatTimeDiff( min.getShuffleFinishTime(), min.getStartTime())); } else { details.append(StringUtils.formatTimeDiff( min.getFinishTime(), min.getShuffleFinishTime())); for (int i = 0; i < showTasks && i < tasks.length; i++) { details.setLength(0); details.append(tasks[i].getAttemptId().getTaskID()).append(" "); if ("map".equals(taskType)) { details.append(StringUtils.formatTimeDiff( tasks[i].getFinishTime(), tasks[i].getStartTime())); } else if ("shuffle".equals(taskType)) { details.append(StringUtils.formatTimeDiff( tasks[i].getShuffleFinishTime(), tasks[i].getStartTime())); } else { details.append(StringUtils.formatTimeDiff(
if (taskType.equals(task.getTaskType())){ taskList.setLength(0); taskList.append(attempt.getAttemptId()).append("\t"); taskList.append(StringUtils.getFormattedTimeWithDiff(dateFormat, attempt.getStartTime(), 0)).append("\t"); if (TaskType.REDUCE.equals(taskType)) { taskList.append(StringUtils.getFormattedTimeWithDiff(dateFormat, attempt.getShuffleFinishTime(), attempt.getStartTime())); taskList.append("\t"); taskList.append(StringUtils.getFormattedTimeWithDiff(dateFormat, attempt.getSortFinishTime(), attempt.getShuffleFinishTime())); attempt.getFinishTime(), attempt.getStartTime())); taskList.append("\t"); taskList.append(attempt.getHostname()).append("\t"); taskList.append(attempt.getError()); String taskLogsUrl = getTaskLogsUrl( WebAppUtils.getHttpSchemePrefix(fs.getConf()), attempt);
if (taskType.equals(task.getTaskType())){ taskList.setLength(0); taskList.append(attempt.getAttemptId()).append("\t"); taskList.append(StringUtils.getFormattedTimeWithDiff(dateFormat, attempt.getStartTime(), 0)).append("\t"); if (TaskType.REDUCE.equals(taskType)) { taskList.append(StringUtils.getFormattedTimeWithDiff(dateFormat, attempt.getShuffleFinishTime(), attempt.getStartTime())); taskList.append("\t"); taskList.append(StringUtils.getFormattedTimeWithDiff(dateFormat, attempt.getSortFinishTime(), attempt.getShuffleFinishTime())); attempt.getFinishTime(), attempt.getStartTime())); taskList.append("\t"); taskList.append(attempt.getHostname()).append("\t"); taskList.append(attempt.getError()); String taskLogsUrl = getTaskLogsUrl( WebAppUtils.getHttpSchemePrefix(fs.getConf()), attempt);
task.getAllTaskAttempts(); for (JobHistoryParser.TaskAttemptInfo attempt : attempts.values()) { if (attempt.getTaskStatus(). equals(TaskStatus.State.SUCCEEDED.toString())) { long avgFinishTime = (attempt.getFinishTime() - attempt.getStartTime()); if (attempt.getTaskType().equals(TaskType.MAP)) { mapTasks[mapIndex++] = attempt; avgMapTime += avgFinishTime; } else if (attempt.getTaskType().equals(TaskType.REDUCE)) { reduceTasks[reduceIndex++] = attempt; avgShuffleTime += (attempt.getShuffleFinishTime() - attempt.getStartTime()); avgReduceTime += (attempt.getFinishTime() - attempt.getShuffleFinishTime());