/** Apply the filter (status) on the parsed job and generate summary */ public FilteredJob(JobInfo job, String status) { filter = status; Map<TaskID, JobHistoryParser.TaskInfo> tasks = job.getAllTasks(); for (JobHistoryParser.TaskInfo task : tasks.values()) { Map<TaskAttemptID, JobHistoryParser.TaskAttemptInfo> attempts = task.getAllTaskAttempts(); for (JobHistoryParser.TaskAttemptInfo attempt : attempts.values()) { if (attempt.getTaskStatus().equals(status)) { String hostname = attempt.getHostname(); TaskID id = attempt.getAttemptId().getTaskID(); Set<TaskID> set = badNodesToFilteredTasks.get(hostname); if (set == null) { set = new TreeSet<TaskID>(); set.add(id); badNodesToFilteredTasks.put(hostname, set); }else{ set.add(id); } } } } } }
/** Apply the filter (status) on the parsed job and generate summary */ public FilteredJob(JobInfo job, String status) { filter = status; Map<TaskID, JobHistoryParser.TaskInfo> tasks = job.getAllTasks(); for (JobHistoryParser.TaskInfo task : tasks.values()) { Map<TaskAttemptID, JobHistoryParser.TaskAttemptInfo> attempts = task.getAllTaskAttempts(); for (JobHistoryParser.TaskAttemptInfo attempt : attempts.values()) { if (attempt.getTaskStatus().equals(status)) { String hostname = attempt.getHostname(); TaskID id = attempt.getAttemptId().getTaskID(); Set<TaskID> set = badNodesToFilteredTasks.get(hostname); if (set == null) { set = new TreeSet<TaskID>(); set.add(id); badNodesToFilteredTasks.put(hostname, set); }else{ set.add(id); } } } } } }
/** Apply the filter (status) on the parsed job and generate summary */ public FilteredJob(JobInfo job, String status) { filter = status; Map<TaskID, JobHistoryParser.TaskInfo> tasks = job.getAllTasks(); for (JobHistoryParser.TaskInfo task : tasks.values()) { Map<TaskAttemptID, JobHistoryParser.TaskAttemptInfo> attempts = task.getAllTaskAttempts(); for (JobHistoryParser.TaskAttemptInfo attempt : attempts.values()) { if (attempt.getTaskStatus().equals(status)) { String hostname = attempt.getHostname(); TaskID id = attempt.getAttemptId().getTaskID(); Set<TaskID> set = badNodesToFilteredTasks.get(hostname); if (set == null) { set = new TreeSet<TaskID>(); set.add(id); badNodesToFilteredTasks.put(hostname, set); }else{ set.add(id); } } } } } }
if (attempt.getTaskStatus().equals (TaskStatus.State.FAILED.toString())) { numFailedMaps++; } else if (attempt.getTaskStatus().equals (TaskStatus.State.KILLED.toString())) { numKilledMaps++; if (attempt.getTaskStatus().equals (TaskStatus.State.FAILED.toString())) { numFailedReduces++; } else if (attempt.getTaskStatus().equals (TaskStatus.State.KILLED.toString())) { numKilledReduces++; if (attempt.getTaskStatus().equals (TaskStatus.State.SUCCEEDED.toString())) { numFinishedCleanups++; } else if (attempt.getTaskStatus().equals (TaskStatus.State.FAILED.toString())) { numFailedCleanups++; } else if (attempt.getTaskStatus().equals (TaskStatus.State.KILLED.toString())) { numKilledCleanups++; if (attempt.getTaskStatus().equals (TaskStatus.State.SUCCEEDED.toString())) { numFinishedSetups++; } else if (attempt.getTaskStatus().equals (TaskStatus.State.FAILED.toString())) {
attempt.getHttpPort()); if (attempt.getTaskStatus().equals("SUCCEEDED")) { String ttHostname = jt.getNode(ttStatus.getHost()).toString(); if (attempt.getTaskStatus().equals("SUCCEEDED")) {
String status = attempt.getTaskStatus(); assertTrue("Unexpected TASK_STATUS \"" + status + "\" is seen in" + " history file for task attempt " + id,
task.getAllTaskAttempts(); for (JobHistoryParser.TaskAttemptInfo attempt : attempts.values()) { if (attempt.getTaskStatus(). equals(TaskStatus.State.SUCCEEDED.toString())) { long avgFinishTime = (attempt.getFinishTime() -
task.getAllTaskAttempts(); for (JobHistoryParser.TaskAttemptInfo attempt : attempts.values()) { if (attempt.getTaskStatus(). equals(TaskStatus.State.SUCCEEDED.toString())) { long avgFinishTime = (attempt.getFinishTime() -
task.getAllTaskAttempts(); for (JobHistoryParser.TaskAttemptInfo attempt : attempts.values()) { if (attempt.getTaskStatus(). equals(TaskStatus.State.SUCCEEDED.toString())) { long avgFinishTime = (attempt.getFinishTime() -
task.getAllTaskAttempts(); for (JobHistoryParser.TaskAttemptInfo attempt : attempts.values()) { if (attempt.getTaskStatus(). equals(TaskStatus.State.SUCCEEDED.toString())) { long avgFinishTime = (attempt.getFinishTime() -
task.getAllTaskAttempts(); for (JobHistoryParser.TaskAttemptInfo attempt : attempts.values()) { if (attempt.getTaskStatus(). equals(TaskStatus.State.SUCCEEDED.toString())) { long avgFinishTime = (attempt.getFinishTime() -
private void loadAllTaskAttempts() { if (taskAttemptsLoaded.get()) { return; } taskAttemptsLock.lock(); try { if (taskAttemptsLoaded.get()) { return; } for (TaskAttemptInfo attemptHistory : taskInfo.getAllTaskAttempts() .values()) { CompletedTaskAttempt attempt = new CompletedTaskAttempt(taskId, attemptHistory); reportDiagnostics.addAll(attempt.getDiagnostics()); attempts.put(attempt.getID(), attempt); if (successfulAttempt == null && attemptHistory.getTaskStatus() != null && attemptHistory.getTaskStatus().equals( TaskState.SUCCEEDED.toString())) { successfulAttempt = TypeConverter.toYarn(attemptHistory.getAttemptId()); } } taskAttemptsLoaded.set(true); } finally { taskAttemptsLock.unlock(); } } }
private void loadAllTaskAttempts() { if (taskAttemptsLoaded.get()) { return; } taskAttemptsLock.lock(); try { if (taskAttemptsLoaded.get()) { return; } for (TaskAttemptInfo attemptHistory : taskInfo.getAllTaskAttempts() .values()) { CompletedTaskAttempt attempt = new CompletedTaskAttempt(taskId, attemptHistory); reportDiagnostics.addAll(attempt.getDiagnostics()); attempts.put(attempt.getID(), attempt); if (successfulAttempt == null && attemptHistory.getTaskStatus() != null && attemptHistory.getTaskStatus().equals( TaskState.SUCCEEDED.toString())) { successfulAttempt = TypeConverter.toYarn(attemptHistory.getAttemptId()); } } taskAttemptsLoaded.set(true); } finally { taskAttemptsLock.unlock(); } } }
private void loadAllTaskAttempts() { if (taskAttemptsLoaded.get()) { return; } taskAttemptsLock.lock(); try { if (taskAttemptsLoaded.get()) { return; } for (TaskAttemptInfo attemptHistory : taskInfo.getAllTaskAttempts() .values()) { CompletedTaskAttempt attempt = new CompletedTaskAttempt(taskId, attemptHistory); reportDiagnostics.addAll(attempt.getDiagnostics()); attempts.put(attempt.getID(), attempt); if (successfulAttempt == null && attemptHistory.getTaskStatus() != null && attemptHistory.getTaskStatus().equals( TaskState.SUCCEEDED.toString())) { successfulAttempt = TypeConverter.toYarn(attemptHistory.getAttemptId()); } } taskAttemptsLoaded.set(true); } finally { taskAttemptsLock.unlock(); } } }
private TaskAttemptInfo getMockTaskAttemptInfo(TaskAttemptID tai, TaskAttemptState tas) { ContainerId ci = mock(ContainerId.class); Counters counters = mock(Counters.class); TaskType tt = TaskType.MAP; long finishTime = System.currentTimeMillis(); TaskAttemptInfo mockTAinfo = mock(TaskAttemptInfo.class); when(mockTAinfo.getAttemptId()).thenReturn(tai); when(mockTAinfo.getContainerId()).thenReturn(ci); when(mockTAinfo.getCounters()).thenReturn(counters); when(mockTAinfo.getError()).thenReturn(""); when(mockTAinfo.getFinishTime()).thenReturn(finishTime); when(mockTAinfo.getHostname()).thenReturn("localhost"); when(mockTAinfo.getHttpPort()).thenReturn(23); when(mockTAinfo.getMapFinishTime()).thenReturn(finishTime - 1000L); when(mockTAinfo.getPort()).thenReturn(24); when(mockTAinfo.getRackname()).thenReturn("defaultRack"); when(mockTAinfo.getShuffleFinishTime()).thenReturn(finishTime - 2000L); when(mockTAinfo.getShufflePort()).thenReturn(25); when(mockTAinfo.getSortFinishTime()).thenReturn(finishTime - 3000L); when(mockTAinfo.getStartTime()).thenReturn(finishTime -10000); when(mockTAinfo.getState()).thenReturn("task in progress"); when(mockTAinfo.getTaskStatus()).thenReturn(tas.toString()); when(mockTAinfo.getTaskType()).thenReturn(tt); when(mockTAinfo.getTrackerName()).thenReturn("TrackerName"); return mockTAinfo; }
/** Apply the filter (status) on the parsed job and generate summary */ public FilteredJob(JobInfo job, String status) { filter = status; Map<TaskID, JobHistoryParser.TaskInfo> tasks = job.getAllTasks(); for (JobHistoryParser.TaskInfo task : tasks.values()) { Map<TaskAttemptID, JobHistoryParser.TaskAttemptInfo> attempts = task.getAllTaskAttempts(); for (JobHistoryParser.TaskAttemptInfo attempt : attempts.values()) { if (attempt.getTaskStatus().equals(status)) { String hostname = attempt.getHostname(); TaskID id = attempt.getAttemptId().getTaskID(); Set<TaskID> set = badNodesToFilteredTasks.get(hostname); if (set == null) { set = new TreeSet<TaskID>(); set.add(id); badNodesToFilteredTasks.put(hostname, set); }else{ set.add(id); } } } } } }
/** Apply the filter (status) on the parsed job and generate summary */ public FilteredJob(JobInfo job, String status) { filter = status; Map<TaskID, JobHistoryParser.TaskInfo> tasks = job.getAllTasks(); for (JobHistoryParser.TaskInfo task : tasks.values()) { Map<TaskAttemptID, JobHistoryParser.TaskAttemptInfo> attempts = task.getAllTaskAttempts(); for (JobHistoryParser.TaskAttemptInfo attempt : attempts.values()) { if (attempt.getTaskStatus().equals(status)) { String hostname = attempt.getHostname(); TaskID id = attempt.getAttemptId().getTaskID(); Set<TaskID> set = badNodesToFilteredTasks.get(hostname); if (set == null) { set = new TreeSet<TaskID>(); set.add(id); badNodesToFilteredTasks.put(hostname, set); }else{ set.add(id); } } } } } }
CompletedTaskAttempt(TaskId taskId, TaskAttemptInfo attemptInfo) { this.attemptInfo = attemptInfo; this.attemptId = TypeConverter.toYarn(attemptInfo.getAttemptId()); if (attemptInfo.getTaskStatus() != null) { this.state = TaskAttemptState.valueOf(attemptInfo.getTaskStatus()); } else { this.state = TaskAttemptState.KILLED; localDiagMessage = "Attmpt state missing from History : marked as KILLED"; diagnostics.add(localDiagMessage); } if (attemptInfo.getError() != null) { diagnostics.add(attemptInfo.getError()); } }
CompletedTaskAttempt(TaskId taskId, TaskAttemptInfo attemptInfo) { this.attemptInfo = attemptInfo; this.attemptId = TypeConverter.toYarn(attemptInfo.getAttemptId()); if (attemptInfo.getTaskStatus() != null) { this.state = TaskAttemptState.valueOf(attemptInfo.getTaskStatus()); } else { this.state = TaskAttemptState.KILLED; localDiagMessage = "Attmpt state missing from History : marked as KILLED"; diagnostics.add(localDiagMessage); } if (attemptInfo.getError() != null) { diagnostics.add(attemptInfo.getError()); } }
CompletedTaskAttempt(TaskId taskId, TaskAttemptInfo attemptInfo) { this.attemptInfo = attemptInfo; this.attemptId = TypeConverter.toYarn(attemptInfo.getAttemptId()); if (attemptInfo.getTaskStatus() != null) { this.state = TaskAttemptState.valueOf(attemptInfo.getTaskStatus()); } else { this.state = TaskAttemptState.KILLED; localDiagMessage = "Attmpt state missing from History : marked as KILLED"; diagnostics.add(localDiagMessage); } if (attemptInfo.getError() != null) { diagnostics.add(attemptInfo.getError()); } }