private void printLast(JobHistoryParser.TaskAttemptInfo [] tasks, String taskType, Comparator<JobHistoryParser.TaskAttemptInfo> cmp ) { Arrays.sort(tasks, cFinishMapRed); JobHistoryParser.TaskAttemptInfo last = tasks[0]; StringBuffer lastBuf = new StringBuffer(); lastBuf.append("The last ").append(taskType); lastBuf.append(" task ").append(last.getAttemptId().getTaskID()); Long finishTime; if ("shuffle".equals(taskType)) { finishTime = last.getShuffleFinishTime(); } else { finishTime = last.getFinishTime(); } lastBuf.append(" finished at (relative to the Job launch time): "); lastBuf.append(StringUtils.getFormattedTimeWithDiff(dateFormat, finishTime, job.getLaunchTime())); System.out.println(lastBuf.toString()); }
private void printLast(JobHistoryParser.TaskAttemptInfo [] tasks, String taskType, Comparator<JobHistoryParser.TaskAttemptInfo> cmp ) { Arrays.sort(tasks, cFinishMapRed); JobHistoryParser.TaskAttemptInfo last = tasks[0]; StringBuffer lastBuf = new StringBuffer(); lastBuf.append("The last ").append(taskType); lastBuf.append(" task ").append(last.getAttemptId().getTaskID()); Long finishTime; if ("shuffle".equals(taskType)) { finishTime = last.getShuffleFinishTime(); } else { finishTime = last.getFinishTime(); } lastBuf.append(" finished at (relative to the Job launch time): "); lastBuf.append(StringUtils.getFormattedTimeWithDiff(dateFormat, finishTime, job.getLaunchTime())); System.out.println(lastBuf.toString()); }
private void printLast(JobHistoryParser.TaskAttemptInfo [] tasks, String taskType, Comparator<JobHistoryParser.TaskAttemptInfo> cmp ) { Arrays.sort(tasks, cFinishMapRed); JobHistoryParser.TaskAttemptInfo last = tasks[0]; StringBuffer lastBuf = new StringBuffer(); lastBuf.append("The last ").append(taskType); lastBuf.append(" task ").append(last.getAttemptId().getTaskID()); Long finishTime; if ("shuffle".equals(taskType)) { finishTime = last.getShuffleFinishTime(); } else { finishTime = last.getFinishTime(); } lastBuf.append(" finished at (relative to the Job launch time): "); lastBuf.append(StringUtils.getFormattedTimeWithDiff(dateFormat, finishTime, job.getLaunchTime())); System.out.println(lastBuf.toString()); }
/** Apply the filter (status) on the parsed job and generate summary */ public FilteredJob(JobInfo job, String status) { filter = status; Map<TaskID, JobHistoryParser.TaskInfo> tasks = job.getAllTasks(); for (JobHistoryParser.TaskInfo task : tasks.values()) { Map<TaskAttemptID, JobHistoryParser.TaskAttemptInfo> attempts = task.getAllTaskAttempts(); for (JobHistoryParser.TaskAttemptInfo attempt : attempts.values()) { if (attempt.getTaskStatus().equals(status)) { String hostname = attempt.getHostname(); TaskID id = attempt.getAttemptId().getTaskID(); Set<TaskID> set = badNodesToFilteredTasks.get(hostname); if (set == null) { set = new TreeSet<TaskID>(); set.add(id); badNodesToFilteredTasks.put(hostname, set); }else{ set.add(id); } } } } } }
/** Apply the filter (status) on the parsed job and generate summary */ public FilteredJob(JobInfo job, String status) { filter = status; Map<TaskID, JobHistoryParser.TaskInfo> tasks = job.getAllTasks(); for (JobHistoryParser.TaskInfo task : tasks.values()) { Map<TaskAttemptID, JobHistoryParser.TaskAttemptInfo> attempts = task.getAllTaskAttempts(); for (JobHistoryParser.TaskAttemptInfo attempt : attempts.values()) { if (attempt.getTaskStatus().equals(status)) { String hostname = attempt.getHostname(); TaskID id = attempt.getAttemptId().getTaskID(); Set<TaskID> set = badNodesToFilteredTasks.get(hostname); if (set == null) { set = new TreeSet<TaskID>(); set.add(id); badNodesToFilteredTasks.put(hostname, set); }else{ set.add(id); } } } } } }
/** Apply the filter (status) on the parsed job and generate summary */ public FilteredJob(JobInfo job, String status) { filter = status; Map<TaskID, JobHistoryParser.TaskInfo> tasks = job.getAllTasks(); for (JobHistoryParser.TaskInfo task : tasks.values()) { Map<TaskAttemptID, JobHistoryParser.TaskAttemptInfo> attempts = task.getAllTaskAttempts(); for (JobHistoryParser.TaskAttemptInfo attempt : attempts.values()) { if (attempt.getTaskStatus().equals(status)) { String hostname = attempt.getHostname(); TaskID id = attempt.getAttemptId().getTaskID(); Set<TaskID> set = badNodesToFilteredTasks.get(hostname); if (set == null) { set = new TreeSet<TaskID>(); set.add(id); badNodesToFilteredTasks.put(hostname, set); }else{ set.add(id); } } } } } }
/** Apply the filter (status) on the parsed job and generate summary */ public FilteredJob(JobInfo job, String status) { filter = status; Map<TaskID, JobHistoryParser.TaskInfo> tasks = job.getAllTasks(); for (JobHistoryParser.TaskInfo task : tasks.values()) { Map<TaskAttemptID, JobHistoryParser.TaskAttemptInfo> attempts = task.getAllTaskAttempts(); for (JobHistoryParser.TaskAttemptInfo attempt : attempts.values()) { if (attempt.getTaskStatus().equals(status)) { String hostname = attempt.getHostname(); TaskID id = attempt.getAttemptId().getTaskID(); Set<TaskID> set = badNodesToFilteredTasks.get(hostname); if (set == null) { set = new TreeSet<TaskID>(); set.add(id); badNodesToFilteredTasks.put(hostname, set); }else{ set.add(id); } } } } } }
/** * Return the TaskLogsUrl of a particular TaskAttempt * * @param attempt * @return the taskLogsUrl. null if http-port or tracker-name or * task-attempt-id are unavailable. */ public static String getTaskLogsUrl(String scheme, JobHistoryParser.TaskAttemptInfo attempt) { if (attempt.getHttpPort() == -1 || attempt.getTrackerName().equals("") || attempt.getAttemptId() == null) { return null; } String taskTrackerName = HostUtil.convertTrackerNameToHostName( attempt.getTrackerName()); return HostUtil.getTaskLogUrl(scheme, taskTrackerName, Integer.toString(attempt.getHttpPort()), attempt.getAttemptId().toString()); }
private void printLast(JobHistoryParser.TaskAttemptInfo [] tasks, String taskType, Comparator<JobHistoryParser.TaskAttemptInfo> cmp ) { Arrays.sort(tasks, cFinishMapRed); JobHistoryParser.TaskAttemptInfo last = tasks[0]; StringBuffer lastBuf = new StringBuffer(); lastBuf.append("The last ").append(taskType); lastBuf.append(" task ").append(last.getAttemptId().getTaskID()); Long finishTime; if ("shuffle".equals(taskType)) { finishTime = last.getShuffleFinishTime(); } else { finishTime = last.getFinishTime(); } lastBuf.append(" finished at (relative to the Job launch time): "); lastBuf.append(StringUtils.getFormattedTimeWithDiff(dateFormat, finishTime, job.getLaunchTime())); System.out.println(lastBuf.toString()); }
/** Apply the filter (status) on the parsed job and generate summary */ public FilteredJob(JobInfo job, String status) { filter = status; Map<TaskID, JobHistoryParser.TaskInfo> tasks = job.getAllTasks(); for (JobHistoryParser.TaskInfo task : tasks.values()) { Map<TaskAttemptID, JobHistoryParser.TaskAttemptInfo> attempts = task.getAllTaskAttempts(); for (JobHistoryParser.TaskAttemptInfo attempt : attempts.values()) { if (attempt.getTaskStatus().equals(status)) { String hostname = attempt.getHostname(); TaskID id = attempt.getAttemptId().getTaskID(); Set<TaskID> set = badNodesToFilteredTasks.get(hostname); if (set == null) { set = new TreeSet<TaskID>(); set.add(id); badNodesToFilteredTasks.put(hostname, set); }else{ set.add(id); } } } } } }
/** * Return the TaskLogsUrl of a particular TaskAttempt * * @param attempt * @return the taskLogsUrl. null if http-port or tracker-name or * task-attempt-id are unavailable. */ public static String getTaskLogsUrl( JobHistoryParser.TaskAttemptInfo attempt) { if (attempt.getHttpPort() == -1 || attempt.getTrackerName().equals("") || attempt.getAttemptId() == null) { return null; } String taskTrackerName = JobInProgress.convertTrackerNameToHostName( attempt.getTrackerName()); return TaskLogServlet.getTaskLogUrl(taskTrackerName, Integer.toString(attempt.getHttpPort()), attempt.getAttemptId().toString()); }
/** * Return the TaskLogsUrl of a particular TaskAttempt * * @param attempt * @return the taskLogsUrl. null if http-port or tracker-name or * task-attempt-id are unavailable. */ public static String getTaskLogsUrl(String scheme, JobHistoryParser.TaskAttemptInfo attempt) { if (attempt.getHttpPort() == -1 || attempt.getTrackerName().equals("") || attempt.getAttemptId() == null) { return null; } String taskTrackerName = HostUtil.convertTrackerNameToHostName( attempt.getTrackerName()); return HostUtil.getTaskLogUrl(scheme, taskTrackerName, Integer.toString(attempt.getHttpPort()), attempt.getAttemptId().toString()); }
/** * Return the TaskLogsUrl of a particular TaskAttempt * * @param attempt * @return the taskLogsUrl. null if http-port or tracker-name or * task-attempt-id are unavailable. */ public static String getTaskLogsUrl(String scheme, JobHistoryParser.TaskAttemptInfo attempt) { if (attempt.getHttpPort() == -1 || attempt.getTrackerName().equals("") || attempt.getAttemptId() == null) { return null; } String taskTrackerName = HostUtil.convertTrackerNameToHostName( attempt.getTrackerName()); return HostUtil.getTaskLogUrl(scheme, taskTrackerName, Integer.toString(attempt.getHttpPort()), attempt.getAttemptId().toString()); }
/** * Return the TaskLogsUrl of a particular TaskAttempt * * @param attempt * @return the taskLogsUrl. null if http-port or tracker-name or * task-attempt-id are unavailable. */ public static String getTaskLogsUrl(String scheme, JobHistoryParser.TaskAttemptInfo attempt) { if (attempt.getHttpPort() == -1 || attempt.getTrackerName().equals("") || attempt.getAttemptId() == null) { return null; } String taskTrackerName = HostUtil.convertTrackerNameToHostName( attempt.getTrackerName()); return HostUtil.getTaskLogUrl(scheme, taskTrackerName, Integer.toString(attempt.getHttpPort()), attempt.getAttemptId().toString()); }
private void printLast(JobHistoryParser.TaskAttemptInfo [] tasks, String taskType, Comparator<JobHistoryParser.TaskAttemptInfo> cmp ) { Arrays.sort(tasks, cFinishMapRed); JobHistoryParser.TaskAttemptInfo last = tasks[0]; StringBuffer lastBuf = new StringBuffer(); lastBuf.append("The last ").append(taskType); lastBuf.append(" task ").append(last.getAttemptId().getTaskID()); Long finishTime; if ("shuffle".equals(taskType)) { finishTime = last.getShuffleFinishTime(); } else { finishTime = last.getFinishTime(); } lastBuf.append(" finished at (relative to the Job launch time): "); lastBuf.append(StringUtils.getFormattedTimeWithDiff(dateFormat, finishTime, job.getLaunchTime())); System.out.println(lastBuf.toString()); }
private void printTask(String logFile, JobHistoryParser.TaskAttemptInfo attempt, JspWriter out) throws IOException{ out.print("<tr>"); out.print("<td>" + "<a href=\"taskdetailshistory.jsp?logFile="+ logFile +"&tipid="+attempt.getAttemptId().getTaskID().toString() +"\">" + attempt.getAttemptId().getTaskID() + "</a></td>"); out.print("<td>" + StringUtils.getFormattedTimeWithDiff(dateFormat, attempt.getStartTime(), 0 ) + "</td>"); out.print("<td>" + StringUtils.getFormattedTimeWithDiff(dateFormat, attempt.getFinishTime(), attempt.getStartTime() ) + "</td>"); out.print("<td>"+ HtmlQuoting.quoteHtmlChars(attempt.getError()) +"</td>"); out.print("</tr>"); }
CompletedTaskAttempt(TaskId taskId, TaskAttemptInfo attemptInfo) { this.attemptInfo = attemptInfo; this.attemptId = TypeConverter.toYarn(attemptInfo.getAttemptId()); if (attemptInfo.getTaskStatus() != null) { this.state = TaskAttemptState.valueOf(attemptInfo.getTaskStatus()); } else { this.state = TaskAttemptState.KILLED; localDiagMessage = "Attmpt state missing from History : marked as KILLED"; diagnostics.add(localDiagMessage); } if (attemptInfo.getError() != null) { diagnostics.add(attemptInfo.getError()); } }
CompletedTaskAttempt(TaskId taskId, TaskAttemptInfo attemptInfo) { this.attemptInfo = attemptInfo; this.attemptId = TypeConverter.toYarn(attemptInfo.getAttemptId()); if (attemptInfo.getTaskStatus() != null) { this.state = TaskAttemptState.valueOf(attemptInfo.getTaskStatus()); } else { this.state = TaskAttemptState.KILLED; localDiagMessage = "Attmpt state missing from History : marked as KILLED"; diagnostics.add(localDiagMessage); } if (attemptInfo.getError() != null) { diagnostics.add(attemptInfo.getError()); } }
CompletedTaskAttempt(TaskId taskId, TaskAttemptInfo attemptInfo) { this.attemptInfo = attemptInfo; this.attemptId = TypeConverter.toYarn(attemptInfo.getAttemptId()); if (attemptInfo.getTaskStatus() != null) { this.state = TaskAttemptState.valueOf(attemptInfo.getTaskStatus()); } else { this.state = TaskAttemptState.KILLED; localDiagMessage = "Attmpt state missing from History : marked as KILLED"; diagnostics.add(localDiagMessage); } if (attemptInfo.getError() != null) { diagnostics.add(attemptInfo.getError()); } }
private static void addTaskAttemptInfo( JobHistoryParser.TaskInfo task, int id) { JobHistoryParser.TaskAttemptInfo attempt = new JobHistoryParser.TaskAttemptInfo(); attempt.attemptId = new TaskAttemptID( TaskID.downgrade(task.getTaskId()), id); attempt.startTime = task.getStartTime(); attempt.finishTime = task.getFinishTime(); attempt.shuffleFinishTime = task.getFinishTime(); attempt.sortFinishTime = task.getFinishTime(); attempt.mapFinishTime = task.getFinishTime(); attempt.status = task.getTaskStatus(); attempt.taskType = task.getTaskType(); attempt.trackerName = "localhost"; attempt.httpPort = 1234; attempt.hostname = "localhost"; task.attemptsMap.put(attempt.getAttemptId(), attempt); } }