try { TaskAttempt bestAttempt = selectBestAttempt(); report.setTaskId(taskId); report.setStartTime(getLaunchTime()); report.setFinishTime(getFinishTime()); report.setTaskState(getState()); report.setProgress(bestAttempt == null ? 0f : bestAttempt.getProgress()); report.setStatus(bestAttempt == null ? "" : bestAttempt.getReport().getStateString()); report.addRunningAttempt(attempt.getID()); report.setSuccessfulAttempt(successfulAttempt); report.addDiagnostics(prefix + cs); report.setCounters(TypeConverter.toYarn(bestAttempt == null ? TaskAttemptImpl.EMPTY_COUNTERS : bestAttempt.getCounters()));
public static TaskReport fromYarn(org.apache.hadoop.mapreduce.v2.api.records.TaskReport report) { String[] diagnostics = null; if (report.getDiagnosticsList() != null) { diagnostics = new String[report.getDiagnosticsCount()]; int i = 0; for (String cs : report.getDiagnosticsList()) { diagnostics[i++] = cs.toString(); } } else { diagnostics = new String[0]; } TaskReport rep = new TaskReport(fromYarn(report.getTaskId()), report.getProgress(), report.getTaskState().toString(), diagnostics, fromYarn(report.getTaskState()), report.getStartTime(), report.getFinishTime(), fromYarn(report.getCounters())); List<org.apache.hadoop.mapreduce.TaskAttemptID> runningAtts = new ArrayList<org.apache.hadoop.mapreduce.TaskAttemptID>(); for (org.apache.hadoop.mapreduce.v2.api.records.TaskAttemptId id : report.getRunningAttemptsList()) { runningAtts.add(fromYarn(id)); } rep.setRunningTaskAttemptIds(runningAtts); if (report.getSuccessfulAttempt() != null) { rep.setSuccessfulAttemptId(fromYarn(report.getSuccessfulAttempt())); } return rep; }
@Override public GetTaskReportResponse getTaskReport(GetTaskReportRequest request) throws IOException { GetTaskReportResponse resp = recordFactory.newRecordInstance(GetTaskReportResponse.class); TaskReport report = recordFactory.newRecordInstance(TaskReport.class); report.setTaskId(request.getTaskId()); report.setTaskState(TaskState.NEW); Counters counters = recordFactory.newRecordInstance(Counters.class); counters.addAllCounterGroups(new HashMap<String, CounterGroup>()); report.setCounters(counters); report.addAllRunningAttempts(new ArrayList<TaskAttemptId>()); return resp; }
private void constructTaskReport() { loadAllTaskAttempts(); this.report = Records.newRecord(TaskReport.class); report.setTaskId(taskId); long minLaunchTime = Long.MAX_VALUE; for(TaskAttempt attempt: attempts.values()) { minLaunchTime = Math.min(minLaunchTime, attempt.getLaunchTime()); } minLaunchTime = minLaunchTime == Long.MAX_VALUE ? -1 : minLaunchTime; report.setStartTime(minLaunchTime); report.setFinishTime(taskInfo.getFinishTime()); report.setTaskState(getState()); report.setProgress(getProgress()); Counters counters = getCounters(); if (counters == null) { counters = EMPTY_COUNTERS; } report.setRawCounters(counters); if (successfulAttempt != null) { report.setSuccessfulAttempt(successfulAttempt); } report.addAllDiagnostics(reportDiagnostics); report .addAllRunningAttempts(new ArrayList<TaskAttemptId>(attempts.keySet())); }
public TaskInfo(Task task) { TaskType ttype = task.getType(); this.type = ttype.toString(); TaskReport report = task.getReport(); this.startTime = report.getStartTime(); this.finishTime = report.getFinishTime(); this.state = report.getTaskState(); this.elapsedTime = Times.elapsed(this.startTime, this.finishTime, this.state == TaskState.RUNNING); if (this.elapsedTime == -1) { this.elapsedTime = 0; } this.progress = report.getProgress() * 100; this.status = report.getStatus(); this.id = MRApps.toString(task.getID()); this.taskNum = task.getID().getId(); this.successful = getSuccessfulAttempt(task); if (successful != null) { this.successfulAttempt = MRApps.toString(successful.getID()); } else { this.successfulAttempt = ""; } }
public static TaskReport newTaskReport(TaskId id) { TaskReport report = Records.newRecord(TaskReport.class); report.setTaskId(id); report .setStartTime(System.currentTimeMillis() - (int) (Math.random() * DT)); report.setFinishTime(System.currentTimeMillis() + (int) (Math.random() * DT) + 1); report.setProgress((float) Math.random()); report.setStatus("Moving average: " + Math.random()); report.setCounters(TypeConverter.toYarn(newCounters())); report.setTaskState(TASK_STATES.next()); return report; }
public void verifyCompleted() { for (Job job : getContext().getAllJobs().values()) { JobReport jobReport = job.getReport(); System.out.println("Job start time :" + jobReport.getStartTime()); System.out.println("Job finish time :" + jobReport.getFinishTime()); Assert.assertTrue("Job start time is not less than finish time", jobReport.getStartTime() <= jobReport.getFinishTime()); Assert.assertTrue("Job finish time is in future", jobReport.getFinishTime() <= System.currentTimeMillis()); for (Task task : job.getTasks().values()) { TaskReport taskReport = task.getReport(); System.out.println("Task start time : " + taskReport.getStartTime()); System.out.println("Task finish time : " + taskReport.getFinishTime()); Assert.assertTrue("Task start time is not less than finish time", taskReport.getStartTime() <= taskReport.getFinishTime()); for (TaskAttempt attempt : task.getAttempts().values()) { TaskAttemptReport attemptReport = attempt.getReport(); Assert.assertTrue("Attempt start time is not less than finish time", attemptReport.getStartTime() <= attemptReport.getFinishTime()); } } } }
@Override public boolean isFinished() { switch (report.getTaskState()) { case SUCCEEDED: case KILLED: case FAILED: return true; } return false; }
@Test public void testSetNonNullCountersToNull() { // Create basic class TaskReport report = Records.newRecord(TaskReport.class); // Set raw counters org.apache.hadoop.mapreduce.Counters rCounters = MockJobs.newCounters(); report.setRawCounters(rCounters); // Verify getCounters converts properly from raw to real Counters counters = report.getCounters(); assertNotEquals(null, counters); // Clear counters to null and then verify report.setCounters(null); assertEquals(null, report.getCounters()); assertEquals(null, report.getRawCounters()); }
public void waitForState(Task task, TaskState finalState) throws Exception { int timeoutSecs = 0; TaskReport report = task.getReport(); while (!finalState.equals(report.getTaskState()) && timeoutSecs++ < 20) { System.out.println("Task State for " + task.getID() + " is : " + report.getTaskState() + " Waiting for state : " + finalState + " progress : " + report.getProgress()); report = task.getReport(); Thread.sleep(500); } System.out.println("Task State is : " + report.getTaskState()); Assert.assertEquals("Task state is not correct (timedout)", finalState, report.getTaskState()); }
@Test public void testSetCountersToNull() { // Create basic class TaskReport report = Records.newRecord(TaskReport.class); // Set raw counters to null report.setCounters(null); // Verify properties still null assertEquals(null, report.getCounters()); assertEquals(null, report.getRawCounters()); }
@Test public void testCountersOverRawCounters() { // Create basic class TaskReport report = Records.newRecord(TaskReport.class); org.apache.hadoop.mapreduce.Counters rCounters = MockJobs.newCounters(); Counters altCounters = TypeConverter.toYarn(rCounters); // Set raw counters report.setRawCounters(rCounters); // Set real counters report.setCounters(altCounters); // Verify real counters has priority over raw Counters counters = report.getCounters(); assertNotEquals(null, counters); assertNotEquals(rCounters, altCounters); assertEquals(counters, altCounters); }
@Override public float getProgress() { return report.getProgress(); }
private void constructTaskReport() { loadAllTaskAttempts(); this.report = Records.newRecord(TaskReport.class); report.setTaskId(taskId); long minLaunchTime = Long.MAX_VALUE; for(TaskAttempt attempt: attempts.values()) { minLaunchTime = Math.min(minLaunchTime, attempt.getLaunchTime()); } minLaunchTime = minLaunchTime == Long.MAX_VALUE ? -1 : minLaunchTime; report.setStartTime(minLaunchTime); report.setFinishTime(taskInfo.getFinishTime()); report.setTaskState(getState()); report.setProgress(getProgress()); Counters counters = getCounters(); if (counters == null) { counters = EMPTY_COUNTERS; } report.setCounters(TypeConverter.toYarn(counters)); if (successfulAttempt != null) { report.setSuccessfulAttempt(successfulAttempt); } report.addAllDiagnostics(reportDiagnostics); report .addAllRunningAttempts(new ArrayList<TaskAttemptId>(attempts.keySet())); }
public TaskInfo(Task task) { TaskType ttype = task.getType(); this.type = ttype.toString(); TaskReport report = task.getReport(); this.startTime = report.getStartTime(); this.finishTime = report.getFinishTime(); this.state = report.getTaskState(); this.elapsedTime = Times.elapsed(this.startTime, this.finishTime, this.state == TaskState.RUNNING); if (this.elapsedTime == -1) { this.elapsedTime = 0; } this.progress = report.getProgress() * 100; this.status = report.getStatus(); this.id = MRApps.toString(task.getID()); this.taskNum = task.getID().getId(); this.successful = getSuccessfulAttempt(task); if (successful != null) { this.successfulAttempt = MRApps.toString(successful.getID()); } else { this.successfulAttempt = ""; } }
@Override public GetTaskReportResponse getTaskReport(GetTaskReportRequest request) throws IOException { GetTaskReportResponse resp = recordFactory.newRecordInstance(GetTaskReportResponse.class); TaskReport report = recordFactory.newRecordInstance(TaskReport.class); report.setTaskId(request.getTaskId()); report.setTaskState(TaskState.NEW); Counters counters = recordFactory.newRecordInstance(Counters.class); counters.addAllCounterGroups(new HashMap<String, CounterGroup>()); report.setCounters(counters); report.addAllRunningAttempts(new ArrayList<TaskAttemptId>()); return resp; }
long task1StartTime = mapTask1.getReport().getStartTime(); long task1FinishTime = mapTask1.getReport().getFinishTime(); jobStartTime, job.getReport().getStartTime()); Assert.assertEquals("Task Start time not correct", task1StartTime, mapTask1.getReport().getStartTime()); Assert.assertEquals("Task Finish time not correct", task1FinishTime, mapTask1.getReport().getFinishTime()); Assert.assertEquals(2, job.getAMInfos().size()); int attemptNum = 1;
@Override public TaskState getState() { return report.getTaskState(); } };
public void waitForInternalState(TaskImpl task, TaskStateInternal finalState) throws Exception { int timeoutSecs = 0; TaskReport report = task.getReport(); TaskStateInternal iState = task.getInternalState(); while (!finalState.equals(iState) && timeoutSecs++ < 20) { System.out.println("Task Internal State is : " + iState + " Waiting for Internal state : " + finalState + " progress : " + report.getProgress()); Thread.sleep(500); report = task.getReport(); iState = task.getInternalState(); } System.out.println("Task Internal State is : " + iState); Assert.assertEquals("Task Internal state is not correct (timedout)", finalState, iState); }
public static TaskReport fromYarn(org.apache.hadoop.mapreduce.v2.api.records.TaskReport report) { String[] diagnostics = null; if (report.getDiagnosticsList() != null) { diagnostics = new String[report.getDiagnosticsCount()]; int i = 0; for (String cs : report.getDiagnosticsList()) { diagnostics[i++] = cs.toString(); } } else { diagnostics = new String[0]; } TaskReport rep = new TaskReport(fromYarn(report.getTaskId()), report.getProgress(), report.getTaskState().toString(), diagnostics, fromYarn(report.getTaskState()), report.getStartTime(), report.getFinishTime(), fromYarn(report.getCounters())); List<org.apache.hadoop.mapreduce.TaskAttemptID> runningAtts = new ArrayList<org.apache.hadoop.mapreduce.TaskAttemptID>(); for (org.apache.hadoop.mapreduce.v2.api.records.TaskAttemptId id : report.getRunningAttemptsList()) { runningAtts.add(fromYarn(id)); } rep.setRunningTaskAttemptIds(runningAtts); if (report.getSuccessfulAttempt() != null) { rep.setSuccessfulAttemptId(fromYarn(report.getSuccessfulAttempt())); } return rep; }