@Override public GetDiagnosticsResponse getDiagnostics(GetDiagnosticsRequest request) throws IOException { TaskAttemptId taskAttemptId = request.getTaskAttemptId(); Job job = verifyAndGetJob(taskAttemptId.getTaskId().getJobId(), true); GetDiagnosticsResponse response = recordFactory.newRecordInstance(GetDiagnosticsResponse.class); response.addAllDiagnostics(job.getTask(taskAttemptId.getTaskId()).getAttempt(taskAttemptId).getDiagnostics()); return response; }
@Override public GetTaskReportsResponse getTaskReports(GetTaskReportsRequest request) throws IOException { JobId jobId = request.getJobId(); TaskType taskType = request.getTaskType(); GetTaskReportsResponse response = recordFactory.newRecordInstance(GetTaskReportsResponse.class); Job job = verifyAndGetJob(jobId, true); Collection<Task> tasks = job.getTasks(taskType).values(); for (Task task : tasks) { response.addTaskReport(task.getReport()); } return response; }
switch (task.getType()) { case MAP: switch (task.getState()) { case RUNNING: ++this.mapsRunning; case REDUCE: switch (task.getState()) { case RUNNING: ++this.reducesRunning; default: throw new IllegalStateException( "Task type is neither map nor reduce: " + task.getType()); Map<TaskAttemptId, TaskAttempt> attempts = task.getAttempts(); int newAttempts, running, successful, failed, killed; for (TaskAttempt attempt : attempts.values()) { switch (task.getType()) { case MAP: this.newMapAttempts += newAttempts; default: throw new IllegalStateException("Task type neither map nor reduce: " + task.getType());
private TaskAttempt getSuccessfulAttempt(Task task) { for (TaskAttempt attempt : task.getAttempts().values()) { if (attempt.getState() == TaskAttemptState.SUCCEEDED) { return attempt; } } return null; }
@Private public void constructFinalFullcounters() { this.fullCounters = new Counters(); this.finalMapCounters = new Counters(); this.finalReduceCounters = new Counters(); this.fullCounters.incrAllCounters(jobCounters); for (Task t : this.tasks.values()) { Counters counters = t.getCounters(); switch (t.getType()) { case MAP: this.finalMapCounters.incrAllCounters(counters); break; case REDUCE: this.finalReduceCounters.incrAllCounters(counters); break; default: throw new IllegalStateException("Task type neither map nor reduce: " + t.getType()); } this.fullCounters.incrAllCounters(counters); } }
public void verifyTaskGeneric(Task task, String id, String state, String type, String successfulAttempt, long startTime, long finishTime, long elapsedTime, float progress, String status) { TaskId taskid = task.getID(); String tid = MRApps.toString(taskid); TaskReport report = task.getReport(); WebServicesTestUtils.checkStringMatch("id", tid, id); WebServicesTestUtils.checkStringMatch("type", task.getType().toString(), type); WebServicesTestUtils.checkStringMatch("state", report.getTaskState() .toString(), state); // not easily checked without duplicating logic, just make sure its here assertNotNull("successfulAttempt null", successfulAttempt); assertEquals("startTime wrong", report.getStartTime(), startTime); assertEquals("finishTime wrong", report.getFinishTime(), finishTime); assertEquals("elapsedTime wrong", finishTime - startTime, elapsedTime); assertEquals("progress wrong", report.getProgress() * 100, progress, 1e-3f); assertEquals("status wrong", report.getStatus(), status); }
private void computeProgress() { this.readLock.lock(); try { float mapProgress = 0f; float reduceProgress = 0f; for (Task task : this.tasks.values()) { if (task.getType() == TaskType.MAP) { mapProgress += (task.isFinished() ? 1f : task.getProgress()); } else { reduceProgress += (task.isFinished() ? 1f : task.getProgress()); } } if (this.numMapTasks != 0) { mapProgress = mapProgress / this.numMapTasks; } if (this.numReduceTasks != 0) { reduceProgress = reduceProgress / this.numReduceTasks; } this.mapProgress = mapProgress; this.reduceProgress = reduceProgress; } finally { this.readLock.unlock(); } }
for (TaskId taskId : job.reduceTasks) { Task task = job.tasks.get(taskId); if (TaskState.RUNNING.equals(task.getState())) { for(TaskAttempt attempt : task.getAttempts().values()) { if(attempt.getPhase() == Phase.SHUFFLE) { shufflingReduceTasks++;
public JobTaskCounterInfo(Task task) { total = task.getCounters(); this.id = MRApps.toString(task.getID()); taskCounterGroup = new ArrayList<TaskCounterGroupInfo>(); if (total != null) { for (CounterGroup g : total) { if (g != null) { TaskCounterGroupInfo cginfo = new TaskCounterGroupInfo(g.getName(), g); taskCounterGroup.add(cginfo); } } } } }
if(!task.isFinished()) { allDone = false; job.eventHandler.handle( new TaskEvent(task.getID(), TaskEventType.T_KILL));
if (!task.isFinished()) { if (task.getType() == TaskType.MAP) { ++undoneMaps; } else { for (TaskAttempt attempt : task.getAttempts().values()) { if (attempt.getState() == TaskAttemptState.NEW && INITIAL_NUMBER_FREE_SLOTS - slotsInUse.get() >= taskTypeSlots(task.getType())) { MyTaskAttemptImpl attemptImpl = (MyTaskAttemptImpl)attempt; SpeculatorEvent event
@Override public void transition(JobImpl job, JobEvent event) { job.addDiagnostic(JOB_KILLED_DIAG); for (Task task : job.tasks.values()) { job.eventHandler.handle( new TaskEvent(task.getID(), TaskEventType.T_KILL)); } job.metrics.endRunningJob(job); } }
public void waitForState(Task task, TaskState finalState) throws Exception { int timeoutSecs = 0; TaskReport report = task.getReport(); while (!finalState.equals(report.getTaskState()) && timeoutSecs++ < 20) { System.out.println("Task State for " + task.getID() + " is : " + report.getTaskState() + " Waiting for state : " + finalState + " progress : " + report.getProgress()); report = task.getReport(); Thread.sleep(500); } System.out.println("Task State is : " + report.getTaskState()); Assert.assertEquals("Task state is not correct (timedout)", finalState, report.getTaskState()); }
void incr(Task task) { TaskType type = task.getType(); boolean finished = task.isFinished(); if (type == TaskType.MAP) { if (finished) { ++completedMaps; } ++maps; } else if (type == TaskType.REDUCE) { if (finished) { ++completedReduces; } ++reduces; } } }
when(mockTask.canCommit(any(TaskAttemptId.class))).thenReturn(true); Job mockJob = mock(Job.class); when(mockJob.getTask(any(TaskId.class))).thenReturn(mockTask); boolean canCommit = listener.canCommit(tid); assertFalse(canCommit); verify(mockTask, never()).canCommit(any(TaskAttemptId.class)); canCommit = listener.canCommit(tid); assertTrue(canCommit); verify(mockTask, times(1)).canCommit(any(TaskAttemptId.class));
public static Counters incrTaskCounters( Counters counters, Collection<Task> tasks) { for (Task task : tasks) { counters.incrAllCounters(task.getCounters()); } return counters; }
@Override public JobStateInternal transition(JobImpl job, JobEvent event) { if(!job.failWaitTriggerScheduledFuture.isCancelled()) { for(Task task: job.tasks.values()) { if(!task.isFinished()) { return JobStateInternal.FAIL_WAIT; } } } //Finished waiting. All tasks finished / were killed job.failWaitTriggerScheduledFuture.cancel(false); job.eventHandler.handle(new CommitterJobAbortEvent(job.jobId, job.jobContext, org.apache.hadoop.mapreduce.JobStatus.State.FAILED)); return JobStateInternal.FAIL_ABORT; } }