protected JobStateInternal checkReadyForCommit() { JobStateInternal currentState = getInternalState(); if (completedTaskCount == tasks.size() && currentState == JobStateInternal.RUNNING) { eventHandler.handle(new CommitterJobCommitEvent(jobId, getJobContext())); return JobStateInternal.COMMITTING; } // return the current state as job not ready to commit yet return getInternalState(); }
protected JobStateInternal checkReadyForCommit() { JobStateInternal currentState = getInternalState(); if (completedTaskCount == tasks.size() && currentState == JobStateInternal.RUNNING) { eventHandler.handle(new CommitterJobCommitEvent(jobId, getJobContext())); return JobStateInternal.COMMITTING; } // return the current state as job not ready to commit yet return getInternalState(); }
protected JobStateInternal checkReadyForCommit() { JobStateInternal currentState = getInternalState(); if (completedTaskCount == tasks.size() && currentState == JobStateInternal.RUNNING) { eventHandler.handle(new CommitterJobCommitEvent(jobId, getJobContext())); return JobStateInternal.COMMITTING; } // return the current state as job not ready to commit yet return getInternalState(); }
@Override public JobState getState() { readLock.lock(); try { JobState state = getExternalState(getInternalState()); if (!appContext.hasSuccessfullyUnregistered() && (state == JobState.SUCCEEDED || state == JobState.FAILED || state == JobState.KILLED || state == JobState.ERROR)) { return lastNonFinalState; } else { return state; } } finally { readLock.unlock(); } }
@Override public JobState getState() { readLock.lock(); try { JobState state = getExternalState(getInternalState()); if (!appContext.hasSuccessfullyUnregistered() && (state == JobState.SUCCEEDED || state == JobState.FAILED || state == JobState.KILLED || state == JobState.ERROR)) { return lastNonFinalState; } else { return state; } } finally { readLock.unlock(); } }
@Override public JobState getState() { readLock.lock(); try { JobState state = getExternalState(getInternalState()); if (!appContext.hasSuccessfullyUnregistered() && (state == JobState.SUCCEEDED || state == JobState.FAILED || state == JobState.KILLED || state == JobState.ERROR)) { return lastNonFinalState; } else { return state; } } finally { readLock.unlock(); } }
private static void assertJobState(JobImpl job, JobStateInternal state) { int timeToWaitMsec = 5 * 1000; while (timeToWaitMsec > 0 && job.getInternalState() != state) { try { Thread.sleep(10); timeToWaitMsec -= 10; } catch (InterruptedException e) { break; } } Assert.assertEquals(state, job.getInternalState()); }
@Override protected JobStateInternal checkJobAfterTaskCompletion(JobImpl job) { if (job.completedTaskCount == job.tasks.size()) { job.setFinishTime(); job.eventHandler.handle(new CommitterJobAbortEvent(job.jobId, job.jobContext, org.apache.hadoop.mapreduce.JobStatus.State.KILLED)); return JobStateInternal.KILL_ABORT; } //return the current state, Job not finished yet return job.getInternalState(); } }
@Override protected JobStateInternal checkJobAfterTaskCompletion(JobImpl job) { if (job.completedTaskCount == job.tasks.size()) { job.setFinishTime(); job.eventHandler.handle(new CommitterJobAbortEvent(job.jobId, job.jobContext, org.apache.hadoop.mapreduce.JobStatus.State.KILLED)); return JobStateInternal.KILL_ABORT; } //return the current state, Job not finished yet return job.getInternalState(); } }
@Override protected JobStateInternal checkJobAfterTaskCompletion(JobImpl job) { if (job.completedTaskCount == job.tasks.size()) { job.setFinishTime(); job.eventHandler.handle(new CommitterJobAbortEvent(job.jobId, job.jobContext, org.apache.hadoop.mapreduce.JobStatus.State.KILLED)); return JobStateInternal.KILL_ABORT; } //return the current state, Job not finished yet return job.getInternalState(); } }
@Override public Counters getAllCounters() { readLock.lock(); try { JobStateInternal state = getInternalState(); if (state == JobStateInternal.ERROR || state == JobStateInternal.FAILED || state == JobStateInternal.KILLED || state == JobStateInternal.SUCCEEDED) { this.mayBeConstructFinalFullCounters(); return fullCounters; } Counters counters = new Counters(); counters.incrAllCounters(jobCounters); return incrTaskCounters(counters, tasks.values()); } finally { readLock.unlock(); } }
@Override public Counters getAllCounters() { readLock.lock(); try { JobStateInternal state = getInternalState(); if (state == JobStateInternal.ERROR || state == JobStateInternal.FAILED || state == JobStateInternal.KILLED || state == JobStateInternal.SUCCEEDED) { this.mayBeConstructFinalFullCounters(); return fullCounters; } Counters counters = new Counters(); counters.incrAllCounters(jobCounters); return incrTaskCounters(counters, tasks.values()); } finally { readLock.unlock(); } }
@Override public Counters getAllCounters() { readLock.lock(); try { JobStateInternal state = getInternalState(); if (state == JobStateInternal.ERROR || state == JobStateInternal.FAILED || state == JobStateInternal.KILLED || state == JobStateInternal.SUCCEEDED) { this.mayBeConstructFinalFullCounters(); return fullCounters; } Counters counters = new Counters(); counters.incrAllCounters(jobCounters); return incrTaskCounters(counters, tasks.values()); } finally { readLock.unlock(); } }
@Override public JobReport getReport() { readLock.lock(); try { JobState state = getState(); // jobFile can be null if the job is not yet inited. String jobFile = remoteJobConfFile == null ? "" : remoteJobConfFile.toString(); StringBuilder diagsb = new StringBuilder(); for (String s : getDiagnostics()) { diagsb.append(s).append("\n"); } if (getInternalState() == JobStateInternal.NEW) { return MRBuilderUtils.newJobReport(jobId, jobName, username, state, appSubmitTime, startTime, finishTime, setupProgress, 0.0f, 0.0f, cleanupProgress, jobFile, amInfos, isUber, diagsb.toString()); } computeProgress(); JobReport report = MRBuilderUtils.newJobReport(jobId, jobName, username, state, appSubmitTime, startTime, finishTime, setupProgress, this.mapProgress, this.reduceProgress, cleanupProgress, jobFile, amInfos, isUber, diagsb.toString()); return report; } finally { readLock.unlock(); } }
public void waitForInternalState(JobImpl job, JobStateInternal finalState) throws Exception { int timeoutSecs = 0; JobStateInternal iState = job.getInternalState(); while (!finalState.equals(iState) && timeoutSecs++ < 20) { System.out.println("Job Internal State is : " + iState + " Waiting for Internal state : " + finalState); Thread.sleep(500); iState = job.getInternalState(); } System.out.println("Task Internal State is : " + iState); Assert.assertEquals("Task Internal state is not correct (timedout)", finalState, iState); }
private void actOnUnusableNode(NodeId nodeId, NodeState nodeState) { // rerun previously successful map tasks // do this only if the job is still in the running state and there are // running reducers if (getInternalState() == JobStateInternal.RUNNING && !allReducersComplete()) { List<TaskAttemptId> taskAttemptIdList = nodesToSucceededTaskAttempts.get(nodeId); if (taskAttemptIdList != null) { String mesg = "TaskAttempt killed because it ran on unusable node " + nodeId; for (TaskAttemptId id : taskAttemptIdList) { if (TaskType.MAP == id.getTaskId().getTaskType()) { // reschedule only map tasks because their outputs maybe unusable LOG.info(mesg + ". AttemptId:" + id); // Kill the attempt and indicate that next map attempt should be // rescheduled (i.e. considered as a fast fail map). eventHandler.handle(new TaskAttemptKillEvent(id, mesg, true)); } } } } // currently running task attempts on unusable nodes are handled in // RMContainerAllocator }
@Override protected Job createJob(Configuration conf, JobStateInternal forcedState, String diagnostic) { JobImpl jobImpl = mock(JobImpl.class); when(jobImpl.getInternalState()).thenReturn(this.jobStateInternal); when(jobImpl.getAllCounters()).thenReturn(new Counters()); JobID jobID = JobID.forName("job_1234567890000_0001"); JobId jobId = TypeConverter.toYarn(jobID); when(jobImpl.getID()).thenReturn(jobId); ((AppContext) getContext()) .getAllJobs().put(jobImpl.getID(), jobImpl); return jobImpl; }
JobStateInternal finished(JobStateInternal finalState) { if (getInternalState() == JobStateInternal.RUNNING) { metrics.endRunningJob(this); } if (finishTime == 0) setFinishTime(); eventHandler.handle(new JobFinishEvent(jobId)); switch (finalState) { case KILLED: metrics.killedJob(this); break; case REBOOT: case ERROR: case FAILED: metrics.failedJob(this); break; case SUCCEEDED: metrics.completedJob(this); break; default: throw new IllegalArgumentException("Illegal job state: " + finalState); } return finalState; }
JobStateInternal finished(JobStateInternal finalState) { if (getInternalState() == JobStateInternal.RUNNING) { metrics.endRunningJob(this); } if (finishTime == 0) setFinishTime(); eventHandler.handle(new JobFinishEvent(jobId)); switch (finalState) { case KILLED: metrics.killedJob(this); break; case REBOOT: case ERROR: case FAILED: metrics.failedJob(this); break; case SUCCEEDED: metrics.completedJob(this); break; default: throw new IllegalArgumentException("Illegal job state: " + finalState); } return finalState; }
JobStateInternal finished(JobStateInternal finalState) { if (getInternalState() == JobStateInternal.RUNNING) { metrics.endRunningJob(this); } if (finishTime == 0) setFinishTime(); eventHandler.handle(new JobFinishEvent(jobId)); switch (finalState) { case KILLED: metrics.killedJob(this); break; case REBOOT: case ERROR: case FAILED: metrics.failedJob(this); break; case SUCCEEDED: metrics.completedJob(this); break; default: throw new IllegalArgumentException("Illegal job state: " + finalState); } return finalState; }