private static JobFinishedEvent createJobFinishedEvent(JobImpl job) { job.mayBeConstructFinalFullCounters(); JobFinishedEvent jfe = new JobFinishedEvent( job.oldJobId, job.finishTime, job.succeededMapTaskCount, job.succeededReduceTaskCount, job.failedMapTaskCount, job.failedReduceTaskCount, job.killedMapTaskCount, job.killedReduceTaskCount, job.finalMapCounters, job.finalReduceCounters, job.fullCounters); return jfe; }
private static JobFinishedEvent createJobFinishedEvent(JobImpl job) { job.mayBeConstructFinalFullCounters(); JobFinishedEvent jfe = new JobFinishedEvent( job.oldJobId, job.finishTime, job.succeededMapTaskCount, job.succeededReduceTaskCount, job.failedMapTaskCount, job.failedReduceTaskCount, job.finalMapCounters, job.finalReduceCounters, job.fullCounters); return jfe; }
private static JobFinishedEvent createJobFinishedEvent(JobImpl job) { job.mayBeConstructFinalFullCounters(); JobFinishedEvent jfe = new JobFinishedEvent( job.oldJobId, job.finishTime, job.succeededMapTaskCount, job.succeededReduceTaskCount, job.failedMapTaskCount, job.failedReduceTaskCount, job.finalMapCounters, job.finalReduceCounters, job.fullCounters); return jfe; }
Bus bus = new Bus(); bus.post(new JobFinishedEvent(42));
HistoryEvent maybeEmitEvent(ParsedLine line, String jobIDName, HistoryEventEmitter thatg) { if (jobIDName == null) { return null; } JobID jobID = JobID.forName(jobIDName); String finishTime = line.get("FINISH_TIME"); String status = line.get("JOB_STATUS"); String finishedMaps = line.get("FINISHED_MAPS"); String finishedReduces = line.get("FINISHED_REDUCES"); String failedMaps = line.get("FAILED_MAPS"); String failedReduces = line.get("FAILED_REDUCES"); String counters = line.get("COUNTERS"); if (status != null && status.equalsIgnoreCase("success") && finishTime != null && finishedMaps != null && finishedReduces != null) { return new JobFinishedEvent(jobID, Long.parseLong(finishTime), Integer .parseInt(finishedMaps), Integer.parseInt(finishedReduces), Integer .parseInt(failedMaps), Integer.parseInt(failedReduces), null, null, maybeParseCounters(counters)); } return null; } }
HistoryEvent maybeEmitEvent(ParsedLine line, String jobIDName, HistoryEventEmitter thatg) { if (jobIDName == null) { return null; } JobID jobID = JobID.forName(jobIDName); String finishTime = line.get("FINISH_TIME"); String status = line.get("JOB_STATUS"); String finishedMaps = line.get("FINISHED_MAPS"); String finishedReduces = line.get("FINISHED_REDUCES"); String failedMaps = line.get("FAILED_MAPS"); String failedReduces = line.get("FAILED_REDUCES"); String counters = line.get("COUNTERS"); if (status != null && status.equalsIgnoreCase("success") && finishTime != null && finishedMaps != null && finishedReduces != null) { return new JobFinishedEvent(jobID, Long.parseLong(finishTime), Integer .parseInt(finishedMaps), Integer.parseInt(finishedReduces), Integer .parseInt(failedMaps), Integer.parseInt(failedReduces), null, null, maybeParseCounters(counters)); } return null; } }
new JobFinishedEvent(jobId, 12346, 1, 1, 0, 0, new Counters(), new Counters(), new Counters()); jh.logEvent(jfe, jobId);
@Test (timeout=50000) public void testDontSetTrackingURLIfHistoryWriteFailed() throws Exception { TestParams t = new TestParams(true); Configuration conf = new Configuration(); JHEvenHandlerForTest realJheh = new JHEvenHandlerForTest(t.mockAppContext, 0, false); JHEvenHandlerForTest jheh = spy(realJheh); jheh.init(conf); try { jheh.start(); doReturn(false).when(jheh).moveToDoneNow(any(Path.class), any(Path.class)); doNothing().when(jheh).moveTmpToDone(any(Path.class)); handleEvent(jheh, new JobHistoryEvent(t.jobId, new AMStartedEvent( t.appAttemptId, 200, t.containerId, "nmhost", 3000, 4000, -1))); verify(jheh, times(0)).processDoneFiles(any(JobId.class)); verify(t.mockAppContext, times(0)).setHistoryUrl(any(String.class)); // Job finishes, but doesn't successfully write history handleEvent(jheh, new JobHistoryEvent(t.jobId, new JobFinishedEvent( TypeConverter.fromYarn(t.jobId), 0, 0, 0, 0, 0, 0, 0, new Counters(), new Counters(), new Counters()))); verify(jheh, times(1)).processDoneFiles(any(JobId.class)); verify(t.mockAppContext, times(0)).setHistoryUrl(any(String.class)); } finally { jheh.stop(); } } @Test (timeout=50000)
handleEvent(jheh, new JobHistoryEvent(t.jobId, new JobFinishedEvent( TypeConverter.fromYarn(t.jobId), 0, 0, 0, 0, 0, 0, 0, new Counters(), new Counters(), new Counters())));
@Test (timeout=50000) public void testSetTrackingURLAfterHistoryIsWritten() throws Exception { TestParams t = new TestParams(true); Configuration conf = new Configuration(); JHEvenHandlerForTest realJheh = new JHEvenHandlerForTest(t.mockAppContext, 0, false); JHEvenHandlerForTest jheh = spy(realJheh); jheh.init(conf); try { jheh.start(); handleEvent(jheh, new JobHistoryEvent(t.jobId, new AMStartedEvent( t.appAttemptId, 200, t.containerId, "nmhost", 3000, 4000, -1))); verify(jheh, times(0)).processDoneFiles(any(JobId.class)); verify(t.mockAppContext, times(0)).setHistoryUrl(any(String.class)); // Job finishes and successfully writes history handleEvent(jheh, new JobHistoryEvent(t.jobId, new JobFinishedEvent( TypeConverter.fromYarn(t.jobId), 0, 0, 0, 0, 0, 0, 0, new Counters(), new Counters(), new Counters()))); verify(jheh, times(1)).processDoneFiles(any(JobId.class)); String historyUrl = MRWebAppUtil.getApplicationWebURLOnJHSWithScheme( conf, t.mockAppContext.getApplicationID()); verify(t.mockAppContext, times(1)).setHistoryUrl(historyUrl); } finally { jheh.stop(); } }
t.taskID, t.taskAttemptID, 0, TaskType.MAP, "", null, 0))); queueEvent(jheh, new JobHistoryEvent(t.jobId, new JobFinishedEvent( TypeConverter.fromYarn(t.jobId), 0, 10, 10, 0, 0, 0, 0, null, null, new Counters())));
t.appAttemptId, 200, t.containerId, "nmhost", 3000, 4000, -1))); handleEvent(jheh, new JobHistoryEvent(t.jobId, new JobFinishedEvent( TypeConverter.fromYarn(t.jobId), 0, 0, 0, 0, 0, 0, 0, new Counters(), new Counters(), new Counters())));
verify(jheh, times(1)).processDoneFiles(any(JobId.class)); handleEvent(jheh, new JobHistoryEvent(t.jobId, new JobFinishedEvent( TypeConverter.fromYarn(t.jobId), 0, 0, 0, 0, 0, 0, 0, new Counters(), new Counters(), new Counters())));
verify(jheh, times(0)).processDoneFiles(t.jobId); handleEvent(jheh, new JobHistoryEvent(t.jobId, new JobFinishedEvent( TypeConverter.fromYarn(t.jobId), 0, 0, 0, 0, 0, 0, 0, new Counters(), new Counters(), new Counters())));
result = new JobInitedEvent(); break; case JOB_FINISHED: result = new JobFinishedEvent(); break; case JOB_PRIORITY_CHANGED: result = new JobPriorityChangeEvent(); break;
result = new JobInitedEvent(); break; case JOB_FINISHED: result = new JobFinishedEvent(); break; case JOB_PRIORITY_CHANGED: result = new JobPriorityChangeEvent(); break;
result = new JobInitedEvent(); break; case JOB_FINISHED: result = new JobFinishedEvent(); break; case JOB_PRIORITY_CHANGED: result = new JobPriorityChangeEvent(); break;
new JobFinishedEvent(this.status.getJobID(), this.finishTime, this.finishedMapTasks,this.finishedReduceTasks, failedMapTasks,
result = new JobInitedEvent(); break; case JOB_FINISHED: result = new JobFinishedEvent(); break; case JOB_PRIORITY_CHANGED: result = new JobPriorityChangeEvent(); break;
result = new JobInitedEvent(); break; case JOB_FINISHED: result = new JobFinishedEvent(); break; case JOB_PRIORITY_CHANGED: result = new JobPriorityChangeEvent(); break;