private static void addTaskInfo(JobHistoryParser.JobInfo job, TaskType type, int id, TaskStatus.State status) { JobHistoryParser.TaskInfo task = new JobHistoryParser.TaskInfo(); task.taskId = new TaskID(job.getJobId(), type, id); task.startTime = job.getLaunchTime() + id * 1000; task.finishTime = task.startTime + id * 1000; task.taskType = type; task.counters = createCounters(); task.status = status.name(); task.attemptsMap = new HashMap<>(); addTaskAttemptInfo(task, 1); job.tasksMap.put(task.getTaskId(), task); }
private static void addTaskAttemptInfo( JobHistoryParser.TaskInfo task, int id) { JobHistoryParser.TaskAttemptInfo attempt = new JobHistoryParser.TaskAttemptInfo(); attempt.attemptId = new TaskAttemptID( TaskID.downgrade(task.getTaskId()), id); attempt.startTime = task.getStartTime(); attempt.finishTime = task.getFinishTime(); attempt.shuffleFinishTime = task.getFinishTime(); attempt.sortFinishTime = task.getFinishTime(); attempt.mapFinishTime = task.getFinishTime(); attempt.status = task.getTaskStatus(); attempt.taskType = task.getTaskType(); attempt.trackerName = "localhost"; attempt.httpPort = 1234; attempt.hostname = "localhost"; task.attemptsMap.put(attempt.getAttemptId(), attempt); } }
private static JobHistoryParser.JobInfo createJobInfo() { JobHistoryParser.JobInfo job = new JobHistoryParser.JobInfo(); job.submitTime = 1317928501754L; job.finishTime = job.submitTime + 15000; job.succeededReduces = 1; job.jobStatus = JobStatus.State.SUCCEEDED.name(); job.totalCounters = createCounters(); job.mapCounters = createCounters(); job.reduceCounters = createCounters(); job.tasksMap = new HashMap<>(); addTaskInfo(job, TaskType.JOB_SETUP, 1, TaskStatus.State.SUCCEEDED); addTaskInfo(job, TaskType.MAP, 2, TaskStatus.State.FAILED); addTaskInfo(job, TaskType.MAP, 3, TaskStatus.State.SUCCEEDED); addTaskInfo(job, TaskType.MAP, 4, TaskStatus.State.SUCCEEDED); addTaskInfo(job, TaskType.MAP, 5, TaskStatus.State.SUCCEEDED); addTaskInfo(job, TaskType.MAP, 6, TaskStatus.State.SUCCEEDED); addTaskInfo(job, TaskType.MAP, 7, TaskStatus.State.SUCCEEDED); addTaskInfo(job, TaskType.REDUCE, 8, TaskStatus.State.SUCCEEDED); addTaskInfo(job, TaskType.JOB_CLEANUP, 9, TaskStatus.State.SUCCEEDED); return job;
private FakeEvent getJobKilledEvent() { FakeEvent result = new FakeEvent(EventType.JOB_KILLED); JobUnsuccessfulCompletion datum = new JobUnsuccessfulCompletion(); datum.setFinishedMaps(1); datum.setFinishedReduces(2); datum.setFinishTime(3L); datum.setJobid("ID"); datum.setJobStatus("STATUS"); datum.setDiagnostics(JobImpl.JOB_KILLED_DIAG); result.setDatum(datum); return result; }
/** * Test {@link TaskAttemptStartedEvent} for various task types. */ private static void testAttemptStartedEventForTypes(EventType expected, TaskAttemptID id, TaskType[] types) { for (TaskType t : types) { TaskAttemptStartedEvent tase = new TaskAttemptStartedEvent(id, t, 0L, "", 0); assertEquals(expected, tase.getEventType()); } }
private FakeEvent getJobPriorityChangedEvent() { FakeEvent result = new FakeEvent(EventType.JOB_PRIORITY_CHANGED); JobPriorityChange datum = new JobPriorityChange(); datum.setJobid("ID"); datum.setPriority("priority"); result.setDatum(datum); return result; }
private FakeEvent getTaskUpdatedEvent() { FakeEvent result = new FakeEvent(EventType.TASK_UPDATED); TaskUpdated datum = new TaskUpdated(); datum.setFinishTime(2L); datum.setTaskid("ID"); result.setDatum(datum); return result; }
private static JobHistoryParser.JobInfo createJobInfo2() { JobHistoryParser.JobInfo job = new JobHistoryParser.JobInfo(); job.submitTime = 1317928501754L; job.finishTime = job.submitTime + 15000; job.jobid = JobID.forName("job_1317928501754_0001"); job.username = "test"; job.jobname = "Dupe counter output"; job.jobQueueName = "root.test"; job.jobConfPath = "/tmp/job.xml"; job.launchTime = job.submitTime + 1000; job.totalMaps = 1; job.totalReduces = 0; job.failedMaps = 0; job.failedReduces = 0; job.succeededMaps = 1; job.succeededReduces = 0; job.jobStatus = JobStatus.State.SUCCEEDED.name(); job.totalCounters = createDeprecatedCounters(); job.mapCounters = createDeprecatedCounters(); job.reduceCounters = createDeprecatedCounters(); job.tasksMap = new HashMap<>(); addTaskInfo(job, TaskType.JOB_SETUP, 1, TaskStatus.State.SUCCEEDED); addTaskInfo(job, TaskType.MAP, 2, TaskStatus.State.SUCCEEDED); addTaskInfo(job, TaskType.JOB_CLEANUP, 3, TaskStatus.State.SUCCEEDED); return job; }
private FakeEvent getReduceAttemptKilledEvent() { FakeEvent result = new FakeEvent(EventType.REDUCE_ATTEMPT_KILLED); result.setDatum(getTaskAttemptUnsuccessfulCompletion()); return result; }
@Test public void testJSONPrinter() throws Exception { JobHistoryParser.JobInfo job = createJobInfo(); JSONHistoryViewerPrinter printer = new JSONHistoryViewerPrinter(job, false, "http://"); String outStr = run(printer); JSONAssert.assertEquals("{\n" + " \"counters\": {\n" +
@Test public void testHumanPrinter() throws Exception { JobHistoryParser.JobInfo job = createJobInfo(); HumanReadableHistoryViewerPrinter printer = new HumanReadableHistoryViewerPrinter(job, false, "http://", TimeZone.getTimeZone("GMT")); String outStr = run(printer); Assert.assertEquals("\n" + "Hadoop job: job_1317928501754_0001\n" +
@Test public void testJSONDupePrinter() throws Exception { JobHistoryParser.JobInfo job = createJobInfo2(); // Counters are part of the overview and task info // Tasks only have bogus counters in the test if that is changed printAll // must then be kept as false for this test to pass JSONHistoryViewerPrinter printer = new JSONHistoryViewerPrinter(job, false, "http://"); String outStr = run(printer); // We are not interested in anything but the duplicate counter int count1 = outStr.indexOf( "\"counterName\":\"MAP_INPUT_RECORDS\""); Assert.assertNotEquals("First counter occurrence not found", -1, count1); int count2 = outStr.indexOf( "\"counterName\":\"MAP_INPUT_RECORDS\"", count1 + 1); Assert.assertEquals("Duplicate counter found at: " + count1 + " and " + count2, -1, count2); }
@Test public void testHumanDupePrinter() throws Exception { JobHistoryParser.JobInfo job = createJobInfo2(); // Counters are only part of the overview so printAll can be false or true // this does not affect the test, task counters are not printed HumanReadableHistoryViewerPrinter printer = new HumanReadableHistoryViewerPrinter(job, false, "http://", TimeZone.getTimeZone("GMT")); String outStr = run(printer); // We are not interested in anything but the duplicate counter int count1 = outStr.indexOf( "|Map-Reduce Framework |Map input records |"); Assert.assertNotEquals("First counter occurrence not found", -1, count1); int count2 = outStr.indexOf( "|Map-Reduce Framework |Map input records |", count1 + 1); Assert.assertEquals("Duplicate counter found at: " + count1 + " and " + count2, -1, count2); }
private String run(HistoryViewerPrinter printer) throws Exception { ByteArrayOutputStream boas = new ByteArrayOutputStream(); PrintStream out = new PrintStream(boas, true); printer.print(out); out.close(); String outStr = boas.toString("UTF-8"); LOG.info("out = " + outStr); return outStr; }
private FakeEvent getCleanupAttemptFiledEvent() { FakeEvent result = new FakeEvent(EventType.CLEANUP_ATTEMPT_FAILED); result.setDatum(getTaskAttemptUnsuccessfulCompletion()); return result; }
@Test public void testJSONPrinterAll() throws Exception { JobHistoryParser.JobInfo job = createJobInfo(); JSONHistoryViewerPrinter printer = new JSONHistoryViewerPrinter(job, true, "http://"); String outStr = run(printer); JSONAssert.assertEquals("{\n" + " \"counters\": {\n" +
@Test public void testHumanPrinterAll() throws Exception { JobHistoryParser.JobInfo job = createJobInfo(); HumanReadableHistoryViewerPrinter printer = new HumanReadableHistoryViewerPrinter(job, true, "http://", TimeZone.getTimeZone("GMT")); String outStr = run(printer); if (System.getProperty("java.version").startsWith("1.7")) { Assert.assertEquals("\n" +
private FakeEvent getSetupAttemptKilledEvent() { FakeEvent result = new FakeEvent(EventType.SETUP_ATTEMPT_KILLED); result.setDatum(getTaskAttemptUnsuccessfulCompletion()); return result; }
private FakeEvent getCleanupAttemptKilledEvent() { FakeEvent result = new FakeEvent(EventType.CLEANUP_ATTEMPT_KILLED); result.setDatum(getTaskAttemptUnsuccessfulCompletion()); return result; }
private FakeEvent getSetupAttemptFieledEvent() { FakeEvent result = new FakeEvent(EventType.SETUP_ATTEMPT_FAILED); result.setDatum(getTaskAttemptUnsuccessfulCompletion()); return result; }