public static JobReport newJobReport(JobId jobId, String jobName, String userName, JobState state, long submitTime, long startTime, long finishTime, float setupProgress, float mapProgress, float reduceProgress, float cleanupProgress, String jobFile, List<AMInfo> amInfos, boolean isUber, String diagnostics) { JobReport report = Records.newRecord(JobReport.class); report.setJobId(jobId); report.setJobName(jobName); report.setUser(userName); report.setJobState(state); report.setSubmitTime(submitTime); report.setStartTime(startTime); report.setFinishTime(finishTime); report.setSetupProgress(setupProgress); report.setCleanupProgress(cleanupProgress); report.setMapProgress(mapProgress); report.setReduceProgress(reduceProgress); report.setJobFile(jobFile); report.setAMInfos(amInfos); report.setIsUber(isUber); report.setDiagnostics(diagnostics); return report; }
public static JobStatus fromYarn(JobReport jobreport, String trackingUrl) { JobPriority jobPriority = JobPriority.NORMAL; JobStatus jobStatus = new org.apache.hadoop.mapred.JobStatus( fromYarn(jobreport.getJobId()), jobreport.getSetupProgress(), jobreport .getMapProgress(), jobreport.getReduceProgress(), jobreport .getCleanupProgress(), fromYarn(jobreport.getJobState()), jobPriority, jobreport.getUser(), jobreport.getJobName(), jobreport .getJobFile(), trackingUrl, jobreport.isUber()); jobStatus.setStartTime(jobreport.getStartTime()); jobStatus.setFinishTime(jobreport.getFinishTime()); jobStatus.setFailureInfo(jobreport.getDiagnostics()); return jobStatus; }
public PartialJob(JobIndexInfo jobIndexInfo, JobId jobId) { this.jobIndexInfo = jobIndexInfo; this.jobId = jobId; jobReport = RecordFactoryProvider.getRecordFactory(null).newRecordInstance(JobReport.class); jobReport.setSubmitTime(jobIndexInfo.getSubmitTime()); jobReport.setStartTime(jobIndexInfo.getJobStartTime()); jobReport.setFinishTime(jobIndexInfo.getFinishTime()); jobReport.setJobState(getState()); }
@Override public GetJobReportResponse getJobReport(GetJobReportRequest request) throws IOException { JobReport jobReport = recordFactory.newRecordInstance(JobReport.class); jobReport.setJobId(request.getJobId()); jobReport.setJobState(jobState); jobReport.setUser(applicationReport.getUser()); jobReport.setStartTime(applicationReport.getStartTime()); jobReport.setDiagnostics(applicationReport.getDiagnostics()); jobReport.setJobName(applicationReport.getName()); jobReport.setTrackingUrl(applicationReport.getTrackingUrl()); jobReport.setFinishTime(applicationReport.getFinishTime()); GetJobReportResponse resp = recordFactory.newRecordInstance(GetJobReportResponse.class); resp.setJobReport(jobReport); return resp; }
public static JobReport newJobReport(JobId id) { JobReport report = Records.newRecord(JobReport.class); report.setJobId(id); report.setSubmitTime(System.currentTimeMillis()-DT); report .setStartTime(System.currentTimeMillis() - (int) (Math.random() * DT)); report.setFinishTime(System.currentTimeMillis() + (int) (Math.random() * DT) + 1); report.setMapProgress((float) Math.random()); report.setReduceProgress((float) Math.random()); report.setJobState(JOB_STATES.next()); return report; }
@Override public GetJobReportResponse getJobReport(GetJobReportRequest request) throws IOException { amContact = true; JobReport jobReport = recordFactory.newRecordInstance(JobReport.class); jobReport.setJobId(request.getJobId()); jobReport.setJobState(JobState.RUNNING); jobReport.setJobName("TestClientRedirect-jobname"); jobReport.setUser("TestClientRedirect-user"); jobReport.setStartTime(0L); jobReport.setFinishTime(1L); GetJobReportResponse response = recordFactory .newRecordInstance(GetJobReportResponse.class); response.setJobReport(jobReport); return response; }
private GetJobReportResponse getJobReportResponseFromHistoryServer() { GetJobReportResponse jobReportResponse = Records .newRecord(GetJobReportResponse.class); JobReport jobReport = Records.newRecord(JobReport.class); jobReport.setJobId(jobId); jobReport.setJobState(JobState.SUCCEEDED); jobReport.setMapProgress(1.0f); jobReport.setReduceProgress(1.0f); jobReport.setJobFile("TestJobFilePath"); jobReport.setTrackingUrl("http://TestTrackingUrl"); jobReportResponse.setJobReport(jobReport); return jobReportResponse; }
public JobInfo(Job job, Boolean hasAccess) { this.id = MRApps.toString(job.getID()); JobReport report = job.getReport(); this.startTime = report.getStartTime(); this.finishTime = report.getFinishTime(); this.elapsedTime = Times.elapsed(this.startTime, this.finishTime); if (this.elapsedTime == -1) { this.mapsTotal = job.getTotalMaps(); this.mapsCompleted = job.getCompletedMaps(); this.mapProgress = report.getMapProgress() * 100; this.mapProgressPercent = percent(report.getMapProgress()); this.reducesTotal = job.getTotalReduces(); this.reducesCompleted = job.getCompletedReduces(); this.reduceProgress = report.getReduceProgress() * 100; this.reduceProgressPercent = percent(report.getReduceProgress());
if (report.getStartTime() < sBegin || report.getStartTime() > sEnd) { continue; if (report.getFinishTime() < fBegin || report.getFinishTime() > fEnd) { continue; if (jobState != null && jobState != report.getJobState()) { continue;
this.reducesTotal = job.getTotalReduces(); this.reducesCompleted = job.getCompletedReduces(); this.submitTime = report.getSubmitTime(); this.startTime = report.getStartTime(); this.finishTime = report.getFinishTime(); this.name = job.getName().toString(); this.queue = job.getQueueName();
private void verifyJobReport(JobReport jobReport, JobId jobId) { List<AMInfo> amInfos = jobReport.getAMInfos(); Assert.assertEquals(1, amInfos.size()); AMInfo amInfo = amInfos.get(0); ApplicationAttemptId appAttemptId = ApplicationAttemptId.newInstance(jobId.getAppId(), 1); ContainerId amContainerId = ContainerId.newContainerId(appAttemptId, 1); Assert.assertEquals(appAttemptId, amInfo.getAppAttemptId()); Assert.assertEquals(amContainerId, amInfo.getContainerId()); Assert.assertTrue(jobReport.getSubmitTime() > 0); Assert.assertTrue(jobReport.getStartTime() > 0 && jobReport.getStartTime() >= jobReport.getSubmitTime()); Assert.assertTrue(jobReport.getFinishTime() > 0 && jobReport.getFinishTime() >= jobReport.getStartTime()); }
private void sendJobEndNotify(JobEndNotifier notifier) { try { LOG.info("Job end notification started for jobID : " + job.getReport().getJobId()); // If unregistration fails, the final state is unavailable. However, // at the last AM Retry, the client will finally be notified FAILED // from RM, so we should let users know FAILED via notifier as well JobReport report = job.getReport(); if (!context.hasSuccessfullyUnregistered()) { report.setJobState(JobState.FAILED); } notifier.notify(report); } catch (InterruptedException ie) { LOG.warn("Job end notification interrupted for jobID : " + job.getReport().getJobId(), ie); } }
public void waitForState(Job job, JobState finalState) throws Exception { int timeoutSecs = 0; JobReport report = job.getReport(); while (!finalState.equals(report.getJobState()) && timeoutSecs++ < 20) { System.out.println("Job State is : " + report.getJobState() + " Waiting for state : " + finalState + " map progress : " + report.getMapProgress() + " reduce progress : " + report.getReduceProgress()); report = job.getReport(); Thread.sleep(500); } System.out.println("Job State is : " + report.getJobState()); Assert.assertEquals("Job state is not correct (timedout)", finalState, job.getState()); }
userUrl = userUrl.replace(JOB_ID, jobReport.getJobId().toString()); userUrl = userUrl.replace(JOB_STATUS, jobReport.getJobState().toString()); Log.warn("Job end notification failed to notify : " + urlToNotify); } else { Log.info("Job end notification succeeded for " + jobReport.getJobId());
private GetJobReportResponse getJobReportResponse() { GetJobReportResponse jobReportResponse = Records .newRecord(GetJobReportResponse.class); JobReport jobReport = Records.newRecord(JobReport.class); jobReport.setJobId(jobId); jobReport.setJobState(JobState.SUCCEEDED); jobReportResponse.setJobReport(jobReport); return jobReportResponse; }
public JobStatus getJobStatus(JobID oldJobID) throws IOException { org.apache.hadoop.mapreduce.v2.api.records.JobId jobId = TypeConverter.toYarn(oldJobID); GetJobReportRequest request = recordFactory.newRecordInstance(GetJobReportRequest.class); request.setJobId(jobId); JobReport report = ((GetJobReportResponse) invoke("getJobReport", GetJobReportRequest.class, request)).getJobReport(); JobStatus jobStatus = null; if (report != null) { if (StringUtils.isEmpty(report.getJobFile())) { String jobFile = MRApps.getJobFile(conf, report.getUser(), oldJobID); report.setJobFile(jobFile); } String historyTrackingUrl = report.getTrackingUrl(); String url = StringUtils.isNotEmpty(historyTrackingUrl) ? historyTrackingUrl : trackingUrl; jobStatus = TypeConverter.fromYarn(report, url); } return jobStatus; }
GetJobReportRequest.class, request)).getJobReport(); if (EnumSet.of(JobState.SUCCEEDED, JobState.FAILED, JobState.KILLED, JobState.ERROR).contains(report.getJobState())) { if (oldTaskAttemptID != null) { GetTaskAttemptReportRequest taRequest = .getApplicationId().toString(), NodeId.newInstance(taReport.getNodeManagerHost(), taReport.getNodeManagerPort()).toString(), report.getUser()); } else { if (report.getAMInfos() == null || report.getAMInfos().size() == 0) { throw new IOException("Unable to get log information for job: " + oldJobID); AMInfo amInfo = report.getAMInfos().get(report.getAMInfos().size() - 1); return new LogParams( amInfo.getContainerId().toString(), amInfo.getAppAttemptId().getApplicationId().toString(), NodeId.newInstance(amInfo.getNodeManagerHost(), amInfo.getNodeManagerPort()).toString(), report.getUser());
public void verifyCompleted() { for (Job job : getContext().getAllJobs().values()) { JobReport jobReport = job.getReport(); System.out.println("Job start time :" + jobReport.getStartTime()); System.out.println("Job finish time :" + jobReport.getFinishTime()); Assert.assertTrue("Job start time is not less than finish time", jobReport.getStartTime() <= jobReport.getFinishTime()); Assert.assertTrue("Job finish time is in future", jobReport.getFinishTime() <= System.currentTimeMillis()); for (Task task : job.getTasks().values()) { TaskReport taskReport = task.getReport(); System.out.println("Task start time : " + taskReport.getStartTime()); System.out.println("Task finish time : " + taskReport.getFinishTime()); Assert.assertTrue("Task start time is not less than finish time", taskReport.getStartTime() <= taskReport.getFinishTime()); for (TaskAttempt attempt : task.getAttempts().values()) { TaskAttemptReport attemptReport = attempt.getReport(); Assert.assertTrue("Attempt start time is not less than finish time", attemptReport.getStartTime() <= attemptReport.getFinishTime()); } } } }
private void verifyJobReport(JobReport jr) { Assert.assertNotNull("JobReport is null", jr); List<AMInfo> amInfos = jr.getAMInfos(); Assert.assertEquals(1, amInfos.size()); Assert.assertEquals(JobState.RUNNING, jr.getJobState()); AMInfo amInfo = amInfos.get(0); Assert.assertEquals(MRApp.NM_HOST, amInfo.getNodeManagerHost()); Assert.assertEquals(MRApp.NM_PORT, amInfo.getNodeManagerPort()); Assert.assertEquals(MRApp.NM_HTTP_PORT, amInfo.getNodeManagerHttpPort()); Assert.assertEquals(1, amInfo.getAppAttemptId().getAttemptId()); Assert.assertEquals(1, amInfo.getContainerId().getApplicationAttemptId() .getAttemptId()); Assert.assertTrue(amInfo.getStartTime() > 0); Assert.assertEquals(false, jr.isUber()); }
@Override public JobState getState() { return report.getJobState(); }