private MapReduceJobState getMapReduceJobState(MapReduceJob job, JobClient jobClient) throws Exception { RunningJob runningJob = getRunningJob(job, jobClient); JobID jobID = runningJob.getID(); TaskReport[] mapTaskReport = jobClient.getMapTaskReports(jobID); TaskReport[] reduceTaskReport = jobClient.getReduceTaskReports(jobID); return new MapReduceJobState(runningJob, mapTaskReport, reduceTaskReport); }
private boolean updateJobState() throws IOException { if (jobProgress == null) { jobProgress = new MapReduceJobState( rj, jobClient.getMapTaskReports(jobId), jobClient.getReduceTaskReports(jobId)); return true; } boolean complete = rj.isComplete(); boolean successful = rj.isSuccessful(); float mapProgress = rj.mapProgress(); float reduceProgress = rj.reduceProgress(); boolean update = !( jobProgress.isComplete() == complete && jobProgress.isSuccessful() == successful && AmbroseHiveUtil.isEqual(jobProgress.getMapProgress(), mapProgress) && AmbroseHiveUtil.isEqual(jobProgress.getReduceProgress(), reduceProgress) ); //do progress report only if necessary if (update) { jobProgress = new MapReduceJobState( rj, jobClient.getMapTaskReports(jobId), jobClient.getReduceTaskReports(jobId)); jobProgress.setJobLastUpdateTime(System.currentTimeMillis()); } return update; }
TaskReport[] reducers = jc.getReduceTaskReports(rj.getID()); if (reducers == null) { logReducer = "no information for number of reducers. ";
TaskReport[] reducers = jc.getReduceTaskReports(rj.getID()); if (reducers == null) { logReducer = "no information for number of reducers. ";
/**@deprecated Applications should rather use {@link #getReduceTaskReports(JobID)}*/ @Deprecated public TaskReport[] getReduceTaskReports(String jobId) throws IOException { return getReduceTaskReports(JobID.forName(jobId)); }
/**@deprecated Applications should rather use {@link #getReduceTaskReports(JobID)}*/ @Deprecated public TaskReport[] getReduceTaskReports(String jobId) throws IOException { return getReduceTaskReports(JobID.forName(jobId)); }
/**@deprecated Applications should rather use {@link #getReduceTaskReports(JobID)}*/ @Deprecated public TaskReport[] getReduceTaskReports(String jobId) throws IOException { return getReduceTaskReports(JobID.forName(jobId)); }
/**@deprecated Applications should rather use {@link #getReduceTaskReports(JobID)}*/ @Deprecated public TaskReport[] getReduceTaskReports(String jobId) throws IOException { return getReduceTaskReports(JobID.forName(jobId)); }
/**@deprecated Applications should rather use {@link #getReduceTaskReports(JobID)}*/ @Deprecated public TaskReport[] getReduceTaskReports(String jobId) throws IOException { return getReduceTaskReports(JobID.forName(jobId)); }
/**@deprecated Applications should rather use {@link #getReduceTaskReports(JobID)}*/ @Deprecated public TaskReport[] getReduceTaskReports(String jobId) throws IOException { return getReduceTaskReports(JobID.forName(jobId)); }
/**@deprecated Applications should rather use {@link #getReduceTaskReports(JobID)}*/ @Deprecated public TaskReport[] getReduceTaskReports(String jobId) throws IOException { return getReduceTaskReports(JobID.forName(jobId)); }
private static DescriptiveStatistics getReduceRuntimes(JobClient client, JobID job, TwoNestedMap<String, String, Long> counters) throws IOException { if(shouldFetch(counters, JobCounter.TOTAL_LAUNCHED_REDUCES)) { return getRuntimes(client.getReduceTaskReports(job)); } return new DescriptiveStatistics(new double[]{0}); }
reports = getMapTaskReports(jobId); } else if (type.equals("reduce")) { reports = getReduceTaskReports(jobId); } else if (type.equals("setup")) { reports = getSetupTaskReports(jobId);
reports = jobClient.getMapTaskReports(jobid); }else{ reports = jobClient.getReduceTaskReports(jobid);
System.out.println("Task ID: "+tr.getTaskID()+" Start TIme: "+StringUtils.getFormattedTimeWithDiff(dateFormat, tr.getStartTime(), 0)+" Finish Time: "+StringUtils.getFormattedTimeWithDiff(dateFormat, tr.getFinishTime(), tr.getStartTime())); TaskReport[] reducetaskreports = jobClient.getReduceTaskReports(js.getJobID()); for(TaskReport tr: reducetaskreports){ System.out.println("Task ID: "+tr.getTaskID()+" Start TIme: "+StringUtils.getFormattedTimeWithDiff(dateFormat, tr.getStartTime(), 0)+" Finish Time: "+StringUtils.getFormattedTimeWithDiff(dateFormat, tr.getFinishTime(), tr.getStartTime()));
TaskReport reduceReports[] = jc.getReduceTaskReports(rJob.getID()); for (TaskReport reduceReport : reduceReports) { if (reduceReport.getCurrentStatus() == TIPStatus.RUNNING) {
/** * @param runId for which information will be returned. * @return a {@link MRJobInfo} containing information about a particular MapReduce program run. * @throws IOException if there is failure to communicate through the JobClient. * @throws NotFoundException if a Job with the given runId is not found. */ @Override public MRJobInfo getMRJobInfo(Id.Run runId) throws IOException, NotFoundException { Preconditions.checkArgument(ProgramType.MAPREDUCE.equals(runId.getProgram().getType())); JobClient jobClient = new JobClient(hConf); JobStatus[] jobs = jobClient.getAllJobs(); JobStatus thisJob = findJobForRunId(jobs, runId.toEntityId()); RunningJob runningJob = jobClient.getJob(thisJob.getJobID()); if (runningJob == null) { throw new IllegalStateException(String.format("JobClient returned null for RunId: '%s', JobId: '%s'", runId, thisJob.getJobID())); } Counters counters = runningJob.getCounters(); TaskReport[] mapTaskReports = jobClient.getMapTaskReports(thisJob.getJobID()); TaskReport[] reduceTaskReports = jobClient.getReduceTaskReports(thisJob.getJobID()); return new MRJobInfo(runningJob.mapProgress(), runningJob.reduceProgress(), groupToMap(counters.getGroup(TaskCounter.class.getName())), toMRTaskInfos(mapTaskReports), toMRTaskInfos(reduceTaskReports), true); }
/** * @param runId for which information will be returned. * @return a {@link MRJobInfo} containing information about a particular MapReduce program run. * @throws IOException if there is failure to communicate through the JobClient. * @throws NotFoundException if a Job with the given runId is not found. */ @Override public MRJobInfo getMRJobInfo(Id.Run runId) throws IOException, NotFoundException { Preconditions.checkArgument(ProgramType.MAPREDUCE.equals(runId.getProgram().getType())); JobClient jobClient = new JobClient(hConf); JobStatus[] jobs = jobClient.getAllJobs(); JobStatus thisJob = findJobForRunId(jobs, runId.toEntityId()); RunningJob runningJob = jobClient.getJob(thisJob.getJobID()); if (runningJob == null) { throw new IllegalStateException(String.format("JobClient returned null for RunId: '%s', JobId: '%s'", runId, thisJob.getJobID())); } Counters counters = runningJob.getCounters(); TaskReport[] mapTaskReports = jobClient.getMapTaskReports(thisJob.getJobID()); TaskReport[] reduceTaskReports = jobClient.getReduceTaskReports(thisJob.getJobID()); return new MRJobInfo(runningJob.mapProgress(), runningJob.reduceProgress(), groupToMap(counters.getGroup(TaskCounter.class.getName())), toMRTaskInfos(mapTaskReports), toMRTaskInfos(reduceTaskReports), true); }
@SuppressWarnings("deprecation") private long getTaskCounterUsage (JobClient client, JobID id, int numReports, int taskId, TaskType type) throws Exception { TaskReport[] reports = null; if (TaskType.MAP.equals(type)) { reports = client.getMapTaskReports(id); } else if (TaskType.REDUCE.equals(type)) { reports = client.getReduceTaskReports(id); } assertNotNull("No reports found for task type '" + type.name() + "' in job " + id, reports); // make sure that the total number of reports match the expected assertEquals("Mismatch in task id", numReports, reports.length); Counters counters = reports[taskId].getCounters(); return counters.getCounter(TaskCounter.COMMITTED_HEAP_BYTES); }
reports = client.getMapTaskReports(jobid); assertEquals("number of maps", 1, reports.length); reports = client.getReduceTaskReports(jobid); assertEquals("number of reduces", 1, reports.length); reports = client.getCleanupTaskReports(jobid);