public double getProgress() throws IOException { if(_runningJob == null) return 0.0; else return (double) (_runningJob.mapProgress() + _runningJob.reduceProgress()) / 2.0d; }
/** * Update counters relevant to this task. */ private void updateCounters(Counters ctrs, RunningJob rj) throws IOException { lastMapProgress = mapProgress; lastReduceProgress = reduceProgress; mapProgress = Math.round(rj.mapProgress() * 100); mapProgress = mapProgress == 100 ? (int)Math.floor(rj.mapProgress() * 100) : mapProgress; reduceProgress = Math.round(rj.reduceProgress() * 100); reduceProgress = reduceProgress == 100 ? (int)Math.floor(rj.reduceProgress() * 100) : reduceProgress; task.taskCounters.put("CNTR_NAME_" + task.getId() + "_MAP_PROGRESS", Long.valueOf(mapProgress)); task.taskCounters.put("CNTR_NAME_" + task.getId() + "_REDUCE_PROGRESS", Long.valueOf(reduceProgress)); if (SessionState.get() != null) { final float progress = (rj.mapProgress() + rj.reduceProgress()) * 0.5f; SessionState.get().updateProgressedPercentage(progress); } }
/** * Update counters relevant to this task. */ private void updateCounters(Counters ctrs, RunningJob rj) throws IOException { lastMapProgress = mapProgress; lastReduceProgress = reduceProgress; mapProgress = Math.round(rj.mapProgress() * 100); mapProgress = mapProgress == 100 ? (int)Math.floor(rj.mapProgress() * 100) : mapProgress; reduceProgress = Math.round(rj.reduceProgress() * 100); reduceProgress = reduceProgress == 100 ? (int)Math.floor(rj.reduceProgress() * 100) : reduceProgress; task.taskCounters.put("CNTR_NAME_" + task.getId() + "_MAP_PROGRESS", Long.valueOf(mapProgress)); task.taskCounters.put("CNTR_NAME_" + task.getId() + "_REDUCE_PROGRESS", Long.valueOf(reduceProgress)); if (SessionState.get() != null) { final float progress = (rj.mapProgress() + rj.reduceProgress()) * 0.5f; SessionState.get().updateProgressedPercentage(progress); } }
private void updateMapRedStatsJson(MapRedStats stats, RunningJob rj) throws IOException, JSONException { if (statsJSON == null) { statsJSON = new JSONObject(); } if (stats != null) { if (stats.getNumMap() >= 0) { statsJSON.put(NUMBER_OF_MAPPERS, stats.getNumMap()); } if (stats.getNumReduce() >= 0) { statsJSON.put(NUMBER_OF_REDUCERS, stats.getNumReduce()); } if (stats.getCounters() != null) { statsJSON.put(COUNTERS, getCountersJson(stats.getCounters())); } } if (rj != null) { statsJSON.put(JOB_ID, rj.getID().toString()); statsJSON.put(JOB_FILE, rj.getJobFile()); statsJSON.put(TRACKING_URL, rj.getTrackingURL()); statsJSON.put(MAP_PROGRESS, Math.round(rj.mapProgress() * 100)); statsJSON.put(REDUCE_PROGRESS, Math.round(rj.reduceProgress() * 100)); statsJSON.put(CLEANUP_PROGRESS, Math.round(rj.cleanupProgress() * 100)); statsJSON.put(SETUP_PROGRESS, Math.round(rj.setupProgress() * 100)); statsJSON.put(COMPLETE, rj.isComplete()); statsJSON.put(SUCCESSFUL, rj.isSuccessful()); } }
isComplete = runningJob.isComplete(); isSuccessful = runningJob.isSuccessful(); mapProgress = runningJob.mapProgress(); reduceProgress = runningJob.reduceProgress(); totalMappers = mapTaskReport.length;
private boolean updateJobState() throws IOException { if (jobProgress == null) { jobProgress = new MapReduceJobState( rj, jobClient.getMapTaskReports(jobId), jobClient.getReduceTaskReports(jobId)); return true; } boolean complete = rj.isComplete(); boolean successful = rj.isSuccessful(); float mapProgress = rj.mapProgress(); float reduceProgress = rj.reduceProgress(); boolean update = !( jobProgress.isComplete() == complete && jobProgress.isSuccessful() == successful && AmbroseHiveUtil.isEqual(jobProgress.getMapProgress(), mapProgress) && AmbroseHiveUtil.isEqual(jobProgress.getReduceProgress(), reduceProgress) ); //do progress report only if necessary if (update) { jobProgress = new MapReduceJobState( rj, jobClient.getMapTaskReports(jobId), jobClient.getReduceTaskReports(jobId)); jobProgress.setJobLastUpdateTime(System.currentTimeMillis()); } return update; }
@Override public float mapProgress() throws IOException { return delegate.mapProgress(); }
/** * Get the <i>progress</i> of the job's map-tasks, as a float between 0.0 * and 1.0. When all map tasks have completed, the function returns 1.0. * * @return the progress of the job's map-tasks. * @throws IOException */ public float mapProgress() throws IOException { ensureState(JobState.RUNNING); return info.mapProgress(); }
protected boolean internalIsStartedRunning() { if( runningJob == null ) return false; try { return runningJob.mapProgress() > 0; } catch( IOException exception ) { flowStep.logWarn( "unable to test for map progress", exception ); return false; } }
protected boolean internalIsStartedRunning() { if( runningJob == null ) return false; try { return runningJob.mapProgress() > 0; } catch( IOException exception ) { flowStep.logWarn( "unable to test for map progress", exception ); return false; } }
/** * Update counters relevant to this task. */ private void updateCounters(Counters ctrs, RunningJob rj) throws IOException { lastMapProgress = mapProgress; lastReduceProgress = reduceProgress; mapProgress = Math.round(rj.mapProgress() * 100); mapProgress = mapProgress == 100 ? (int)Math.floor(rj.mapProgress() * 100) : mapProgress; reduceProgress = Math.round(rj.reduceProgress() * 100); reduceProgress = reduceProgress == 100 ? (int)Math.floor(rj.reduceProgress() * 100) : reduceProgress; task.taskCounters.put("CNTR_NAME_" + task.getId() + "_MAP_PROGRESS", Long.valueOf(mapProgress)); task.taskCounters.put("CNTR_NAME_" + task.getId() + "_REDUCE_PROGRESS", Long.valueOf(reduceProgress)); }
JobClient theJobClient = new JobClient(new InetSocketAddress("your.job.tracker", 8021), new Configuration()); RunningJob theJob = theJobClient.getJob("job_id_string"); // caution, deprecated float mapProgress = theJob.mapProgress(); // similar for reduceProgress // etc (see RunningJob)
JobClient jobClient = new JobClient(jobConf); RunningJob job = jobClient.submitJob(jobConf); float mapProgress = job.mapProgress(); float redProgress = job.reduceProgress();
/** * Update counters relevant to this task. */ private void updateCounters(Counters ctrs, RunningJob rj) throws IOException { mapProgress = Math.round(rj.mapProgress() * 100); reduceProgress = Math.round(rj.reduceProgress() * 100); taskCounters.put("CNTR_NAME_" + getId() + "_MAP_PROGRESS", Long.valueOf(mapProgress)); taskCounters.put("CNTR_NAME_" + getId() + "_REDUCE_PROGRESS", Long.valueOf(reduceProgress)); if (ctrs == null) { // hadoop might return null if it cannot locate the job. // we may still be able to retrieve the job status - so ignore return; } for (Operator<? extends Serializable> op : work.getAliasToWork().values()) { op.updateCounters(ctrs); } if (work.getReducer() != null) { work.getReducer().updateCounters(ctrs); } }
progress = jc.getJob(rJob.getID()).mapProgress(); } else { progress = jc.getJob(rJob.getID()).reduceProgress();
/** * Returns the progress of a Job j which is part of a submitted JobControl object. The progress is for this Job. So * it has to be scaled down by the number of jobs that are present in the JobControl. * * @param cjob * - The Job for which progress is required * @param jobClient * - the JobClient to which it has been submitted * @return Returns the percentage progress of this Job * @throws IOException * In case any IOException connecting to JobTracker. */ protected double progressOfRunningJob(ControlledJob cjob, JobClient jobClient) throws IOException { @SuppressWarnings("deprecation") RunningJob rj = jobClient.getJob(cjob.getJob().getJobID().toString()); if(rj == null && cjob.getJobState() == ControlledJob.State.SUCCESS) return 1; else if(rj == null) return 0; else { return rj.mapProgress(); } }
/** * Returns the progress of a Job j which is part of a submitted JobControl object. The progress is for this Job. So * it has to be scaled down by the number of jobs that are present in the JobControl. * * @param cjob * - The Job for which progress is required * @param jobClient * - the JobClient to which it has been submitted * @return Returns the percentage progress of this Job * @throws IOException * In case any IOException connecting to JobTracker. */ protected double progressOfRunningJob(ControlledJob cjob, JobClient jobClient) throws IOException { @SuppressWarnings("deprecation") RunningJob rj = jobClient.getJob(cjob.getJob().getJobID().toString()); if(rj == null && cjob.getJobState() == ControlledJob.State.SUCCESS) return 1; else if(rj == null) return 0; else { return rj.mapProgress(); } }
/** * @param runId for which information will be returned. * @return a {@link MRJobInfo} containing information about a particular MapReduce program run. * @throws IOException if there is failure to communicate through the JobClient. * @throws NotFoundException if a Job with the given runId is not found. */ @Override public MRJobInfo getMRJobInfo(Id.Run runId) throws IOException, NotFoundException { Preconditions.checkArgument(ProgramType.MAPREDUCE.equals(runId.getProgram().getType())); JobClient jobClient = new JobClient(hConf); JobStatus[] jobs = jobClient.getAllJobs(); JobStatus thisJob = findJobForRunId(jobs, runId.toEntityId()); RunningJob runningJob = jobClient.getJob(thisJob.getJobID()); if (runningJob == null) { throw new IllegalStateException(String.format("JobClient returned null for RunId: '%s', JobId: '%s'", runId, thisJob.getJobID())); } Counters counters = runningJob.getCounters(); TaskReport[] mapTaskReports = jobClient.getMapTaskReports(thisJob.getJobID()); TaskReport[] reduceTaskReports = jobClient.getReduceTaskReports(thisJob.getJobID()); return new MRJobInfo(runningJob.mapProgress(), runningJob.reduceProgress(), groupToMap(counters.getGroup(TaskCounter.class.getName())), toMRTaskInfos(mapTaskReports), toMRTaskInfos(reduceTaskReports), true); }
/** * @param runId for which information will be returned. * @return a {@link MRJobInfo} containing information about a particular MapReduce program run. * @throws IOException if there is failure to communicate through the JobClient. * @throws NotFoundException if a Job with the given runId is not found. */ @Override public MRJobInfo getMRJobInfo(Id.Run runId) throws IOException, NotFoundException { Preconditions.checkArgument(ProgramType.MAPREDUCE.equals(runId.getProgram().getType())); JobClient jobClient = new JobClient(hConf); JobStatus[] jobs = jobClient.getAllJobs(); JobStatus thisJob = findJobForRunId(jobs, runId.toEntityId()); RunningJob runningJob = jobClient.getJob(thisJob.getJobID()); if (runningJob == null) { throw new IllegalStateException(String.format("JobClient returned null for RunId: '%s', JobId: '%s'", runId, thisJob.getJobID())); } Counters counters = runningJob.getCounters(); TaskReport[] mapTaskReports = jobClient.getMapTaskReports(thisJob.getJobID()); TaskReport[] reduceTaskReports = jobClient.getReduceTaskReports(thisJob.getJobID()); return new MRJobInfo(runningJob.mapProgress(), runningJob.reduceProgress(), groupToMap(counters.getGroup(TaskCounter.class.getName())), toMRTaskInfos(mapTaskReports), toMRTaskInfos(reduceTaskReports), true); }
progress = jc.getJob(rJob.getID()).mapProgress(); } else { progress = jc.getJob(rJob.getID()).reduceProgress();