private void updateMapRedStatsJson(MapRedStats stats, RunningJob rj) throws IOException, JSONException { if (statsJSON == null) { statsJSON = new JSONObject(); } if (stats != null) { if (stats.getNumMap() >= 0) { statsJSON.put(NUMBER_OF_MAPPERS, stats.getNumMap()); } if (stats.getNumReduce() >= 0) { statsJSON.put(NUMBER_OF_REDUCERS, stats.getNumReduce()); } if (stats.getCounters() != null) { statsJSON.put(COUNTERS, getCountersJson(stats.getCounters())); } } if (rj != null) { statsJSON.put(JOB_ID, rj.getID().toString()); statsJSON.put(JOB_FILE, rj.getJobFile()); statsJSON.put(TRACKING_URL, rj.getTrackingURL()); statsJSON.put(MAP_PROGRESS, Math.round(rj.mapProgress() * 100)); statsJSON.put(REDUCE_PROGRESS, Math.round(rj.reduceProgress() * 100)); statsJSON.put(CLEANUP_PROGRESS, Math.round(rj.cleanupProgress() * 100)); statsJSON.put(SETUP_PROGRESS, Math.round(rj.setupProgress() * 100)); statsJSON.put(COMPLETE, rj.isComplete()); statsJSON.put(SUCCESSFUL, rj.isSuccessful()); } }
jobName = runningJob.getJobName(); trackingURL = runningJob.getTrackingURL(); isComplete = runningJob.isComplete(); isSuccessful = runningJob.isSuccessful(); mapProgress = runningJob.mapProgress();
updateMapRedTaskWebUIStatistics(mapRedStats, rj); while (!rj.isComplete()) { if (th.getContext() != null) { th.getContext().checkHeartbeaterLockException();
final boolean localMode = ShimLoader.getHadoopShims().isLocalMode(job); while (!rj.isComplete()) { if (th.getContext() != null) { th.getContext().checkHeartbeaterLockException();
private boolean updateJobState() throws IOException { if (jobProgress == null) { jobProgress = new MapReduceJobState( rj, jobClient.getMapTaskReports(jobId), jobClient.getReduceTaskReports(jobId)); return true; } boolean complete = rj.isComplete(); boolean successful = rj.isSuccessful(); float mapProgress = rj.mapProgress(); float reduceProgress = rj.reduceProgress(); boolean update = !( jobProgress.isComplete() == complete && jobProgress.isSuccessful() == successful && AmbroseHiveUtil.isEqual(jobProgress.getMapProgress(), mapProgress) && AmbroseHiveUtil.isEqual(jobProgress.getReduceProgress(), reduceProgress) ); //do progress report only if necessary if (update) { jobProgress = new MapReduceJobState( rj, jobClient.getMapTaskReports(jobId), jobClient.getReduceTaskReports(jobId)); jobProgress.setJobLastUpdateTime(System.currentTimeMillis()); } return update; }
protected boolean internalNonBlockingIsComplete() throws IOException { return runningJob.isComplete(); }
protected boolean internalNonBlockingIsComplete() throws IOException { return runningJob.isComplete(); }
@Override public boolean evaluate() throws Exception { return mrJob.isComplete(); } });
@Override public boolean evaluate() throws Exception { return launcherJob.isComplete(); } });
protected void internalBlockOnStop() throws IOException { if( runningJob != null && !runningJob.isComplete() ) runningJob.killJob(); }
protected void internalBlockOnStop() throws IOException { if( runningJob != null && !runningJob.isComplete() ) runningJob.killJob(); }
@Override public boolean evaluate() throws Exception { return launcherJob.isComplete(); } });
@Override public boolean evaluate() throws Exception { return launcherJob2.isComplete(); } });
@Override public boolean evaluate() throws Exception { return mrJob.isComplete(); } });
... RunningJob runjob = jobclient.submitJob(conf); while (!runjob.isComplete()) { System.out.println("sleeping for 5 sec..."); Thread.sleep(5000); } TaskReport[] maps = jobclient.getMapTaskReports(runjob.getID()); ...
/** * Check if the job is finished or not. * This is a non-blocking call. * * @return <code>true</code> if the job is complete, else <code>false</code>. * @throws IOException */ public boolean isComplete() throws IOException { ensureState(JobState.RUNNING); return info.isComplete(); }
public void testKeyMismatch() throws Exception { configure(); // Set bad MapOutputKeyClass and MapOutputValueClass conf.setMapOutputKeyClass(IntWritable.class); conf.setMapOutputValueClass(IntWritable.class); RunningJob r_job = jc.submitJob(conf); while (!r_job.isComplete()) { Thread.sleep(1000); } if (r_job.isSuccessful()) { fail("Oops! The job was supposed to break due to an exception"); } }
public void testNoMismatch() throws Exception{ configure(); // Set good MapOutputKeyClass and MapOutputValueClass conf.setMapOutputKeyClass(Text.class); conf.setMapOutputValueClass(Text.class); RunningJob r_job = jc.submitJob(conf); while (!r_job.isComplete()) { Thread.sleep(1000); } if (!r_job.isSuccessful()) { fail("Oops! The job broke due to an unexpected error"); } } }
@Test public void testKeyMismatch() throws Exception { // Set bad MapOutputKeyClass and MapOutputValueClass conf.setMapOutputKeyClass(IntWritable.class); conf.setMapOutputValueClass(IntWritable.class); RunningJob r_job = jc.submitJob(conf); while (!r_job.isComplete()) { Thread.sleep(1000); } if (r_job.isSuccessful()) { fail("Oops! The job was supposed to break due to an exception"); } }
@Test public void testValueMismatch() throws Exception { conf.setMapOutputKeyClass(Text.class); conf.setMapOutputValueClass(IntWritable.class); RunningJob r_job = jc.submitJob(conf); while (!r_job.isComplete()) { Thread.sleep(1000); } if (r_job.isSuccessful()) { fail("Oops! The job was supposed to break due to an exception"); } }