protected int waitForCompletion(Job job) throws IOException, InterruptedException, ClassNotFoundException { int retVal = 0; long start = System.nanoTime(); if (isAsync) { job.submit(); } else { job.waitForCompletion(true); retVal = job.isSuccessful() ? 0 : 1; logger.debug("Job '" + job.getJobName() + "' finished " + (job.isSuccessful() ? "successfully in " : "with failures. Time taken ") + formatTime((System.nanoTime() - start) / 1000000L)); } return retVal; }
/** * Entry point to start job. * * @param args Command line parameters. * @throws Exception If fails. */ public static void main(String[] args) throws Exception { if (args.length != 2) { System.out.println("usage: [input] [output]"); System.exit(-1); } Job job = getJob(args[0], args[1]); job.submit(); }
private void submitAndWait(Job job) throws ClassNotFoundException, IOException, InterruptedException { job.submit(); MRCompactor.addRunningHadoopJob(this.dataset, job); LOG.info(String.format("MR job submitted for dataset %s, input %s, url: %s", this.dataset, getInputPaths(), job.getTrackingURL())); while (!job.isComplete()) { if (this.policy == Policy.ABORT_ASAP) { LOG.info(String.format( "MR job for dataset %s, input %s killed due to input data incompleteness." + " Will try again later", this.dataset, getInputPaths())); job.killJob(); return; } Thread.sleep(MR_JOB_CHECK_COMPLETE_INTERVAL_MS); } if (!job.isSuccessful()) { throw new RuntimeException(String.format("MR job failed for topic %s, input %s, url: %s", this.dataset, getInputPaths(), job.getTrackingURL())); } }
job.submit();
); job.submit(); log.info("Job %s submitted, status available at %s", job.getJobName(), job.getTrackingURL());
job.submit(); log.info("Job %s submitted, status available at %s", job.getJobName(), job.getTrackingURL()); final boolean success = job.waitForCompletion(true);
@Override public void run() { try { Job job = createJob(); if (job == null) { log.info("No MR job created. Skipping."); this.workingState = WorkUnitState.WorkingState.SUCCESSFUL; this.eventSubmitter.submit(Events.MR_JOB_SKIPPED); onSkippedMRJob(); return; } job.submit(); log.info("MR tracking URL {} for job {}", job.getTrackingURL(), job.getJobName()); this.eventSubmitter.submit(Events.MR_JOB_STARTED_EVENT, Events.JOB_URL, job.getTrackingURL()); job.waitForCompletion(false); this.mrJob = job; if (job.isSuccessful()) { this.eventSubmitter.submit(Events.MR_JOB_SUCCESSFUL, Events.JOB_URL, job.getTrackingURL()); this.onMRTaskComplete(true, null); } else { this.eventSubmitter.submit(Events.MR_JOB_FAILED, Events.JOB_URL, job.getTrackingURL()); this.onMRTaskComplete (false, new IOException(String.format("MR Job:%s is not successful", job.getTrackingURL()))); } } catch (Throwable t) { log.error("Failed to run MR job.", t); this.eventSubmitter.submit(Events.MR_JOB_FAILED, Events.FAILURE_CONTEXT, t.getMessage()); this.onMRTaskComplete (false, t); } }
/** * Test job submission. * * @param conf Hadoop configuration. * @throws Exception If failed. */ private void checkJobSubmit(Configuration conf) throws Exception { final Job job = Job.getInstance(conf); try { job.setJobName(JOB_NAME); job.setOutputKeyClass(Text.class); job.setOutputValueClass(IntWritable.class); job.setInputFormatClass(TextInputFormat.class); job.setOutputFormatClass(OutFormat.class); job.setMapperClass(TestMapper.class); job.setReducerClass(TestReducer.class); job.setNumReduceTasks(0); FileInputFormat.setInputPaths(job, new Path("igfs://" + igfsName + "@" + PATH_INPUT)); job.submit(); job.waitForCompletion(false); assert job.getStatus().getState() == JobStatus.State.SUCCEEDED : job.getStatus().getState(); } finally { job.getCluster().close(); } }
FileOutputFormat.setOutputPath(groupByJob, config.makeGroupedDataDir()); groupByJob.submit(); log.info("Job %s submitted, status available at: %s", groupByJob.getJobName(), groupByJob.getTrackingURL());
FileOutputFormat.setOutputPath(groupByJob, config.makeGroupedDataDir()); groupByJob.submit(); log.info("Job %s submitted, status available at: %s", groupByJob.getJobName(), groupByJob.getTrackingURL()); FileOutputFormat.setOutputPath(dimSelectionJob, config.makeIntermediatePath()); dimSelectionJob.submit(); log.info( "Job %s submitted, status available at: %s",
this.job.submit(); this.hadoopJobSubmitted = true;
int runJob(Job job, boolean isForeground) throws Exception { if (isForeground) { LOG.info("Running UpdateStatisticsTool in Foreground. " + "Runs full table scans. This may take a long time!."); return (job.waitForCompletion(true)) ? 0 : 1; } else { LOG.info("Running UpdateStatisticsTool in Background - Submit async and exit"); job.submit(); return 0; } }
FileOutputFormat.setOutputPath(job, new Path("igfs://" + igfsName + "@" + PATH_OUTPUT)); job.submit();
protected void submitMrJob() throws IOException, InterruptedException, ClassNotFoundException { job.submit(); runningJobs.add(job); log.info("job running for: " + job.getJobName() + ", url: " + job.getTrackingURL()); job.waitForCompletion(false); if (!job.isSuccessful()) { throw new RuntimeException("hadoop job failed."); } }
FileOutputFormat.setOutputPath(job, new Path(PATH_OUTPUT)); job.submit();
@Override protected void submitMrJob() throws IOException, InterruptedException, ClassNotFoundException { job.submit(); runningJobs.add(job); CamusSingleFolderSweeper.this.metrics.recordMrStartRunningTimeByTopic(this.topicAndHour, System.currentTimeMillis()); LOG.info("job running for: " + props.getProperty(TOPIC_AND_HOUR) + ", url: " + job.getTrackingURL()); job.waitForCompletion(false); CamusSingleFolderSweeper.this.metrics.recordMrFinishTimeByTopic(this.topicAndHour, System.currentTimeMillis()); if (!job.isSuccessful()) { throw new RuntimeException("hadoop job failed."); } } }
protected int waitForCompletion(Job job) throws IOException, InterruptedException, ClassNotFoundException { int retVal = 0; long start = System.nanoTime(); if (isAsync) { job.submit(); } else { job.waitForCompletion(true); retVal = job.isSuccessful() ? 0 : 1; logger.debug("Job '" + job.getJobName() + "' finished " + (job.isSuccessful() ? "successfully in " : "with failures. Time taken ") + formatTime((System.nanoTime() - start) / 1000000L)); } return retVal; }
job.submit(); job.waitForCompletion(true); return 0;
LOG.info("Running Index Scrutiny in Background - Submit async and exit"); for (Job job : jobs) { job.submit();
mrJob.submit(); jobStatus.withState(State.SUBMITTED).withDescription("Job submitted") .withJobId(mrJob.getJobID().toString());