protected int waitForCompletion(Job job) throws IOException, InterruptedException, ClassNotFoundException { int retVal = 0; long start = System.nanoTime(); if (isAsync) { job.submit(); } else { job.waitForCompletion(true); retVal = job.isSuccessful() ? 0 : 1; logger.debug("Job '" + job.getJobName() + "' finished " + (job.isSuccessful() ? "successfully in " : "with failures. Time taken ") + formatTime((System.nanoTime() - start) / 1000000L)); } return retVal; }
if (!job.isSuccessful()) { throw new RuntimeException("Job failed : " + job);
protected void runJob(String jobName, Configuration c, List<Scan> scans) throws IOException, InterruptedException, ClassNotFoundException { Job job = new Job(c, jobName); initJob(scans, job); job.setReducerClass(ScanReducer.class); job.setNumReduceTasks(1); // one to get final "first" and "last" key FileOutputFormat.setOutputPath(job, new Path(job.getJobName())); LOG.info("Started " + job.getJobName()); job.waitForCompletion(true); assertTrue(job.isSuccessful()); LOG.info("After map/reduce completion - job " + jobName); }
private void submitAndWait(Job job) throws ClassNotFoundException, IOException, InterruptedException { job.submit(); MRCompactor.addRunningHadoopJob(this.dataset, job); LOG.info(String.format("MR job submitted for dataset %s, input %s, url: %s", this.dataset, getInputPaths(), job.getTrackingURL())); while (!job.isComplete()) { if (this.policy == Policy.ABORT_ASAP) { LOG.info(String.format( "MR job for dataset %s, input %s killed due to input data incompleteness." + " Will try again later", this.dataset, getInputPaths())); job.killJob(); return; } Thread.sleep(MR_JOB_CHECK_COMPLETE_INTERVAL_MS); } if (!job.isSuccessful()) { throw new RuntimeException(String.format("MR job failed for topic %s, input %s, url: %s", this.dataset, getInputPaths(), job.getTrackingURL())); } }
+ job.isSuccessful()); Counters ctrs = job.getCounters(); LOG.debug(Objects.toString(ctrs)); if (job.isComplete() && !job.isSuccessful()) { throw new Exception("DistCp job-id: " + jobID + " failed");
@Override public void run() { try { Job job = createJob(); if (job == null) { log.info("No MR job created. Skipping."); this.workingState = WorkUnitState.WorkingState.SUCCESSFUL; this.eventSubmitter.submit(Events.MR_JOB_SKIPPED); onSkippedMRJob(); return; } job.submit(); log.info("MR tracking URL {} for job {}", job.getTrackingURL(), job.getJobName()); this.eventSubmitter.submit(Events.MR_JOB_STARTED_EVENT, Events.JOB_URL, job.getTrackingURL()); job.waitForCompletion(false); this.mrJob = job; if (job.isSuccessful()) { this.eventSubmitter.submit(Events.MR_JOB_SUCCESSFUL, Events.JOB_URL, job.getTrackingURL()); this.onMRTaskComplete(true, null); } else { this.eventSubmitter.submit(Events.MR_JOB_FAILED, Events.JOB_URL, job.getTrackingURL()); this.onMRTaskComplete (false, new IOException(String.format("MR Job:%s is not successful", job.getTrackingURL()))); } } catch (Throwable t) { log.error("Failed to run MR job.", t); this.eventSubmitter.submit(Events.MR_JOB_FAILED, Events.FAILURE_CONTEXT, t.getMessage()); this.onMRTaskComplete (false, t); } }
/** * Run the RowCounter map reduce job and verify the row count. * * @param args the command line arguments to be used for rowcounter job. * @param expectedCount the expected row count (result of map reduce job). * @throws Exception */ private void runRowCount(String[] args, int expectedCount) throws Exception { Job job = RowCounter.createSubmittableJob(TEST_UTIL.getConfiguration(), args); long start = System.currentTimeMillis(); job.waitForCompletion(true); long duration = System.currentTimeMillis() - start; LOG.debug("row count duration (ms): " + duration); assertTrue(job.isSuccessful()); Counter counter = job.getCounters().findCounter(RowCounter.RowCounterMapper.Counters.ROWS); assertEquals(expectedCount, counter.getValue()); }
Assert.assertTrue(job.isSuccessful()==false);
try { Job j = runMRCreate(null, partitionColumns, writeRecords, 20, true); assertEquals(!isTableImmutable(),j.isSuccessful()); } catch (IOException e) { exc = e; assertFalse(j.isSuccessful()); } catch (IOException e) { exc = e;
try { Job j = runMRCreate(partitionMap, partitionColumns, writeRecords, 20, true); assertEquals(!isTableImmutable(),j.isSuccessful()); } catch (IOException e) { exc = e; assertFalse(j.isSuccessful()); } catch (IOException e) { exc = e;
protected int waitForCompletion(Job job) throws IOException, InterruptedException, ClassNotFoundException { int retVal = 0; long start = System.nanoTime(); if (isAsync) { job.submit(); } else { job.waitForCompletion(true); retVal = job.isSuccessful() ? 0 : 1; logger.debug("Job '" + job.getJobName() + "' finished " + (job.isSuccessful() ? "successfully in " : "with failures. Time taken ") + formatTime((System.nanoTime() - start) / 1000000L)); } return retVal; }
protected void submitMrJob() throws IOException, InterruptedException, ClassNotFoundException { job.submit(); runningJobs.add(job); log.info("job running for: " + job.getJobName() + ", url: " + job.getTrackingURL()); job.waitForCompletion(false); if (!job.isSuccessful()) { throw new RuntimeException("hadoop job failed."); } }
@Override protected void submitMrJob() throws IOException, InterruptedException, ClassNotFoundException { job.submit(); runningJobs.add(job); CamusSingleFolderSweeper.this.metrics.recordMrStartRunningTimeByTopic(this.topicAndHour, System.currentTimeMillis()); LOG.info("job running for: " + props.getProperty(TOPIC_AND_HOUR) + ", url: " + job.getTrackingURL()); job.waitForCompletion(false); CamusSingleFolderSweeper.this.metrics.recordMrFinishTimeByTopic(this.topicAndHour, System.currentTimeMillis()); if (!job.isSuccessful()) { throw new RuntimeException("hadoop job failed."); } } }
private static boolean runImport(String[] args, Configuration configuration) throws IOException, InterruptedException, ClassNotFoundException { // need to make a copy of the configuration because to make sure different temp dirs are used. GenericOptionsParser opts = new GenericOptionsParser(new Configuration(configuration), args); Configuration newConf = opts.getConfiguration(); args = opts.getRemainingArgs(); Job job = Import.createSubmittableJob(newConf, args); job.waitForCompletion(false); return job.isSuccessful(); }
/** * Tests an index with the same # of rows as the data table, but one of the index rows is * incorrect Scrutiny should report the invalid rows. */ @Test public void testEqualRowCountIndexIncorrect() throws Exception { // insert one valid row upsertRow(dataTableUpsertStmt, 1, "name-1", 94010); conn.commit(); // disable the index and insert another row which is not indexed disableIndex(); upsertRow(dataTableUpsertStmt, 2, "name-2", 95123); conn.commit(); // insert a bad row into the index upsertIndexRow("badName", 2, 9999); conn.commit(); // scrutiny should report the bad row List<Job> completedJobs = runScrutiny(schemaName, dataTableName, indexTableName); Job job = completedJobs.get(0); assertTrue(job.isSuccessful()); Counters counters = job.getCounters(); assertEquals(1, getCounterValue(counters, VALID_ROW_COUNT)); assertEquals(1, getCounterValue(counters, INVALID_ROW_COUNT)); }
/** * Tests when there are more data table rows than index table rows Scrutiny should report the * number of incorrect rows */ @Test public void testMoreDataRows() throws Exception { upsertRow(dataTableUpsertStmt, 1, "name-1", 95123); conn.commit(); disableIndex(); // these rows won't have a corresponding index row upsertRow(dataTableUpsertStmt, 2, "name-2", 95124); upsertRow(dataTableUpsertStmt, 3, "name-3", 95125); conn.commit(); List<Job> completedJobs = runScrutiny(schemaName, dataTableName, indexTableName); Job job = completedJobs.get(0); assertTrue(job.isSuccessful()); Counters counters = job.getCounters(); assertEquals(1, getCounterValue(counters, VALID_ROW_COUNT)); assertEquals(2, getCounterValue(counters, INVALID_ROW_COUNT)); }
/** * Tests when there are more index table rows than data table rows Scrutiny should report the * number of incorrect rows when run with the index as the source table */ @Test public void testMoreIndexRows() throws Exception { upsertRow(dataTableUpsertStmt, 1, "name-1", 95123); conn.commit(); disableIndex(); // these index rows won't have a corresponding data row upsertIndexRow("name-2", 2, 95124); upsertIndexRow("name-3", 3, 95125); conn.commit(); List<Job> completedJobs = runScrutiny(schemaName, dataTableName, indexTableName, 10L, SourceTable.INDEX_TABLE_SOURCE); Job job = completedJobs.get(0); assertTrue(job.isSuccessful()); Counters counters = job.getCounters(); assertEquals(1, getCounterValue(counters, VALID_ROW_COUNT)); assertEquals(2, getCounterValue(counters, INVALID_ROW_COUNT)); }
/** * Tests running with both the index and data tables as the source table If we have an * incorrectly indexed row, it should be reported in each direction */ @Test public void testBothDataAndIndexAsSource() throws Exception { // insert one valid row upsertRow(dataTableUpsertStmt, 1, "name-1", 94010); conn.commit(); // disable the index and insert another row which is not indexed disableIndex(); upsertRow(dataTableUpsertStmt, 2, "name-2", 95123); conn.commit(); // insert a bad row into the index upsertIndexRow("badName", 2, 9999); conn.commit(); List<Job> completedJobs = runScrutiny(schemaName, dataTableName, indexTableName, 10L, SourceTable.BOTH); assertEquals(2, completedJobs.size()); for (Job job : completedJobs) { assertTrue(job.isSuccessful()); Counters counters = job.getCounters(); assertEquals(1, getCounterValue(counters, VALID_ROW_COUNT)); assertEquals(1, getCounterValue(counters, INVALID_ROW_COUNT)); } }
/** * Tests an index where the index pk is correct (indexed col values are indexed correctly), but * a covered index value is incorrect. Scrutiny should report the invalid row */ @Test public void testCoveredValueIncorrect() throws Exception { // insert one valid row upsertRow(dataTableUpsertStmt, 1, "name-1", 94010); conn.commit(); // disable index and insert another data row disableIndex(); upsertRow(dataTableUpsertStmt, 2, "name-2", 95123); conn.commit(); // insert a bad index row for the above data row upsertIndexRow("name-2", 2, 9999); conn.commit(); // scrutiny should report the bad row List<Job> completedJobs = runScrutiny(schemaName, dataTableName, indexTableName); Job job = completedJobs.get(0); assertTrue(job.isSuccessful()); Counters counters = job.getCounters(); assertEquals(1, getCounterValue(counters, VALID_ROW_COUNT)); assertEquals(1, getCounterValue(counters, INVALID_ROW_COUNT)); assertEquals(1, getCounterValue(counters, BAD_COVERED_COL_VAL_COUNT)); }
/** * Tests a data table that is correctly indexed. Scrutiny should report all rows as valid. */ @Test public void testValidIndex() throws Exception { // insert two rows upsertRow(dataTableUpsertStmt, 1, "name-1", 94010); upsertRow(dataTableUpsertStmt, 2, "name-2", 95123); conn.commit(); int numDataRows = countRows(dataTableFullName); int numIndexRows = countRows(indexTableFullName); // scrutiny should report everything as ok List<Job> completedJobs = runScrutiny(schemaName, dataTableName, indexTableName); Job job = completedJobs.get(0); assertTrue(job.isSuccessful()); Counters counters = job.getCounters(); assertEquals(2, getCounterValue(counters, VALID_ROW_COUNT)); assertEquals(0, getCounterValue(counters, INVALID_ROW_COUNT)); // make sure row counts weren't modified by scrutiny assertEquals(numDataRows, countRows(dataTableFullName)); assertEquals(numIndexRows, countRows(indexTableFullName)); }