public Counters getCounters() throws JobException { if (job != null) { try { return job.getCounters(); } catch (IOException e) { throw new JobException(e); } } else { throw new JobException("Job is null"); } }
return DEFAULT_COMPAT.getMetrics(job.getCounters());
@Override public int run(String[] args) throws Exception { String[] otherArgs = new GenericOptionsParser(getConf(), args).getRemainingArgs(); if (!doCommandLine(otherArgs)) { return 1; } Job job = createSubmittableJob(otherArgs); if (!job.waitForCompletion(true)) { LOG.info("Map-reduce job failed!"); return 1; } counters = job.getCounters(); return 0; }
counters = job.get().getCounters(); } catch (IOException e) { LOG.debug("Failed to get job counters. Record count will not be set. ", e);
/** * Create a {@link org.apache.gobblin.metrics.GobblinMetrics} instance for this job run from the Hadoop counters. */ @VisibleForTesting void countersToMetrics(GobblinMetrics metrics) throws IOException { Optional<Counters> counters = Optional.fromNullable(this.job.getCounters()); if (counters.isPresent()) { // Write job-level counters CounterGroup jobCounterGroup = counters.get().getGroup(MetricGroup.JOB.name()); for (Counter jobCounter : jobCounterGroup) { metrics.getCounter(jobCounter.getName()).inc(jobCounter.getValue()); } // Write task-level counters CounterGroup taskCounterGroup = counters.get().getGroup(MetricGroup.TASK.name()); for (Counter taskCounter : taskCounterGroup) { metrics.getCounter(taskCounter.getName()).inc(taskCounter.getValue()); } } }
@Override public int run(String[] args) throws Exception { if (args.length < 2) { usage("Wrong number of arguments: " + args.length); return -1; } String inputVersionString = System.getProperty(ResultSerialization.IMPORT_FORMAT_VER); if (inputVersionString != null) { getConf().set(ResultSerialization.IMPORT_FORMAT_VER, inputVersionString); } Job job = createSubmittableJob(getConf(), args); boolean isJobSuccessful = job.waitForCompletion(true); if(isJobSuccessful){ // Flush all the regions of the table flushRegionsIfNecessary(getConf()); } long inputRecords = job.getCounters().findCounter(TaskCounter.MAP_INPUT_RECORDS).getValue(); long outputRecords = job.getCounters().findCounter(TaskCounter.MAP_OUTPUT_RECORDS).getValue(); if (outputRecords < inputRecords) { System.err.println("Warning, not all records were imported (maybe filtered out)."); if (outputRecords == 0) { System.err.println("If the data was exported from HBase 0.94 "+ "consider using -Dhbase.import.version=0.94."); } } return (isJobSuccessful ? 0 : 1); }
@Override public int run(String[] args) throws Exception { if (args.length < 1) { printUsage("Wrong number of parameters: " + args.length); return -1; } Job job = createSubmittableJob(getConf(), args); if (job == null) { return -1; } boolean success = job.waitForCompletion(true); final long expectedCount = getConf().getLong(EXPECTED_COUNT_KEY, -1); if (success && expectedCount != -1) { final Counter counter = job.getCounters().findCounter(RowCounterMapper.Counters.ROWS); success = expectedCount == counter.getValue(); if (!success) { LOG.error("Failing job because count of '" + counter.getValue() + "' does not match expected count of '" + expectedCount + "'"); } } return (success ? 0 : 1); }
@Override protected Job doLoad(Configuration conf, HTableDescriptor htd) throws Exception { Job job = super.doLoad(conf, htd); this.numRowsLoadedWithExp1 = job.getCounters().findCounter(Counters.ROWS_VIS_EXP_1).getValue(); this.numRowsLoadedWithExp2 = job.getCounters().findCounter(Counters.ROWS_VIS_EXP_2).getValue(); this.numRowsLoadWithExp3 = job.getCounters().findCounter(Counters.ROWS_VIS_EXP_3).getValue(); this.numRowsLoadWithExp4 = job.getCounters().findCounter(Counters.ROWS_VIS_EXP_4).getValue(); System.out.println("Rows loaded with cell visibility " + VISIBILITY_EXPS[0] + " : " + this.numRowsLoadedWithExp1); System.out.println("Rows loaded with cell visibility " + VISIBILITY_EXPS[1] + " : " + this.numRowsLoadedWithExp2); System.out.println("Rows loaded with cell visibility " + VISIBILITY_EXPS[2] + " : " + this.numRowsLoadWithExp3); System.out.println("Rows loaded with cell visibility " + VISIBILITY_EXPS[3] + " : " + this.numRowsLoadWithExp4); return job; }
public boolean verify(long expectedReferenced) throws Exception { if (job == null) { throw new IllegalStateException("You should call run() first"); } Counters counters = job.getCounters(); if (counters == null) { LOG.info("Counters object was null, write verification cannot be performed." + " This is commonly a result of insufficient YARN configuration."); return false; } // Run through each check, even if we fail one early boolean success = verifyExpectedValues(expectedReferenced, counters); if (!verifyUnexpectedValues(counters)) { // We found counter objects which imply failure success = false; } if (!success) { handleFailure(counters); } return success; }
void testInputFormat(Class<? extends InputFormat> clazz) throws IOException, InterruptedException, ClassNotFoundException { final Job job = MapreduceTestingShim.createJob(UTIL.getConfiguration()); job.setInputFormatClass(clazz); job.setOutputFormatClass(NullOutputFormat.class); job.setMapperClass(ExampleVerifier.class); job.setNumReduceTasks(0); LOG.debug("submitting job."); assertTrue("job failed!", job.waitForCompletion(true)); assertEquals("Saw the wrong number of instances of the filtered-for row.", 2, job.getCounters() .findCounter(TestTableInputFormat.class.getName() + ":row", "aaa").getValue()); assertEquals("Saw any instances of the filtered out row.", 0, job.getCounters() .findCounter(TestTableInputFormat.class.getName() + ":row", "bbb").getValue()); assertEquals("Saw the wrong number of instances of columnA.", 1, job.getCounters() .findCounter(TestTableInputFormat.class.getName() + ":family", "columnA").getValue()); assertEquals("Saw the wrong number of instances of columnB.", 1, job.getCounters() .findCounter(TestTableInputFormat.class.getName() + ":family", "columnB").getValue()); assertEquals("Saw the wrong count of values for the filtered-for row.", 2, job.getCounters() .findCounter(TestTableInputFormat.class.getName() + ":value", "value aaa").getValue()); assertEquals("Saw the wrong count of values for the filtered-out row.", 0, job.getCounters() .findCounter(TestTableInputFormat.class.getName() + ":value", "value bbb").getValue()); }
protected void doVerify(Configuration conf, HTableDescriptor htd) throws Exception { Path outputDir = getTestDir(TEST_NAME, "verify-output"); LOG.info("Verify output dir: " + outputDir); Job job = Job.getInstance(conf); job.setJarByClass(this.getClass()); job.setJobName(TEST_NAME + " Verification for " + htd.getTableName()); setJobScannerConf(job); Scan scan = new Scan(); TableMapReduceUtil.initTableMapperJob( htd.getTableName().getNameAsString(), scan, VerifyMapper.class, BytesWritable.class, BytesWritable.class, job); TableMapReduceUtil.addDependencyJarsForClasses(job.getConfiguration(), AbstractHBaseTool.class); int scannerCaching = conf.getInt("verify.scannercaching", SCANNER_CACHING); TableMapReduceUtil.setScannerCaching(job, scannerCaching); job.setReducerClass(VerifyReducer.class); job.setNumReduceTasks(conf.getInt(NUM_REDUCE_TASKS_KEY, NUM_REDUCE_TASKS_DEFAULT)); FileOutputFormat.setOutputPath(job, outputDir); assertTrue(job.waitForCompletion(true)); long numOutputRecords = job.getCounters().findCounter(Counters.ROWS_WRITTEN).getValue(); assertEquals(0, numOutputRecords); }
/** * Run the RowCounter map reduce job and verify the row count. * * @param args the command line arguments to be used for rowcounter job. * @param expectedCount the expected row count (result of map reduce job). * @throws Exception */ private void runRowCount(String[] args, int expectedCount) throws Exception { Job job = RowCounter.createSubmittableJob(TEST_UTIL.getConfiguration(), args); long start = System.currentTimeMillis(); job.waitForCompletion(true); long duration = System.currentTimeMillis() - start; LOG.debug("row count duration (ms): " + duration); assertTrue(job.isSuccessful()); Counter counter = job.getCounters().findCounter(RowCounter.RowCounterMapper.Counters.ROWS); assertEquals(expectedCount, counter.getValue()); }
/** * Run MR job to check the number of mapper = expectedNumOfSplits */ protected void testNumOfSplitsMR(int splitsPerRegion, int expectedNumOfSplits) throws IOException, InterruptedException, ClassNotFoundException { String jobName = "TestJobForNumOfSplits-MR"; LOG.info("Before map/reduce startup - job " + jobName); JobConf c = new JobConf(TEST_UTIL.getConfiguration()); Scan scan = new Scan(); scan.addFamily(INPUT_FAMILYS[0]); scan.addFamily(INPUT_FAMILYS[1]); c.setInt("hbase.mapreduce.tableinput.mappers.per.region", splitsPerRegion); c.set(KEY_STARTROW, ""); c.set(KEY_LASTROW, ""); Job job = Job.getInstance(c, jobName); TableMapReduceUtil.initTableMapperJob(TABLE_NAME.getNameAsString(), scan, ScanMapper.class, ImmutableBytesWritable.class, ImmutableBytesWritable.class, job); job.setReducerClass(ScanReducer.class); job.setNumReduceTasks(1); job.setOutputFormatClass(NullOutputFormat.class); assertTrue("job failed!", job.waitForCompletion(true)); // for some reason, hbase does not expose JobCounter.TOTAL_LAUNCHED_MAPS, // we use TaskCounter.SHUFFLED_MAPS to get total launched maps assertEquals("Saw the wrong count of mappers per region", expectedNumOfSplits, job.getCounters().findCounter(TaskCounter.SHUFFLED_MAPS).getValue()); }
/** * Verify scan counters are emitted from the job * @param job * @throws IOException */ private void verifyJobCountersAreEmitted(Job job) throws IOException { Counters counters = job.getCounters(); Counter counter = counters.findCounter(TableRecordReaderImpl.HBASE_COUNTER_GROUP_NAME, "RPC_CALLS"); assertNotNull("Unable to find Job counter for HBase scan metrics, RPC_CALLS", counter); assertTrue("Counter value for RPC_CALLS should be larger than 0", counter.getValue() > 0); }
private void runVerifyReplication(String[] args, int expectedGoodRows, int expectedBadRows) throws IOException, InterruptedException, ClassNotFoundException { Job job = new VerifyReplication().createSubmittableJob(new Configuration(conf1), args); if (job == null) { fail("Job wasn't created, see the log"); } if (!job.waitForCompletion(true)) { fail("Job failed, see the log"); } assertEquals(expectedGoodRows, job.getCounters().findCounter(VerifyReplication.Verifier.Counters.GOODROWS).getValue()); assertEquals(expectedBadRows, job.getCounters().findCounter(VerifyReplication.Verifier.Counters.BADROWS).getValue()); }
public boolean verify() { try { Counters counters = job.getCounters(); if (counters == null) { LOG.info("Counters object was null, Generator verification cannot be performed." + " This is commonly a result of insufficient YARN configuration."); return false; } if (counters.findCounter(Counts.TERMINATING).getValue() > 0 || counters.findCounter(Counts.UNDEFINED).getValue() > 0 || counters.findCounter(Counts.IOEXCEPTION).getValue() > 0) { LOG.error("Concurrent walker failed to verify during Generation phase"); LOG.error("TERMINATING nodes: " + counters.findCounter(Counts.TERMINATING).getValue()); LOG.error("UNDEFINED nodes: " + counters.findCounter(Counts.UNDEFINED).getValue()); LOG.error("IOEXCEPTION nodes: " + counters.findCounter(Counts.IOEXCEPTION).getValue()); return false; } } catch (IOException e) { LOG.info("Generator verification could not find counter"); return false; } return true; } }
@Override public Map<String, Object> getStats() { if (groupByJob == null) { return null; } try { Counters jobCounters = groupByJob.getCounters(); Map<String, Object> metrics = TaskMetricsUtils.makeIngestionRowMetrics( jobCounters.findCounter(HadoopDruidIndexerConfig.IndexJobCounters.ROWS_PROCESSED_COUNTER).getValue(), jobCounters.findCounter(HadoopDruidIndexerConfig.IndexJobCounters.ROWS_PROCESSED_WITH_ERRORS_COUNTER).getValue(), jobCounters.findCounter(HadoopDruidIndexerConfig.IndexJobCounters.ROWS_UNPARSEABLE_COUNTER).getValue(), jobCounters.findCounter(HadoopDruidIndexerConfig.IndexJobCounters.ROWS_THROWN_AWAY_COUNTER).getValue() ); return metrics; } catch (IllegalStateException ise) { log.debug("Couldn't get counters due to job state"); return null; } catch (Exception e) { log.debug(e, "Encountered exception in getStats()."); return null; } }
@Override public Map<String, Object> getStats() { if (job == null) { return null; } try { Counters jobCounters = job.getCounters(); Map<String, Object> metrics = TaskMetricsUtils.makeIngestionRowMetrics( jobCounters.findCounter(HadoopDruidIndexerConfig.IndexJobCounters.ROWS_PROCESSED_COUNTER).getValue(), jobCounters.findCounter(HadoopDruidIndexerConfig.IndexJobCounters.ROWS_PROCESSED_WITH_ERRORS_COUNTER) .getValue(), jobCounters.findCounter(HadoopDruidIndexerConfig.IndexJobCounters.ROWS_UNPARSEABLE_COUNTER).getValue(), jobCounters.findCounter(HadoopDruidIndexerConfig.IndexJobCounters.ROWS_THROWN_AWAY_COUNTER).getValue() ); return metrics; } catch (IllegalStateException ise) { log.debug("Couldn't get counters due to job state"); return null; } catch (Exception e) { log.debug(e, "Encountered exception in getStats()."); return null; } }
@Override public Map<String, Object> getStats() { if (groupByJob == null) { return null; } try { Counters jobCounters = groupByJob.getCounters(); Map<String, Object> metrics = TaskMetricsUtils.makeIngestionRowMetrics( jobCounters.findCounter(HadoopDruidIndexerConfig.IndexJobCounters.ROWS_PROCESSED_COUNTER).getValue(), jobCounters.findCounter(HadoopDruidIndexerConfig.IndexJobCounters.ROWS_PROCESSED_WITH_ERRORS_COUNTER).getValue(), jobCounters.findCounter(HadoopDruidIndexerConfig.IndexJobCounters.ROWS_UNPARSEABLE_COUNTER).getValue(), jobCounters.findCounter(HadoopDruidIndexerConfig.IndexJobCounters.ROWS_THROWN_AWAY_COUNTER).getValue() ); return metrics; } catch (IllegalStateException ise) { log.debug("Couldn't get counters due to job state"); return null; } catch (Exception e) { log.debug(e, "Encountered exception in getStats()."); return null; } }
@Override public TimingResult call() throws Exception { PerformanceEvaluation.TestOptions opts = PerformanceEvaluation.parseOpts(argv); PerformanceEvaluation.checkTable(admin, opts); PerformanceEvaluation.RunResult results[] = null; long numRows = opts.totalRows; long elapsedTime = 0; if (opts.nomapred) { results = PerformanceEvaluation.doLocalClients(opts, admin.getConfiguration()); for (PerformanceEvaluation.RunResult r : results) { elapsedTime = Math.max(elapsedTime, r.duration); } } else { Job job = PerformanceEvaluation.doMapReduce(opts, admin.getConfiguration()); Counters counters = job.getCounters(); numRows = counters.findCounter(PerformanceEvaluation.Counter.ROWS).getValue(); elapsedTime = counters.findCounter(PerformanceEvaluation.Counter.ELAPSED_TIME).getValue(); } return new TimingResult(numRows, elapsedTime, results); } }