@Override public Counters getCounters() throws IOException { return rj.getCounters(); } }
@Override public Counters getCounters() throws IOException { return rj.getCounters(); } }
public Counters getCounters() throws IOException { return _runningJob.getCounters(); }
void testInputFormat(Class<? extends InputFormat> clazz) throws IOException { Configuration conf = UTIL.getConfiguration(); final JobConf job = new JobConf(conf); job.setInputFormat(clazz); job.setOutputFormat(NullOutputFormat.class); job.setMapperClass(ExampleVerifier.class); job.setNumReduceTasks(0); LOG.debug("submitting job."); final RunningJob run = JobClient.runJob(job); assertTrue("job failed!", run.isSuccessful()); assertEquals("Saw the wrong number of instances of the filtered-for row.", 2, run.getCounters() .findCounter(TestTableInputFormat.class.getName() + ":row", "aaa").getCounter()); assertEquals("Saw any instances of the filtered out row.", 0, run.getCounters() .findCounter(TestTableInputFormat.class.getName() + ":row", "bbb").getCounter()); assertEquals("Saw the wrong number of instances of columnA.", 1, run.getCounters() .findCounter(TestTableInputFormat.class.getName() + ":family", "columnA").getCounter()); assertEquals("Saw the wrong number of instances of columnB.", 1, run.getCounters() .findCounter(TestTableInputFormat.class.getName() + ":family", "columnB").getCounter()); assertEquals("Saw the wrong count of values for the filtered-for row.", 2, run.getCounters() .findCounter(TestTableInputFormat.class.getName() + ":value", "value aaa").getCounter()); assertEquals("Saw the wrong count of values for the filtered-out row.", 0, run.getCounters() .findCounter(TestTableInputFormat.class.getName() + ":value", "value bbb").getCounter()); }
counters = runningJob.getCounters(); counters = runningJob.getCounters(); long numberOfRecords = counters.getCounter(Task.Counter.REDUCE_INPUT_GROUPS);
public void run(JobConf conf) throws Exception { _runningJob = new JobClient(conf).submitJob(conf); info("See " + _runningJob.getTrackingURL() + " for details."); _runningJob.waitForCompletion(); if(!_runningJob.isSuccessful()) { throw new Exception("Hadoop job:" + getId() + " failed!"); } // dump all counters Counters counters = _runningJob.getCounters(); for(String groupName: counters.getGroupNames()) { Counters.Group group = counters.getGroup(groupName); info("Group: " + group.getDisplayName()); for(Counter counter: group) info(counter.getDisplayName() + ":\t" + counter.getValue()); } }
RunningJob job = JobClient.runJob(configStage2()); Counters c = job.getCounters(); long changed = c.getCounter(PrCounters.CONVERGE_CHECK); System.out.println("Iteration = " + i + ", changed reducer = " + changed);
Counters c = job.getCounters(); long changed = c.getCounter(PrCounters.CONVERGE_CHECK); System.out.println("Iteration = " + i + ", changed reducer = " + changed);
@Override public Counters getCounters() throws IOException { return rj.getCounters(); } }
@Override public Counters getCounters() throws IOException { return rj.getCounters(); } }
// in the Reducer class... private long mapperCounter; @Override public void configure(JobConf conf) { JobClient client = new JobClient(conf); RunningJob parentJob = client.getJob(JobID.forName( conf.get("mapred.job.id") )); mapperCounter = parentJob.getCounters().getCounter(MAP_COUNTER_NAME); }
public void finish(RunningJob job) throws IOException { if (job != null) counters = job.getCounters() ; FSDataOutputStream out = hdfs.create(getFinishPath()); out.writeUTF("finished") ; out.close(); }
public void finish(RunningJob runningJob) throws IOException { super.finish(runningJob) ; counts = new HashMap<Counts,Long>() ; for (Counts c:Counts.values()) { Counters.Counter counter = runningJob.getCounters().findCounter(c) ; if (counter != null) counts.put(c, counter.getCounter()) ; else counts.put(c, 0L) ; } saveCounts() ; }
public static void verifyCounters(RunningJob runningJob, int expected) throws IOException { assertEquals(expected, runningJob.getCounters().getCounter(EnumCounter.MAP_RECORDS)); assertEquals(expected, runningJob.getCounters().getGroup("StringCounter") .getCounter("MapRecords")); }
public void finish(RunningJob runningJob) throws IOException { super.finish(runningJob) ; counts = new HashMap<Counts,Long>() ; for (Counts count:Counts.values()) { Counters.Counter counter = runningJob.getCounters().findCounter(count) ; if (counter != null) counts.put(count, counter.getCounter()) ; else counts.put(count, 0L) ; } saveCounts() ; }
public void finish(RunningJob runningJob) throws IOException { super.finish(runningJob) ; unforwardedCounts = new HashMap<Unforwarded,Long>() ; for (Unforwarded u:Unforwarded.values()) { Counters.Counter counter = runningJob.getCounters().findCounter(u) ; if (counter != null) unforwardedCounts.put(u, counter.getCounter()) ; else unforwardedCounts.put(u, 0L) ; } saveUnforwardedCounts() ; }
public static Long get(FlowStepStats step, Enum value) throws IOException { if (step instanceof HadoopStepStats) { HadoopStepStats hadoopStep = (HadoopStepStats)step; return hadoopStep.getRunningJob().getCounters().getCounter(value); } else { return step.getCounterValue(value); } }
public static boolean isMainSuccessful(RunningJob runningJob) throws IOException { boolean succeeded = runningJob.isSuccessful(); if (succeeded) { Counters counters = runningJob.getCounters(); if (counters != null) { Counters.Group group = counters.getGroup(LauncherAMUtils.COUNTER_GROUP); if (group != null) { succeeded = group.getCounter(LauncherAMUtils.COUNTER_LAUNCHER_ERROR) == 0; } } } return succeeded; }
public static boolean hasCredentials(RunningJob runningJob) throws IOException { boolean output = false; Counters counters = runningJob.getCounters(); if (counters != null) { Counters.Group group = counters.getGroup(COUNTER_GROUP); if (group != null) { output = group.getCounter(COUNTER_OUTPUT_DATA) > 0; } } return output; } }
public static void verifyCounters(RunningJob runningJob, int expected) throws IOException { assertEquals(expected, runningJob.getCounters().getCounter(EnumCounter.MAP_RECORDS)); assertEquals(expected, runningJob.getCounters().getGroup("StringCounter") .getCounter("MapRecords")); }