Refine search
public void run(JobConf conf) throws Exception { _runningJob = new JobClient(conf).submitJob(conf); info("See " + _runningJob.getTrackingURL() + " for details."); _runningJob.waitForCompletion(); if(!_runningJob.isSuccessful()) { throw new Exception("Hadoop job:" + getId() + " failed!"); } // dump all counters Counters counters = _runningJob.getCounters(); for(String groupName: counters.getGroupNames()) { Counters.Group group = counters.getGroup(groupName); info("Group: " + group.getDisplayName()); for(Counter counter: group) info(counter.getDisplayName() + ":\t" + counter.getValue()); } }
private Map<String, Double> extractAllCounterValues(Counters counters) { Map<String, Double> exctractedCounters = new HashMap<String, Double>(); for (Counters.Group cg : counters) { for (Counter c : cg) { exctractedCounters.put(cg.getName() + "::" + c.getName(), new Double(c.getCounter())); } } return exctractedCounters; }
private JSONObject getCountersJson(Counters ctrs) throws JSONException { JSONObject countersJson = new JSONObject(); Iterator<Counters.Group> iterator = ctrs.iterator(); while(iterator.hasNext()) { Counters.Group group = iterator.next(); Iterator<Counters.Counter> groupIterator = group.iterator(); JSONObject groupJson = new JSONObject(); while(groupIterator.hasNext()) { Counters.Counter counter = groupIterator.next(); groupJson.put(counter.getDisplayName(), counter.getCounter()); } countersJson.put(group.getDisplayName(), groupJson); } return countersJson; }
/** * Find the counter for the given enum. The same enum will always return the * same counter. * @param key the counter key * @return the matching counter object */ public synchronized Counter findCounter(Enum key) { Counter counter = cache.get(key); if (counter == null) { Group group = getGroup(key.getDeclaringClass().getName()); counter = group.getCounterForName(key.toString()); cache.put(key, counter); } return counter; }
public synchronized Counter findCounter(String group, String name) { if (name.equals("MAP_INPUT_BYTES")) { LOG.warn("Counter name MAP_INPUT_BYTES is deprecated. " + "Use FileInputFormatCounters as group name and " + " BYTES_READ as counter name instead"); return findCounter(FileInputFormatCounter.BYTES_READ); } String newGroupKey = getNewGroupKey(group); if (newGroupKey != null) { group = newGroupKey; } return getGroup(group).getCounterForName(name); }
Counters counters = new Counters(); IntWritable index = new IntWritable(0); getBlock(compactString, GROUP_OPEN, GROUP_CLOSE, index); getBlock(groupString, UNIT_OPEN, UNIT_CLOSE, groupIndex); groupName = unescape(groupName); Group group = counters.getGroup(groupName); group.setDisplayName(groupDisplayName); Counter counter = group.getCounterForName(counterName); counter.setDisplayName(counterDisplayName); counter.increment(value);
@Test public void testFileSystemGroupIteratorConcurrency() { Counters counters = new Counters(); // create 2 filesystem counter groups counters.findCounter("fs1", FileSystemCounter.BYTES_READ).increment(1); counters.findCounter("fs2", FileSystemCounter.BYTES_READ).increment(1); // Iterate over the counters in this group while updating counters in // the group Group group = counters.getGroup(FileSystemCounter.class.getName()); Iterator<Counter> iterator = group.iterator(); counters.findCounter("fs3", FileSystemCounter.BYTES_READ).increment(1); assertTrue(iterator.hasNext()); iterator.next(); counters.findCounter("fs3", FileSystemCounter.BYTES_READ).increment(1); assertTrue(iterator.hasNext()); iterator.next(); }
@SuppressWarnings("deprecation") @Test public void testGroupIteratorConcurrency() { Counters counters = new Counters(); counters.incrCounter("group1", "counter1", 1); Group group = counters.getGroup("group1"); Iterator<Counter> iterator = group.iterator(); counters.incrCounter("group1", "counter2", 1); iterator.next(); }
mapInputRecords = taskgroup.getCounterForName( MRPigStatsUtil.MAP_INPUT_RECORDS).getCounter(); mapOutputRecords = taskgroup.getCounterForName( MRPigStatsUtil.MAP_OUTPUT_RECORDS).getCounter(); reduceInputRecords = taskgroup.getCounterForName( MRPigStatsUtil.REDUCE_INPUT_RECORDS).getCounter(); reduceOutputRecords = taskgroup.getCounterForName( MRPigStatsUtil.REDUCE_OUTPUT_RECORDS).getCounter(); hdfsBytesRead = hdfsgroup.getCounterForName( MRPigStatsUtil.HDFS_BYTES_READ).getCounter(); hdfsBytesWritten = hdfsgroup.getCounterForName( MRPigStatsUtil.HDFS_BYTES_WRITTEN).getCounter(); spillCount = counters.findCounter( PigCounters.PROACTIVE_SPILL_COUNT_RECS).getCounter(); Iterator<Counter> iter = multistoregroup.iterator(); while (iter.hasNext()) { Counter cter = iter.next(); Iterator<Counter> iter2 = multiloadgroup.iterator(); while (iter2.hasNext()) { Counter cter = iter2.next();
.getGroup("org.apache.hadoop.mapred.Task$Counter") .getCounter("Map input bytes"); switch (level) { case 0: jobCounters.incrCounter(Counter.LOCAL_MAP_INPUT_BYTES, inputBytes); metrics.addLocalMapInputBytes(inputBytes); break; case 1: jobCounters.incrCounter(Counter.RACK_MAP_INPUT_BYTES, inputBytes); metrics.addRackMapInputBytes(inputBytes);
private void printCounters(StringBuffer buff, JobInfo job) throws ParseException { Counters mapCounters = Counters.fromEscapedCompactString(job.get(Keys.MAP_COUNTERS)); Counters reduceCounters = Counters.fromEscapedCompactString(job.get(Keys.REDUCE_COUNTERS)); Counters totalCounters = Counters.fromEscapedCompactString(job.get(Keys.COUNTERS)); Group reduceGroup = reduceCounters.getGroup(groupName); Format decimal = new DecimalFormat(); Iterator<Counter> ctrItr = totalGroup.iterator(); while (ctrItr.hasNext()) { Counter counter = ctrItr.next(); String name = counter.getDisplayName(); String mapValue = decimal.format(mapGroup.getCounter(name)); String reduceValue = decimal.format(reduceGroup.getCounter(name)); String totalValue = decimal.format(counter.getValue()); buff.append( String.format("\n|%1$-30s|%2$-30s|%3$-10s|%4$-10s|%5$-10s", totalGroup.getDisplayName(), counter.getDisplayName(), mapValue, reduceValue, totalValue));
public static void verifyCounters(RunningJob runningJob, int expected) throws IOException { assertEquals(expected, runningJob.getCounters().getCounter(EnumCounter.MAP_RECORDS)); assertEquals(expected, runningJob.getCounters().getGroup("StringCounter") .getCounter("MapRecords")); }
/** * Return textual representation of the counter values. */ public synchronized String toString() { StringBuilder sb = new StringBuilder("Counters: " + size()); for (Group group: this) { sb.append("\n\t" + group.getDisplayName()); for (Counter counter: group) { sb.append("\n\t\t" + counter.getDisplayName() + "=" + counter.getCounter()); } } return sb.toString(); }
private Map<String, Double> extractAllCounterValues(Counters counters) { Map<String, Double> exctractedCounters = new HashMap<String, Double>(); for (Counters.Group cg : counters) { for (Counter c : cg) { exctractedCounters.put(cg.getName() + "::" + c.getName(), new Double(c.getCounter())); } } return exctractedCounters; }
public CounterGroup(Counters.Group group) { this.groupName = group.getName(); this.groupDisplayName = group.getDisplayName(); this.counterInfoMap = Maps.newHashMap(); for (Counter counter : group) { CounterInfo counterInfo = new CounterInfo(counter); counterInfoMap.put(counterInfo.getName(), counterInfo); } }
sb.append(','); sb.append(group.getDisplayName()); sb.append('.'); sb.append(counter.getDisplayName()); sb.append(':'); sb.append(counter.getCounter()); String tab = getRowCountTableName(counter.getDisplayName()); if (tab != null) { if (sb1.length() > 0) {
sb.append(','); sb.append(group.getDisplayName()); sb.append('.'); sb.append(counter.getDisplayName()); sb.append(':'); sb.append(counter.getCounter()); String tab = getRowCountTableName(counter.getDisplayName()); if (tab != null) { if (sb1.length() > 0) {
private Map<String, Double> extractAllCounterValues(Counters counters) { Map<String, Double> exctractedCounters = new HashMap<String, Double>(); for (Counters.Group cg : counters) { for (Counter c : cg) { exctractedCounters.put(cg.getName() + "::" + c.getName(), new Double(c.getCounter())); } } return exctractedCounters; }
@Override public synchronized void incrAllCounters( org.apache.hadoop.mapred.Counters other) { for (Group otherGroup: other) { Group group = getGroup(otherGroup.getName()); group.setDisplayName(otherGroup.getDisplayName()); for (Counter otherCounter : otherGroup) { Counter counter = group.getCounterForName(otherCounter.getName()); counter.setDisplayName(otherCounter.getDisplayName()); counter.increment(otherCounter.getValue()); } } }
@Override public synchronized boolean equals(Object genericRight) { if (genericRight instanceof CounterGroupBase<?>) { @SuppressWarnings("unchecked") CounterGroupBase<Counter> right = ((CounterGroupBase<Counter>) genericRight).getUnderlyingGroup(); return Iterators.elementsEqual(iterator(), right.iterator()); } return false; }