@Override public void incrementCustom(String metric, long delta) { counters.getGroup(HadoopContextScanMetrics.CUSTOM_COUNTER_GROUP).findCounter(metric).increment(delta); }
/** {@inheritDoc} */ @Override public synchronized CounterGroup addGroup(CounterGroup grp) { return addGroup(grp.getName(), grp.getDisplayName()); }
Counters counters = job.getCounters(); for (CounterGroup group : counters) { System.out.println("* Counter Group: " + group.getDisplayName() + " (" + group.getName() + ")"); System.out.println(" number of counters in this group: " + group.size()); for (Counter counter : group) { System.out.println(" - " + counter.getDisplayName() + ": " + counter.getName() + ": "+counter.getValue()); } }
buff.append("\n------------------------------------------"+ "---------------------------------------------"); for (String groupName : totalCounters.getGroupNames()) { CounterGroup totalGroup = totalCounters.getGroup(groupName); CounterGroup mapGroup = mapCounters.getGroup(groupName); CounterGroup reduceGroup = reduceCounters.getGroup(groupName); totalGroup.iterator(); while(ctrItr.hasNext()) { org.apache.hadoop.mapreduce.Counter counter = ctrItr.next(); String name = counter.getName(); String mapValue = decimal.format(mapGroup.findCounter(name).getValue()); String reduceValue = decimal.format(reduceGroup.findCounter(name).getValue()); String totalValue = decimal.format(counter.getValue()); totalGroup.getDisplayName(), counter.getDisplayName(), mapValue, reduceValue, totalValue));
@Override public long getCustom(String metric) { return counters.getGroup(HadoopContextScanMetrics.CUSTOM_COUNTER_GROUP).findCounter(metric).getValue(); }
Counter referenced = counters.findCounter(Counts.REFERENCED); Counter unreferenced = counters.findCounter(Counts.UNREFERENCED); Counter undefined = counters.findCounter(Counts.UNDEFINED); Counter multiref = counters.findCounter(Counts.EXTRAREFERENCES); if (expectedReferenced != referenced.getValue()) { LOG.error("Expected referenced count does not match with actual referenced count. " + "Expected referenced=" + expectedReferenced + ", actual=" + referenced.getValue()); success = false; if (unreferenced.getValue() > 0) { boolean couldBeMultiRef = (multiref.getValue() == unreferenced.getValue()); LOG.error("Unreferenced nodes were not expected. Unreferenced count=" + CounterGroup g = counters.getGroup("undef"); Iterator<Counter> it = g.iterator(); while (it.hasNext()) { String keyString = it.next().getName(); g = counters.getGroup("unref"); it = g.iterator(); while (it.hasNext()) { String keyString = it.next().getName();
public synchronized void incrAllCounters(CounterGroup rightGroup) { for(Counter right: rightGroup.counters.values()) { Counter left = findCounter(right.getName(), right.getDisplayName()); left.increment(right.getValue()); } } }
for (String groupName : counters.getGroupNames()) { CounterGroup group = counters.getGroup(groupName); log.info("Group: " + group.getDisplayName()); for (Counter counter : group) { log.info(counter.getDisplayName() + ":\t" + counter.getValue());
private void setCounters( TaskReport taskReport ) { this.counters = new HashMap<>(); Counters hadoopCounters = taskReport.getTaskCounters(); for( CounterGroup group : hadoopCounters ) { Map<String, Long> values = new HashMap<String, Long>(); this.counters.put( group.getName(), values ); for( Counter counter : group ) values.put( counter.getName(), counter.getValue() ); } }
job.getCounters().getGroup(MultipleOutputs.class.getName()); assertEquals(9, counters.size()); assertEquals(4, counters.findCounter(TEXT).getValue()); assertEquals(2, counters.findCounter(SEQUENCE + "_A").getValue()); assertEquals(4, counters.findCounter(SEQUENCE + "_B").getValue()); assertEquals(2, counters.findCounter(SEQUENCE + "_C").getValue()); assertEquals(2, counters.findCounter("a").getValue()); assertEquals(2, counters.findCounter("b").getValue()); assertEquals(4, counters.findCounter("c").getValue()); assertEquals(4, counters.findCounter("d").getValue()); assertEquals(4, counters.findCounter("e").getValue());
private void validateCounters(org.apache.hadoop.mapreduce.Counters counters) { Iterator<org.apache.hadoop.mapreduce.CounterGroup> it = counters.iterator(); while (it.hasNext()) { org.apache.hadoop.mapreduce.CounterGroup group = it.next(); LOG.info("Group " + group.getDisplayName()); Iterator<org.apache.hadoop.mapreduce.Counter> itc = group.iterator(); while (itc.hasNext()) { LOG.info("Counter is " + itc.next().getDisplayName()); } } Assert.assertEquals(1, counters.countCounters()); }
Counters ctrs = job.getCounters(); System.out.println("Counters: " + ctrs); long mapIn = ctrs.findCounter(FileInputFormat.COUNTER_GROUP, FileInputFormat.BYTES_READ).getValue(); assertTrue(mapIn != 0); long combineIn = ctrs.findCounter(COUNTER_GROUP, "COMBINE_INPUT_RECORDS").getValue(); long combineOut = ctrs.findCounter(COUNTER_GROUP, "COMBINE_OUTPUT_RECORDS").getValue(); long reduceIn = ctrs.findCounter(COUNTER_GROUP, "REDUCE_INPUT_RECORDS").getValue(); String group = "Random Group"; CounterGroup ctrGrp = ctrs.getGroup(group); assertEquals(0, ctrGrp.size());
/** * Return textual representation of the counter values. */ public synchronized String toString() { StringBuilder sb = new StringBuilder("Counters: " + countCounters()); for (CounterGroup group: this) { sb.append("\n\t" + group.getDisplayName()); for (Counter counter: group) { sb.append("\n\t\t" + counter.getDisplayName() + "=" + counter.getValue()); } } return sb.toString(); }
private void submitCountersToStatsd(Job job) throws IOException { Counters counters = job.getCounters(); if (getStatsdEnabled(job)) { StatsDClient statsd = new NonBlockingStatsDClient("Camus", getStatsdHost(job), getStatsdPort(job), new String[] { "camus:counters" }); for (CounterGroup counterGroup : counters) { for (Counter counter : counterGroup) { statsd.gauge(counterGroup.getDisplayName() + "." + counter.getDisplayName(), counter.getValue()); } } } }
@Override protected void handleFailure(Counters counters) throws IOException { try (Connection conn = ConnectionFactory.createConnection(job.getConfiguration())) { TableName tableName = TableName.valueOf(COMMON_TABLE_NAME); CounterGroup g = counters.getGroup("undef"); Iterator<Counter> it = g.iterator(); while (it.hasNext()) { String keyString = it.next().getName(); byte[] key = Bytes.toBytes(keyString); HRegionLocation loc = conn.getRegionLocator(tableName).getRegionLocation(key, true); LOG.error("undefined row " + keyString + ", " + loc); } g = counters.getGroup("unref"); it = g.iterator(); while (it.hasNext()) { String keyString = it.next().getName(); byte[] key = Bytes.toBytes(keyString); HRegionLocation loc = conn.getRegionLocator(tableName).getRegionLocation(key, true); LOG.error("unreferred row " + keyString + ", " + loc); } } } }
private Counters getCounters() { Counters c = new Counters(); Map<String, Map<String, Long>> values = counters.value(); for (Map.Entry<String, Map<String, Long>> e : values.entrySet()) { CounterGroup cg = c.getGroup(e.getKey()); for (Map.Entry<String, Long> f : e.getValue().entrySet()) { cg.findCounter(f.getKey()).setValue(f.getValue()); } } return c; }
private static String checkForIngestLabelOverride(Counters ingestJobCounters) { CounterGroup jobQueueName = ingestJobCounters.getGroup(IngestProcess.METRICS_LABEL_OVERRIDE.name()); if (jobQueueName.size() > 0) { Counter myCounter = jobQueueName.iterator().next(); return myCounter.getName(); } return null; }
public JobCounterInfo(AppContext ctx, Job job) { getCounters(ctx, job); counterGroup = new ArrayList<CounterGroupInfo>(); this.id = MRApps.toString(job.getID()); if (total != null) { for (CounterGroup g : total) { if (g != null) { CounterGroup mg = map == null ? null : map.getGroup(g.getName()); CounterGroup rg = reduce == null ? null : reduce .getGroup(g.getName()); CounterGroupInfo cginfo = new CounterGroupInfo(g.getName(), g, mg, rg); counterGroup.add(cginfo); } } } }
static Counters fromAvro(JhCounters counters) { Counters result = new Counters(); if(counters != null) { for (JhCounterGroup g : counters.groups) { CounterGroup group = result.addGroup(StringInterner.weakIntern(g.name.toString()), StringInterner.weakIntern(g.displayName.toString())); for (JhCounter c : g.counts) { group.addCounter(StringInterner.weakIntern(c.name.toString()), StringInterner.weakIntern(c.displayName.toString()), c.value); } } } return result; }