/** {@inheritDoc} */ @Override public synchronized CounterGroup addGroup(CounterGroup grp) { return addGroup(grp.getName(), grp.getDisplayName()); }
for (String groupName : counters.getGroupNames()) { CounterGroup group = counters.getGroup(groupName); log.info("Group: " + group.getDisplayName()); for (Counter counter : group) { log.info(counter.getDisplayName() + ":\t" + counter.getValue());
/** * Increments multiple counters by their amounts in another Counters * instance. * @param other the other Counters instance */ public synchronized void incrAllCounters(Counters other) { for(Map.Entry<String, CounterGroup> rightEntry: other.groups.entrySet()) { CounterGroup left = groups.get(rightEntry.getKey()); CounterGroup right = rightEntry.getValue(); if (left == null) { left = new CounterGroup(right.getName(), right.getDisplayName()); groups.put(rightEntry.getKey(), left); } left.incrAllCounters(right); } }
/** * Return textual representation of the counter values. */ public synchronized String toString() { StringBuilder sb = new StringBuilder("Counters: " + countCounters()); for (CounterGroup group: this) { sb.append("\n\t" + group.getDisplayName()); for (Counter counter: group) { sb.append("\n\t\t" + counter.getDisplayName() + "=" + counter.getValue()); } } return sb.toString(); }
/** * Increments multiple counters by their amounts in another Counters * instance. * @param other the other Counters instance */ public synchronized void incrAllCounters(Counters other) { for(Map.Entry<String, CounterGroup> rightEntry: other.groups.entrySet()) { CounterGroup left = groups.get(rightEntry.getKey()); CounterGroup right = rightEntry.getValue(); if (left == null) { left = new CounterGroup(right.getName(), right.getDisplayName()); groups.put(rightEntry.getKey(), left); } left.incrAllCounters(right); } }
/** * Return textual representation of the counter values. */ public synchronized String toString() { StringBuilder sb = new StringBuilder("Counters: " + countCounters()); for (CounterGroup group: this) { sb.append("\n\t" + group.getDisplayName()); for (Counter counter: group) { sb.append("\n\t\t" + counter.getDisplayName() + "=" + counter.getValue()); } } return sb.toString(); }
@Override public JsonElement serialize(CounterGroup cg, Type t, JsonSerializationContext ctx) { JsonObject obj = new JsonObject(); if (!cg.getName().equals(cg.getDisplayName())) obj.addProperty("displayName", cg.getDisplayName()); JsonObject dns = new JsonObject(); boolean anyNamesDiffer = false; for (Counter c : cg) { obj.addProperty(c.getName(), c.getValue()); if (!c.getName().equals(c.getDisplayName())) anyNamesDiffer = true; dns.addProperty(c.getName(), c.getDisplayName()); } if (anyNamesDiffer) obj.add("displayNames", dns); return obj; } }
Counters counters = job.getCounters(); for (CounterGroup group : counters) { System.out.println("* Counter Group: " + group.getDisplayName() + " (" + group.getName() + ")"); System.out.println(" number of counters in this group: " + group.size()); for (Counter counter : group) { System.out.println(" - " + counter.getDisplayName() + ": " + counter.getName() + ": "+counter.getValue()); } }
private void validateCounters(org.apache.hadoop.mapreduce.Counters counters) { Iterator<org.apache.hadoop.mapreduce.CounterGroup> it = counters.iterator(); while (it.hasNext()) { org.apache.hadoop.mapreduce.CounterGroup group = it.next(); LOG.info("Group " + group.getDisplayName()); Iterator<org.apache.hadoop.mapreduce.Counter> itc = group.iterator(); while (itc.hasNext()) { LOG.info("Counter is " + itc.next().getDisplayName()); } } Assert.assertEquals(1, counters.countCounters()); }
private static void saveJobStats(Job job, StepExecution stepExecution) { if (stepExecution == null) { return; } ExecutionContext executionContext = stepExecution.getExecutionContext(); String statusPrefix = "Job Status::"; executionContext.put(statusPrefix + "ID", JobUtils.getJobId(job).toString()); executionContext.put(statusPrefix + "Name", job.getJobName()); executionContext.put(statusPrefix + "Tracking URL", job.getTrackingURL()); executionContext.put(statusPrefix + "State", JobUtils.getStatus(job).toString()); try { for (String cgName : job.getCounters().getGroupNames()) { CounterGroup group = job.getCounters().getGroup(cgName); Iterator<Counter> ci = group.iterator(); while (ci.hasNext()) { Counter c = ci.next(); executionContext.put(group.getDisplayName().trim() + "::" + c.getDisplayName().trim(), c.getValue()); } } } catch (Exception ignore) {} }
static JhCounters toAvro(Counters counters, String name) { JhCounters result = new JhCounters(); result.name = new Utf8(name); result.groups = new ArrayList<JhCounterGroup>(0); if (counters == null) return result; for (CounterGroup group : counters) { JhCounterGroup g = new JhCounterGroup(); g.name = new Utf8(group.getName()); g.displayName = new Utf8(group.getDisplayName()); g.counts = new ArrayList<JhCounter>(group.size()); for (Counter counter : group) { JhCounter c = new JhCounter(); c.name = new Utf8(counter.getName()); c.displayName = new Utf8(counter.getDisplayName()); c.value = counter.getValue(); g.counts.add(c); } result.groups.add(g); } return result; }
static JhCounters toAvro(Counters counters, String name) { JhCounters result = new JhCounters(); result.name = new Utf8(name); result.groups = new ArrayList<JhCounterGroup>(0); if (counters == null) return result; for (CounterGroup group : counters) { JhCounterGroup g = new JhCounterGroup(); g.name = new Utf8(group.getName()); g.displayName = new Utf8(group.getDisplayName()); g.counts = new ArrayList<JhCounter>(group.size()); for (Counter counter : group) { JhCounter c = new JhCounter(); c.name = new Utf8(counter.getName()); c.displayName = new Utf8(counter.getDisplayName()); c.value = counter.getValue(); g.counts.add(c); } result.groups.add(g); } return result; }
static JhCounters toAvro(Counters counters, String name) { JhCounters result = new JhCounters(); result.name = new Utf8(name); result.groups = new ArrayList<JhCounterGroup>(0); if (counters == null) return result; for (CounterGroup group : counters) { JhCounterGroup g = new JhCounterGroup(); g.name = new Utf8(group.getName()); g.displayName = new Utf8(group.getDisplayName()); g.counts = new ArrayList<JhCounter>(group.size()); for (Counter counter : group) { JhCounter c = new JhCounter(); c.name = new Utf8(counter.getName()); c.displayName = new Utf8(counter.getDisplayName()); c.value = counter.getValue(); g.counts.add(c); } result.groups.add(g); } return result; }
private void submitCountersToStatsd(Job job) throws IOException { Counters counters = job.getCounters(); if (getStatsdEnabled(job)) { StatsDClient statsd = new NonBlockingStatsDClient("Camus", getStatsdHost(job), getStatsdPort(job), new String[] { "camus:counters" }); for (CounterGroup counterGroup : counters) { for (Counter counter : counterGroup) { statsd.gauge(counterGroup.getDisplayName() + "." + counter.getDisplayName(), counter.getValue()); } } } }
private void submitCountersToStatsd(Job job) throws IOException { Counters counters = job.getCounters(); if (getStatsdEnabled(job)) { StatsDClient statsd = new NonBlockingStatsDClient("Camus", getStatsdHost(job), getStatsdPort(job), new String[] { "camus:counters" }); for (CounterGroup counterGroup : counters) { for (Counter counter : counterGroup) { statsd.gauge(counterGroup.getDisplayName() + "." + counter.getDisplayName(), counter.getValue()); } } } }
static JhCounters toAvro(Counters counters, String name) { JhCounters result = new JhCounters(); result.name = new Utf8(name); result.groups = new GenericData.Array<JhCounterGroup>(0, GROUPS); if (counters == null) return result; for (CounterGroup group : counters) { JhCounterGroup g = new JhCounterGroup(); g.name = new Utf8(group.getName()); g.displayName = new Utf8(group.getDisplayName()); g.counts = new GenericData.Array<JhCounter>(group.size(), COUNTERS); for (Counter counter : group) { JhCounter c = new JhCounter(); c.name = new Utf8(counter.getName()); c.displayName = new Utf8(counter.getDisplayName()); c.value = counter.getValue(); g.counts.add(c); } result.groups.add(g); } return result; }
public static Counters toYarn(org.apache.hadoop.mapreduce.Counters counters) { if (counters == null) { return null; } Counters yCntrs = recordFactory.newRecordInstance(Counters.class); yCntrs.addAllCounterGroups(new HashMap<String, CounterGroup>()); for (org.apache.hadoop.mapreduce.CounterGroup grp : counters) { CounterGroup yGrp = recordFactory.newRecordInstance(CounterGroup.class); yGrp.setName(grp.getName()); yGrp.setDisplayName(grp.getDisplayName()); yGrp.addAllCounters(new HashMap<String, Counter>()); for (org.apache.hadoop.mapreduce.Counter cntr : grp) { Counter yCntr = recordFactory.newRecordInstance(Counter.class); yCntr.setName(cntr.getName()); yCntr.setDisplayName(cntr.getDisplayName()); yCntr.setValue(cntr.getValue()); yGrp.setCounter(yCntr.getName(), yCntr); } yCntrs.setCounterGroup(yGrp.getName(), yGrp); } return yCntrs; }
public static Counters toYarn(org.apache.hadoop.mapreduce.Counters counters) { if (counters == null) { return null; } Counters yCntrs = recordFactory.newRecordInstance(Counters.class); yCntrs.addAllCounterGroups(new HashMap<String, CounterGroup>()); for (org.apache.hadoop.mapreduce.CounterGroup grp : counters) { CounterGroup yGrp = recordFactory.newRecordInstance(CounterGroup.class); yGrp.setName(grp.getName()); yGrp.setDisplayName(grp.getDisplayName()); yGrp.addAllCounters(new HashMap<String, Counter>()); for (org.apache.hadoop.mapreduce.Counter cntr : grp) { Counter yCntr = recordFactory.newRecordInstance(Counter.class); yCntr.setName(cntr.getName()); yCntr.setDisplayName(cntr.getDisplayName()); yCntr.setValue(cntr.getValue()); yGrp.setCounter(yCntr.getName(), yCntr); } yCntrs.setCounterGroup(yGrp.getName(), yGrp); } return yCntrs; }
@Private public JsonNode countersToJSON(Counters counters) { ObjectMapper mapper = new ObjectMapper(); ArrayNode nodes = mapper.createArrayNode(); if (counters != null) { for (CounterGroup counterGroup : counters) { ObjectNode groupNode = nodes.addObject(); groupNode.put("NAME", counterGroup.getName()); groupNode.put("DISPLAY_NAME", counterGroup.getDisplayName()); ArrayNode countersNode = groupNode.putArray("COUNTERS"); for (Counter counter : counterGroup) { ObjectNode counterNode = countersNode.addObject(); counterNode.put("NAME", counter.getName()); counterNode.put("DISPLAY_NAME", counter.getDisplayName()); counterNode.put("VALUE", counter.getValue()); } } } return nodes; }
@Private public JsonNode countersToJSON(Counters counters) { ArrayNode nodes = FACTORY.arrayNode(); if (counters != null) { for (CounterGroup counterGroup : counters) { ObjectNode groupNode = nodes.addObject(); groupNode.put("NAME", counterGroup.getName()); groupNode.put("DISPLAY_NAME", counterGroup.getDisplayName()); ArrayNode countersNode = groupNode.putArray("COUNTERS"); for (Counter counter : counterGroup) { ObjectNode counterNode = countersNode.addObject(); counterNode.put("NAME", counter.getName()); counterNode.put("DISPLAY_NAME", counter.getDisplayName()); counterNode.put("VALUE", counter.getValue()); } } } return nodes; }