private void checkIfTooManySkippedMsg(Counters counters) { double maxPercentSkippedSchemaNotFound = Double.parseDouble(props.getProperty(ETL_MAX_PERCENT_SKIPPED_SCHEMANOTFOUND, ETL_MAX_PERCENT_SKIPPED_SCHEMANOTFOUND_DEFAULT)); double maxPercentSkippedOther = Double.parseDouble(props.getProperty(ETL_MAX_PERCENT_SKIPPED_OTHER, ETL_MAX_PERCENT_SKIPPED_OTHER_DEFAULT)); long actualSkippedSchemaNotFound = 0; long actualSkippedOther = 0; long actualDecodeSuccessful = 0; for (String groupName : counters.getGroupNames()) { if (groupName.equals(EtlRecordReader.KAFKA_MSG.class.getName())) { CounterGroup group = counters.getGroup(groupName); for (Counter counter : group) { if (counter.getDisplayName().equals(EtlRecordReader.KAFKA_MSG.DECODE_SUCCESSFUL.toString())) { actualDecodeSuccessful = counter.getValue(); } else if (counter.getDisplayName().equals(EtlRecordReader.KAFKA_MSG.SKIPPED_SCHEMA_NOT_FOUND.toString())) { actualSkippedSchemaNotFound = counter.getValue(); } else if (counter.getDisplayName().equals(EtlRecordReader.KAFKA_MSG.SKIPPED_OTHER.toString())) { actualSkippedOther = counter.getValue(); } } } } checkIfTooManySkippedMsg(maxPercentSkippedSchemaNotFound, actualSkippedSchemaNotFound, actualDecodeSuccessful, "schema not found"); checkIfTooManySkippedMsg(maxPercentSkippedOther, actualSkippedOther, actualDecodeSuccessful, "other"); }
for (String groupName : counters.getGroupNames()) { CounterGroup group = counters.getGroup(groupName); log.info("Group: " + group.getDisplayName());
@Override protected Collection<String> getGroupNames( Counters groups ) { LinkedHashSet<String> result = new LinkedHashSet<>(); for( String value : groups.getGroupNames() ) result.add( value ); return result; }
@Override protected Collection<String> getGroupNames( Counters groups ) { LinkedHashSet<String> result = new LinkedHashSet<>(); for( String value : groups.getGroupNames() ) result.add( value ); return result; }
public Iterable<String> getGroupNames() { if (mapred != null) { return mapred.getGroupNames(); } else { return mapreduce.getGroupNames(); } }
public String toString() { StringBuilder builder = new StringBuilder(); while (source.hasNext()) { Entry<String,Counters> nextCntr = source.next(); builder.append("\n").append(nextCntr.getKey()).append("\n----------------------\n"); Counters counters = nextCntr.getValue(); for (String groupName : counters.getGroupNames()) { CounterGroup group = counters.getGroup(groupName); Iterator<Counter> cntrItr = group.iterator(); while (cntrItr.hasNext()) { Counter counter = cntrItr.next(); builder.append(groupName).append("\t").append(counter.getDisplayName()).append("=").append(counter.getValue()).append("\n"); } } } return builder.toString(); }
/** * Extract the counters from a Job. * * @param job Job from which to get counters. * @return a map from counters to their counts. Keys are group:name. * @throws IOException in case of an error getting the counters. */ private static Map<String, Long> getCounters( final Job job ) throws IOException { final Counters counters = job.getCounters(); final Map<String, Long> countersMap = Maps.newHashMap(); for (String group : counters.getGroupNames()) { for (Counter counter : counters.getGroup(group)) { countersMap.put(String.format("%s:%s", group, counter.getName()), counter.getValue()); } } return countersMap; }
buff.append("\n------------------------------------------"+ "---------------------------------------------"); for (String groupName : totalCounters.getGroupNames()) { CounterGroup totalGroup = totalCounters.getGroup(groupName); CounterGroup mapGroup = mapCounters.getGroup(groupName);
private void checkIfTooManySkippedMsg(Counters counters) { double maxPercentSkippedSchemaNotFound = Double.parseDouble(props.getProperty(ETL_MAX_PERCENT_SKIPPED_SCHEMANOTFOUND, ETL_MAX_PERCENT_SKIPPED_SCHEMANOTFOUND_DEFAULT)); double maxPercentSkippedOther = Double.parseDouble(props.getProperty(ETL_MAX_PERCENT_SKIPPED_OTHER, ETL_MAX_PERCENT_SKIPPED_OTHER_DEFAULT)); long actualSkippedSchemaNotFound = 0; long actualSkippedOther = 0; long actualDecodeSuccessful = 0; for (String groupName : counters.getGroupNames()) { if (groupName.equals(EtlRecordReader.KAFKA_MSG.class.getName())) { CounterGroup group = counters.getGroup(groupName); for (Counter counter : group) { if (counter.getDisplayName().equals(EtlRecordReader.KAFKA_MSG.DECODE_SUCCESSFUL.toString())) { actualDecodeSuccessful = counter.getValue(); } else if (counter.getDisplayName().equals(EtlRecordReader.KAFKA_MSG.SKIPPED_SCHEMA_NOT_FOUND.toString())) { actualSkippedSchemaNotFound = counter.getValue(); } else if (counter.getDisplayName().equals(EtlRecordReader.KAFKA_MSG.SKIPPED_OTHER.toString())) { actualSkippedOther = counter.getValue(); } } } } checkIfTooManySkippedMsg(maxPercentSkippedSchemaNotFound, actualSkippedSchemaNotFound, actualDecodeSuccessful, "schema not found"); checkIfTooManySkippedMsg(maxPercentSkippedOther, actualSkippedOther, actualDecodeSuccessful, "other"); }
buff.append("\n------------------------------------------"+ "---------------------------------------------"); for (String groupName : totalCounters.getGroupNames()) { CounterGroup totalGroup = totalCounters.getGroup(groupName); CounterGroup mapGroup = mapCounters.getGroup(groupName);
buff.append("\n------------------------------------------"+ "---------------------------------------------"); for (String groupName : totalCounters.getGroupNames()) { CounterGroup totalGroup = totalCounters.getGroup(groupName); CounterGroup mapGroup = mapCounters.getGroup(groupName);
buff.append("\n------------------------------------------"+ "---------------------------------------------"); for (String groupName : totalCounters.getGroupNames()) { CounterGroup totalGroup = totalCounters.getGroup(groupName); CounterGroup mapGroup = mapCounters.getGroup(groupName);
buff.append("\n------------------------------------------"+ "---------------------------------------------"); for (String groupName : totalCounters.getGroupNames()) { CounterGroup totalGroup = totalCounters.getGroup(groupName); CounterGroup mapGroup = mapCounters.getGroup(groupName);
for (String groupName : counters.getGroupNames()) { CounterGroup group = counters.getGroup(groupName); log.info("Group: " + group.getDisplayName());
private static void saveJobStats(Job job, StepExecution stepExecution) { if (stepExecution == null) { return; } ExecutionContext executionContext = stepExecution.getExecutionContext(); String statusPrefix = "Job Status::"; executionContext.put(statusPrefix + "ID", JobUtils.getJobId(job).toString()); executionContext.put(statusPrefix + "Name", job.getJobName()); executionContext.put(statusPrefix + "Tracking URL", job.getTrackingURL()); executionContext.put(statusPrefix + "State", JobUtils.getStatus(job).toString()); try { for (String cgName : job.getCounters().getGroupNames()) { CounterGroup group = job.getCounters().getGroup(cgName); Iterator<Counter> ci = group.iterator(); while (ci.hasNext()) { Counter c = ci.next(); executionContext.put(group.getDisplayName().trim() + "::" + c.getDisplayName().trim(), c.getValue()); } } } catch (Exception ignore) {} }
BufferedOutputStream buffer = new BufferedOutputStream(counterStream,256*1024); OutputStreamWriter writer = new OutputStreamWriter(buffer); for (String groupName : getCounters().getGroupNames())
for (String groupName : counters.getGroupNames()) { CounterGroup group = counters.getGroup(groupName); String displayGroupName = group.getDisplayName();
if(total == null || total.getGroupNames() == null || total.countCounters() == 0) { String type = $(TASK_ID); if(type == null || type.isEmpty()) {
if(total == null || total.getGroupNames() == null || total.countCounters() == 0) { String type = $(TASK_ID); if(type == null || type.isEmpty()) {
if(total == null || total.getGroupNames() == null || total.countCounters() == 0) { String type = $(TASK_ID); if(type == null || type.isEmpty()) {