static long getCounterValueByGroupName(TezCounters vertexCounters, String groupNamePattern, String counterName) { TezCounter tezCounter = vertexCounters.getGroup(groupNamePattern).findCounter(counterName); return (tezCounter == null) ? 0 : tezCounter.getValue(); }
@Override public String aggregateStats(String keyPrefix, String statType) { String result; if (delegate) { result = mrAggregator.aggregateStats(keyPrefix, statType); } else { long value = 0; for (String groupName : counters.getGroupNames()) { if (groupName.startsWith(keyPrefix)) { value += counters.getGroup(groupName).findCounter(statType).getValue(); } } result = String.valueOf(value); } LOG.info("Counter based stats for ("+keyPrefix+") are: "+result); return result; }
private long getCounterValueByGroupName(TezCounters counters, String pattern, String counterName) { TezCounter tezCounter = counters.getGroup(pattern).findCounter(counterName); return (tezCounter == null) ? 0 : tezCounter.getValue(); }
boolean found = false; while (iterator.hasNext()) { CounterGroup counterGroup = tezCounters.getGroup(iterator.next()); if (counterGroupName != null) { String groupName = counterGroup.getName();
static long getCounterValueByGroupName(TezCounters vertexCounters, String groupNamePattern, String counterName) { TezCounter tezCounter = vertexCounters.getGroup(groupNamePattern).findCounter(counterName); return (tezCounter == null) ? 0 : tezCounter.getValue(); }
private long getCounterValueByGroupName(TezCounters counters, String pattern, String counterName) { TezCounter tezCounter = counters.getGroup(pattern).findCounter(counterName); return (tezCounter == null) ? 0 : tezCounter.getValue(); }
private long hiveInputRecordsFromTezCounters(String vertexName, String inputVertexName) { // Get the counters for the input vertex. Set<StatusGetOpts> statusOptions = new HashSet<>(1); statusOptions.add(StatusGetOpts.GET_COUNTERS); VertexStatus inputVertexStatus = vertexStatus(statusOptions, inputVertexName); final TezCounters inputVertexCounters = inputVertexStatus.getVertexCounters(); // eg, group name TaskCounter_Map_7_OUTPUT_Reducer_8, counter name OUTPUT_RECORDS String groupName = formattedName("TaskCounter", inputVertexName, vertexName); String counterName = "OUTPUT_RECORDS"; // Do not create counter if it does not exist - // instead fall back to default behavior for determining input records. TezCounter tezCounter = inputVertexCounters.getGroup(groupName).findCounter(counterName, false); if (tezCounter == null) { return -1; } else { return tezCounter.getValue(); } }
private long hiveInputRecordsFromTezCounters(String vertexName, String inputVertexName) { // Get the counters for the input vertex. Set<StatusGetOpts> statusOptions = new HashSet<>(1); statusOptions.add(StatusGetOpts.GET_COUNTERS); VertexStatus inputVertexStatus = vertexStatus(statusOptions, inputVertexName); final TezCounters inputVertexCounters = inputVertexStatus.getVertexCounters(); // eg, group name TaskCounter_Map_7_OUTPUT_Reducer_8, counter name OUTPUT_RECORDS String groupName = formattedName("TaskCounter", inputVertexName, vertexName); String counterName = "OUTPUT_RECORDS"; // Do not create counter if it does not exist - // instead fall back to default behavior for determining input records. TezCounter tezCounter = inputVertexCounters.getGroup(groupName).findCounter(counterName, false); if (tezCounter == null) { return -1; } else { return tezCounter.getValue(); } }
OperatorStats operatorStats = null; String counterName = Operator.Counter.RECORDS_OUT_OPERATOR.toString() + "_" + operatorId; TezCounter tezCounter = counters.getGroup(groupName).findCounter(counterName, false); if (tezCounter != null) { if (operatorStats == null) {
@Override protected Set<String> getCountersFor( TezCounters counterGroups, String group ) { Set<String> results = new HashSet<>(); for( TezCounter counter : counterGroups.getGroup( group ) ) results.add( counter.getName() ); return results; }
@Override protected Set<String> getCountersFor( TezCounters counterGroups, String group ) { Set<String> results = new HashSet<>(); for( TezCounter counter : counterGroups.getGroup( group ) ) results.add( counter.getName() ); return results; }
@Override public synchronized org.apache.hadoop.mapred.Counters.Group getGroup(String groupName) { return new MRCounterGroup(raw.getGroup(groupName)); }
@Override protected long getCounterValue( TezCounters counterGroups, String groupName, String counterName ) { CounterGroup counterGroup = counterGroups.getGroup( groupName ); if( counterGroup == null ) return 0; TezCounter counterValue = counterGroup.findCounter( counterName ); if( counterValue == null ) return 0; return counterValue.getValue(); }
@Override protected long getCounterValue( TezCounters counterGroups, String groupName, String counterName ) { CounterGroup counterGroup = counterGroups.getGroup( groupName ); if( counterGroup == null ) return 0; TezCounter counterValue = counterGroup.findCounter( counterName ); if( counterValue == null ) return 0; return counterValue.getValue(); }
private static long getCounterValueByGroupName(TezCounters vertexCounters, String groupNamePattern, String counterName) { TezCounter tezCounter = vertexCounters.getGroup(groupNamePattern).findCounter(counterName); return (tezCounter == null) ? 0 : tezCounter.getValue(); }
for (Entry<String, Set<String>> entry : counterNames.entrySet()) { Map<String, Long> matchedCounters = new HashMap<String, Long>(); CounterGroup grpCounters = counters.getGroup(entry.getKey()); for (TezCounter counter : grpCounters) { if (entry.getValue().isEmpty() || entry.getValue().contains(counter.getName())) {
@Override public boolean incrCounter(String group, String name, long delta) { if (context == null) { return false; } TezCounter counter = context.getCounters().getGroup(group).findCounter(name); counter.increment(delta); return true; } }
@Override public Counter getCounter(String group, String name) { if (context == null) { return null; } TezCounter tezCounter = context.getCounters().getGroup(group).findCounter(name); return new MRCounter(tezCounter); }
.getGroup(MRPigStatsUtil.MULTI_INPUTS_COUNTER_GROUP); if (multiInputGroup == null) { processorContext.getCounters().addGroup(
@Override public void attachOutputs(Map<String, LogicalOutput> outputs, Configuration conf) throws ExecException { LogicalOutput logicalOut = outputs.get(outputKey); if (logicalOut == null || !(logicalOut instanceof MROutput)) { throw new ExecException("POStoreTez only accepts MROutput. key = " + getOperatorKey() + ", outputs = " + outputs); } output = (MROutput) logicalOut; try { writer = output.getWriter(); } catch (IOException e) { throw new ExecException(e); } // Even if there is a single hdfs output, we add multi store counter // Makes it easier for user to see records for a particular store from // the DAG counter CounterGroup multiStoreGroup = processorContext.getCounters() .getGroup(MRPigStatsUtil.MULTI_STORE_COUNTER_GROUP); if (multiStoreGroup == null) { processorContext.getCounters().addGroup( MRPigStatsUtil.MULTI_STORE_COUNTER_GROUP, MRPigStatsUtil.MULTI_STORE_COUNTER_GROUP); } String name = MRPigStatsUtil.getMultiStoreCounterName(this); if (name != null) { outputRecordCounter = multiStoreGroup.addCounter(name, name, 0); } }