Refine search
public void incrWallClockCounter(LlapIOCounters counter, long startTime) { if (!doUseTimeCounters) return; long delta = System.nanoTime() - startTime; fixedCounters.addAndGet(counter.ordinal(), delta); if (tezCounters != null) { tezCounters.findCounter(LlapIOCounters.values()[counter.ordinal()]).increment(delta); } }
String vertexName = null; if (inputInitializerContext != null) { tezCounters = new TezCounters(); groupName = HiveInputCounters.class.getName(); vertexName = jobConf.get(Operator.CONTEXT_NAME_KEY, ""); counterName = Utilities.getVertexCounterName(HiveInputCounters.RAW_INPUT_SPLITS.name(), vertexName); tezCounters.findCounter(groupName, counterName).increment(splits.length); final List<Path> paths = Utilities.getInputPathsTez(jobConf, work); counterName = Utilities.getVertexCounterName(HiveInputCounters.INPUT_DIRECTORIES.name(), vertexName); tezCounters.findCounter(groupName, counterName).increment(paths.size()); final Set<String> files = new HashSet<>(); for (InputSplit inputSplit : splits) { tezCounters.findCounter(groupName, counterName).increment(files.size()); if (inputInitializerContext != null) { counterName = Utilities.getVertexCounterName(HiveInputCounters.GROUPED_INPUT_SPLITS.name(), vertexName); tezCounters.findCounter(groupName, counterName).setValue(flatSplits.length);
@Override protected long getCounterValue( TezCounters counterGroups, Enum counter ) { TezCounter tezCounter = counterGroups.findCounter( counter ); if( tezCounter == null ) return 0; return tezCounter.getValue(); }
public void setCounter(LlapIOCounters counter, long value) { fixedCounters.set(counter.ordinal(), value); if (tezCounters != null) { tezCounters.findCounter(LlapIOCounters.values()[counter.ordinal()]).setValue(value); } }
@Override public TezCounter findCounter(String groupName, String counterName) { String simpleGroupName; if (groupName.equals(TaskCounter.class.getName())) { simpleGroupName = TaskCounter.class.getSimpleName(); } else { simpleGroupName = groupName; } String modifiedGroupName = simpleGroupName + "_" + this.groupModifier; final TezCounter modifiedGroupCounter = original.findCounter(modifiedGroupName, counterName); final TezCounter originalGroupCounter = original.findCounter(groupName, counterName); return new CompositeCounter(modifiedGroupCounter, originalGroupCounter); }
Map<String, Long> matchedCounters = new HashMap<String, Long>(); for (TezCounter counter : grpCounters) { matchedCounters.put(counter.getName(), counter.getValue()); for (Entry<String, Set<String>> entry : counterNames.entrySet()) { Map<String, Long> matchedCounters = new HashMap<String, Long>(); CounterGroup grpCounters = counters.getGroup(entry.getKey()); for (TezCounter counter : grpCounters) { if (entry.getValue().isEmpty() || entry.getValue().contains(counter.getName())) { matchedCounters.put(counter.getName(), counter.getValue());
public void setLocalityCounter(DAGCounter localityCounter) { if (!localitySet.get()) { localitySet.set(true); if (counters == null) { counters = new TezCounters(); } if (localityCounter != null) { counters.findCounter(localityCounter).increment(1); // TODO Maybe validate that the correct value is being set. } } } }
@Override public String aggregateStats(String keyPrefix, String statType) { String result; if (delegate) { result = mrAggregator.aggregateStats(keyPrefix, statType); } else { long value = 0; for (String groupName : counters.getGroupNames()) { if (groupName.startsWith(keyPrefix)) { value += counters.getGroup(groupName).findCounter(statType).getValue(); } } result = String.valueOf(value); } LOG.info("Counter based stats for ("+keyPrefix+") are: "+result); return result; }
public TezCounters getCounters() { TezCounters fullCounters = new TezCounters(); fullCounters.incrAllCounters(tezCounters); for (TezCounters counter : counterMap.values()) { fullCounters.incrAllCounters(counter); } return fullCounters; }
@Override public void attachOutputs(Map<String, LogicalOutput> outputs, Configuration conf) throws ExecException { LogicalOutput logicalOut = outputs.get(outputKey); if (logicalOut == null || !(logicalOut instanceof MROutput)) { throw new ExecException("POStoreTez only accepts MROutput. key = " + getOperatorKey() + ", outputs = " + outputs); } output = (MROutput) logicalOut; try { writer = output.getWriter(); } catch (IOException e) { throw new ExecException(e); } // Even if there is a single hdfs output, we add multi store counter // Makes it easier for user to see records for a particular store from // the DAG counter CounterGroup multiStoreGroup = processorContext.getCounters() .getGroup(MRPigStatsUtil.MULTI_STORE_COUNTER_GROUP); if (multiStoreGroup == null) { processorContext.getCounters().addGroup( MRPigStatsUtil.MULTI_STORE_COUNTER_GROUP, MRPigStatsUtil.MULTI_STORE_COUNTER_GROUP); } String name = MRPigStatsUtil.getMultiStoreCounterName(this); if (name != null) { outputRecordCounter = multiStoreGroup.addCounter(name, name, 0); } }
public static TezCounters convertTezCountersFromProto(TezCountersProto proto) { TezCounters counters = new TezCounters(); for (TezCounterGroupProto counterGroupProto : proto.getCounterGroupsList()) { CounterGroup group = counters.addGroup(counterGroupProto.getName(), counterGroupProto.getDisplayName()); for (TezCounterProto counterProto : counterGroupProto.getCountersList()) { TezCounter counter = group.findCounter( counterProto.getName(), counterProto.getDisplayName()); counter.setValue(counterProto.getValue()); } } return counters; }
Iterator<String> iterator = tezCounters.getGroupNames().iterator(); boolean found = false; while (iterator.hasNext()) { CounterGroup counterGroup = tezCounters.getGroup(iterator.next()); if (counterGroupName != null) { String groupName = counterGroup.getName();
@Override public synchronized org.apache.hadoop.mapred.Counters.Group getGroup(String groupName) { return new MRCounterGroup(raw.getGroup(groupName)); }
@Override protected Collection<String> getGroupNames( TezCounters counterGroups ) { Iterable<String> iterable = counterGroups.getGroupNames(); if( iterable == null ) // it's hadoop, be defensive return Collections.emptySet(); Set<String> groupNames = new HashSet<>(); for( String groupName : iterable ) groupNames.add( groupName ); return groupNames; }
TezCounters counters = new TezCounters(); Constants.COUNTER_GROUP_DISPLAY_NAME, groupName); CounterGroup group = counters.addGroup(groupName, groupDisplayName);
public TezCounters addAndGetTezCounter(String name) { TezCounters counter = new TezCounters(); counterMap.put(name, counter); return counter; }
@Override public void readFields(DataInput in) throws IOException { progress = in.readFloat(); progressNotified = in.readBoolean(); if (in.readBoolean()) { tezCounters = new TezCounters(); tezCounters.readFields(in); } if (in.readBoolean()) { statistics = new TaskStatistics(); statistics.readFields(in); } }