Refine search
console.printInfo(tezTask.getId() + " HIVE COUNTERS:", false); for (TezCounter counter : group) { console.printInfo(" " + counter.getDisplayName() + ": " + counter.getValue(), false); console.printInfo(" " + counter.getDisplayName() + ": " + counter.getValue(), false); if (counter.getName().contains("HDFS")) { console.printInfo(" " + counter.getDisplayName() + ": " + counter.getValue(), false); List<String> testSafeCounters = LlapIOCounters.testSafeCounterNames(); for (TezCounter counter : group) { if (testSafeCounters.contains(counter.getDisplayName())) { console.printInfo(" " + counter.getDisplayName() + ": " + counter.getValue(), false);
public void incrWallClockCounter(LlapIOCounters counter, long startTime) { if (!doUseTimeCounters) return; long delta = System.nanoTime() - startTime; fixedCounters.addAndGet(counter.ordinal(), delta); if (tezCounters != null) { tezCounters.findCounter(LlapIOCounters.values()[counter.ordinal()]).increment(delta); } }
public void dumpToTezCounters(TezCounters tezCounters, boolean isLast) { if (isLast) { changeStateDone(); // Record the final counters. } for (int i = 0; i < fixedCounters.length(); ++i) { tezCounters.findCounter(LlapWmCounters.values()[i]).setValue(fixedCounters.get(i)); } }
console.printInfo(tezTask.getId() + " LLAP IO COUNTERS:", false); for (TezCounter counter : group) { if (counter.getDisplayName().equals(LlapIOCounters.SELECTED_ROWGROUPS.name())) { console.printInfo(" " + counter.getDisplayName() + ": " + counter.getValue(), false);
@Override protected long getCounterValue( TezCounters counterGroups, Enum counter ) { TezCounter tezCounter = counterGroups.findCounter( counter ); if( tezCounter == null ) return 0; return tezCounter.getValue(); }
String vertexName = null; if (inputInitializerContext != null) { tezCounters = new TezCounters(); groupName = HiveInputCounters.class.getName(); vertexName = jobConf.get(Operator.CONTEXT_NAME_KEY, ""); counterName = Utilities.getVertexCounterName(HiveInputCounters.RAW_INPUT_SPLITS.name(), vertexName); tezCounters.findCounter(groupName, counterName).increment(splits.length); final List<Path> paths = Utilities.getInputPathsTez(jobConf, work); counterName = Utilities.getVertexCounterName(HiveInputCounters.INPUT_DIRECTORIES.name(), vertexName); tezCounters.findCounter(groupName, counterName).increment(paths.size()); final Set<String> files = new HashSet<>(); for (InputSplit inputSplit : splits) { tezCounters.findCounter(groupName, counterName).increment(files.size()); if (inputInitializerContext != null) { counterName = Utilities.getVertexCounterName(HiveInputCounters.GROUPED_INPUT_SPLITS.name(), vertexName); tezCounters.findCounter(groupName, counterName).setValue(flatSplits.length);
private void adjustSpillCounters(long rawLength, long compLength) { if (!isFinalMergeEnabled()) { outputBytesWithOverheadCounter.increment(rawLength); } else { if (numSpills > 0) { additionalSpillBytesWritten.increment(compLength); // Reset the value will be set during the final merge. outputBytesWithOverheadCounter.setValue(0); } else { // Set this up for the first write only. Subsequent ones will be handled in the final merge. outputBytesWithOverheadCounter.increment(rawLength); } } }
@Override public void incrAllCounters(CounterGroupBase<T> rightGroup) { try { for (TezCounter right : rightGroup) { TezCounter left = findCounter(right.getName(), right.getDisplayName()); left.increment(right.getValue()); } } catch (LimitExceededException e) { counters.clear(); throw e; } } }
void init(TezCounter readsCounter, TezCounter bytesReadCounter) throws IOException { if (mapOutputsCounter != null) { mapOutputsCounter.increment(1); } }
Map<String, Long> matchedCounters = new HashMap<String, Long>(); for (TezCounter counter : grpCounters) { matchedCounters.put(counter.getName(), counter.getValue()); for (Entry<String, Set<String>> entry : counterNames.entrySet()) { Map<String, Long> matchedCounters = new HashMap<String, Long>(); CounterGroup grpCounters = counters.getGroup(entry.getKey()); for (TezCounter counter : grpCounters) { if (entry.getValue().isEmpty() || entry.getValue().contains(counter.getName())) { matchedCounters.put(counter.getName(), counter.getValue());
public static TezCountersProto convertTezCountersToProto( TezCounters counters) { TezCountersProto.Builder builder = TezCountersProto.newBuilder(); Iterator<CounterGroup> groupIterator = counters.iterator(); int groupIndex = 0; while (groupIterator.hasNext()) { CounterGroup counterGroup = groupIterator.next(); TezCounterGroupProto.Builder groupBuilder = TezCounterGroupProto.newBuilder(); groupBuilder.setName(counterGroup.getName()); groupBuilder.setDisplayName(counterGroup.getDisplayName()); Iterator<TezCounter> counterIterator = counterGroup.iterator(); int counterIndex = 0; while (counterIterator.hasNext()) { TezCounter counter = counterIterator.next(); TezCounterProto tezCounterProto = TezCounterProto.newBuilder() .setName(counter.getName()) .setDisplayName(counter.getDisplayName()) .setValue(counter.getValue()) .build(); groupBuilder.addCounters(counterIndex, tezCounterProto); ++counterIndex; } builder.addCounterGroups(groupIndex, groupBuilder.build()); ++groupIndex; } return builder.build(); }
public void setLocalityCounter(DAGCounter localityCounter) { if (!localitySet.get()) { localitySet.set(true); if (counters == null) { counters = new TezCounters(); } if (localityCounter != null) { counters.findCounter(localityCounter).increment(1); // TODO Maybe validate that the correct value is being set. } } } }
@Override public void incrAllCounters(CounterGroupBase<C> other) { if (checkNotNull(other.getUnderlyingGroup(), "other group") instanceof FileSystemCounterGroup<?>) { for (TezCounter counter : other) { FSCounter c = (FSCounter) ((TezCounter)counter).getUnderlyingCounter(); findCounter(c.scheme, c.key) .increment(counter.getValue()); } } }
@SuppressWarnings("rawtypes") @Override public void incrAllCounters(CounterGroupBase<C> other) { if (checkNotNull(other, "other counter group") instanceof FrameworkCounterGroup<?, ?>) { for (TezCounter counter : other) { findCounter(((FrameworkCounter) counter).key.name()) .increment(counter.getValue()); } } }