static long getCounterValueByGroupName(TezCounters vertexCounters, String groupNamePattern, String counterName) { TezCounter tezCounter = vertexCounters.getGroup(groupNamePattern).findCounter(counterName); return (tezCounter == null) ? 0 : tezCounter.getValue(); }
private void maybeLogCounters() { if (LOG.isDebugEnabled()) { if (nonOobHeartbeatCounter.get() == nextHeartbeatNumToLog) { LOG.debug("Counters: " + task.getCounters().toShortString()); nextHeartbeatNumToLog = (int) (nextHeartbeatNumToLog * (LOG_COUNTER_BACKOFF)); } } }
String hiveCountersGroup = HiveConf.getVar(conf, HiveConf.ConfVars.HIVECOUNTERGROUP); for (CounterGroup group : counters) { if (hiveCountersGroup.equals(group.getDisplayName())) { console.printInfo(tezTask.getId() + " HIVE COUNTERS:", false); for (TezCounter counter : group) { console.printInfo(" " + counter.getDisplayName() + ": " + counter.getValue(), false); } else if (group.getName().equals(HiveInputCounters.class.getName())) { console.printInfo(tezTask.getId() + " INPUT COUNTERS:", false); for (TezCounter counter : group) { console.printInfo(" " + counter.getDisplayName() + ": " + counter.getValue(), false); } else if (group.getName().equals(FileSystemCounter.class.getName())) { console.printInfo(tezTask.getId() + " FILE SYSTEM COUNTERS:", false); for (TezCounter counter : group) { if (counter.getName().contains("HDFS")) { console.printInfo(" " + counter.getDisplayName() + ": " + counter.getValue(), false); } else if (group.getName().equals(LlapIOCounters.class.getName())) { console.printInfo(tezTask.getId() + " LLAP IO COUNTERS:", false); List<String> testSafeCounters = LlapIOCounters.testSafeCounterNames(); for (TezCounter counter : group) { if (testSafeCounters.contains(counter.getDisplayName())) { console.printInfo(" " + counter.getDisplayName() + ": " + counter.getValue(), false);
public void incrCounter(LlapIOCounters counter, long delta) { fixedCounters.addAndGet(counter.ordinal(), delta); if (tezCounters != null) { tezCounters.findCounter(LlapIOCounters.values()[counter.ordinal()]).increment(delta); } }
if (counters != null) { for (CounterGroup group : counters) { if (group.getName().equals(LlapIOCounters.class.getName())) { console.printInfo(tezTask.getId() + " LLAP IO COUNTERS:", false); for (TezCounter counter : group) { if (counter.getDisplayName().equals(LlapIOCounters.SELECTED_ROWGROUPS.name())) { console.printInfo(" " + counter.getDisplayName() + ": " + counter.getValue(), false);
public void setCounter(LlapIOCounters counter, long value) { fixedCounters.set(counter.ordinal(), value); if (tezCounters != null) { tezCounters.findCounter(LlapIOCounters.values()[counter.ordinal()]).setValue(value); } }
private String summary(String scheme, String vertexName, TezCounters vertexCounters) { final String counterGroup = FileSystemCounter.class.getName(); final long bytesRead = getCounterValueByGroupName(vertexCounters, counterGroup, scheme + "_" + FileSystemCounter.BYTES_READ.name()); final long bytesWritten = getCounterValueByGroupName(vertexCounters, counterGroup, scheme + "_" + FileSystemCounter.BYTES_WRITTEN.name()); final long readOps = getCounterValueByGroupName(vertexCounters, counterGroup, scheme + "_" + FileSystemCounter.READ_OPS.name()); final long largeReadOps = getCounterValueByGroupName(vertexCounters, counterGroup, scheme + "_" + FileSystemCounter.LARGE_READ_OPS.name()); final long writeOps = getCounterValueByGroupName(vertexCounters, counterGroup, scheme + "_" + FileSystemCounter.WRITE_OPS.name()); return String.format(FORMATTING_PATTERN, vertexName, Utilities.humanReadableByteCount(bytesRead), readOps, largeReadOps, Utilities.humanReadableByteCount(bytesWritten), writeOps); }
private long getCounterValueByGroupName(TezCounters counters, String pattern, String counterName) { TezCounter tezCounter = counters.getGroup(pattern).findCounter(counterName); return (tezCounter == null) ? 0 : tezCounter.getValue(); }
String hiveCountersGroup = HiveConf.getVar(conf, HiveConf.ConfVars.HIVECOUNTERGROUP); for (CounterGroup group : counters) { if (hiveCountersGroup.equals(group.getDisplayName())) { console.printInfo(tezTask.getId() + " HIVE COUNTERS:", false); for (TezCounter counter : group) { console.printInfo(" " + counter.getDisplayName() + ": " + counter.getValue(), false); } else if (group.getName().equals(FileSystemCounter.class.getName())) { console.printInfo(tezTask.getId() + " FILE SYSTEM COUNTERS:", false); for (TezCounter counter : group) { if (counter.getName().contains("HDFS")) { console.printInfo(" " + counter.getDisplayName() + ": " + counter.getValue(), false); } else if (group.getName().equals(LlapIOCounters.class.getName())) { console.printInfo(tezTask.getId() + " LLAP IO COUNTERS:", false); List<String> testSafeCounters = LlapIOCounters.testSafeCounterNames(); for (TezCounter counter : group) { if (testSafeCounters.contains(counter.getDisplayName())) { console.printInfo(" " + counter.getDisplayName() + ": " + counter.getValue(), false);
public void incrWallClockCounter(LlapIOCounters counter, long startTime) { if (!doUseTimeCounters) return; long delta = System.nanoTime() - startTime; fixedCounters.addAndGet(counter.ordinal(), delta); if (tezCounters != null) { tezCounters.findCounter(LlapIOCounters.values()[counter.ordinal()]).increment(delta); } }
if (counters != null) { for (CounterGroup group : counters) { if (group.getName().equals(LlapIOCounters.class.getName())) { console.printInfo(tezTask.getId() + " LLAP IO COUNTERS:", false); for (TezCounter counter : group) { if (counter.getDisplayName().equals(LlapIOCounters.SELECTED_ROWGROUPS.name())) { console.printInfo(" " + counter.getDisplayName() + ": " + counter.getValue(), false);
public void dumpToTezCounters(TezCounters tezCounters, boolean isLast) { if (isLast) { changeStateDone(); // Record the final counters. } for (int i = 0; i < fixedCounters.length(); ++i) { tezCounters.findCounter(LlapWmCounters.values()[i]).setValue(fixedCounters.get(i)); } }
private String summary(String scheme, String vertexName, TezCounters vertexCounters) { final String counterGroup = FileSystemCounter.class.getName(); final long bytesRead = getCounterValueByGroupName(vertexCounters, counterGroup, scheme + "_" + FileSystemCounter.BYTES_READ.name()); final long bytesWritten = getCounterValueByGroupName(vertexCounters, counterGroup, scheme + "_" + FileSystemCounter.BYTES_WRITTEN.name()); final long readOps = getCounterValueByGroupName(vertexCounters, counterGroup, scheme + "_" + FileSystemCounter.READ_OPS.name()); final long largeReadOps = getCounterValueByGroupName(vertexCounters, counterGroup, scheme + "_" + FileSystemCounter.LARGE_READ_OPS.name()); final long writeOps = getCounterValueByGroupName(vertexCounters, counterGroup, scheme + "_" + FileSystemCounter.WRITE_OPS.name()); return String.format(FORMATTING_PATTERN, vertexName, Utilities.humanReadableByteCount(bytesRead), readOps, largeReadOps, Utilities.humanReadableByteCount(bytesWritten), writeOps); }
static long getCounterValueByGroupName(TezCounters vertexCounters, String groupNamePattern, String counterName) { TezCounter tezCounter = vertexCounters.getGroup(groupNamePattern).findCounter(counterName); return (tezCounter == null) ? 0 : tezCounter.getValue(); }
tezCounters.findCounter(LlapExecutorCounters.EXECUTOR_CPU_NS).increment(cpuTime); tezCounters.findCounter(LlapExecutorCounters.EXECUTOR_USER_NS).increment(userTime); writeOpsDelta = threadFSStats.getWriteOps(); tezCounters.findCounter(scheme, FileSystemCounter.BYTES_READ) .increment(bytesReadDelta); tezCounters.findCounter(scheme, FileSystemCounter.BYTES_WRITTEN) .increment(bytesWrittenDelta); tezCounters.findCounter(scheme, FileSystemCounter.READ_OPS).increment(readOpsDelta); tezCounters.findCounter(scheme, FileSystemCounter.LARGE_READ_OPS) .increment(largeReadOpsDelta); tezCounters.findCounter(scheme, FileSystemCounter.WRITE_OPS) .increment(writeOpsDelta);
private long getCounterValueByGroupName(TezCounters counters, String pattern, String counterName) { TezCounter tezCounter = counters.getGroup(pattern).findCounter(counterName); return (tezCounter == null) ? 0 : tezCounter.getValue(); }
private long hiveInputRecordsFromTezCounters(String vertexName, String inputVertexName) { // Get the counters for the input vertex. Set<StatusGetOpts> statusOptions = new HashSet<>(1); statusOptions.add(StatusGetOpts.GET_COUNTERS); VertexStatus inputVertexStatus = vertexStatus(statusOptions, inputVertexName); final TezCounters inputVertexCounters = inputVertexStatus.getVertexCounters(); // eg, group name TaskCounter_Map_7_OUTPUT_Reducer_8, counter name OUTPUT_RECORDS String groupName = formattedName("TaskCounter", inputVertexName, vertexName); String counterName = "OUTPUT_RECORDS"; // Do not create counter if it does not exist - // instead fall back to default behavior for determining input records. TezCounter tezCounter = inputVertexCounters.getGroup(groupName).findCounter(counterName, false); if (tezCounter == null) { return -1; } else { return tezCounter.getValue(); } }
private long hiveInputRecordsFromTezCounters(String vertexName, String inputVertexName) { // Get the counters for the input vertex. Set<StatusGetOpts> statusOptions = new HashSet<>(1); statusOptions.add(StatusGetOpts.GET_COUNTERS); VertexStatus inputVertexStatus = vertexStatus(statusOptions, inputVertexName); final TezCounters inputVertexCounters = inputVertexStatus.getVertexCounters(); // eg, group name TaskCounter_Map_7_OUTPUT_Reducer_8, counter name OUTPUT_RECORDS String groupName = formattedName("TaskCounter", inputVertexName, vertexName); String counterName = "OUTPUT_RECORDS"; // Do not create counter if it does not exist - // instead fall back to default behavior for determining input records. TezCounter tezCounter = inputVertexCounters.getGroup(groupName).findCounter(counterName, false); if (tezCounter == null) { return -1; } else { return tezCounter.getValue(); } }
OperatorStats operatorStats = null; String counterName = Operator.Counter.RECORDS_OUT_OPERATOR.toString() + "_" + operatorId; TezCounter tezCounter = counters.getGroup(groupName).findCounter(counterName, false); if (tezCounter != null) { if (operatorStats == null) { operatorStats = new OperatorStats(operatorId); operatorStats.setOutputRecords(tezCounter.getValue());