String vertexName = null; if (inputInitializerContext != null) { tezCounters = new TezCounters(); groupName = HiveInputCounters.class.getName(); vertexName = jobConf.get(Operator.CONTEXT_NAME_KEY, "");
public TezCounters addAndGetTezCounter(String name) { TezCounters counter = new TezCounters(); counterMap.put(name, counter); return counter; }
public TezCounters getCounters() { TezCounters fullCounters = new TezCounters(); fullCounters.incrAllCounters(tezCounters); for (TezCounters counter : counterMap.values()) { fullCounters.incrAllCounters(counter); } return fullCounters; }
public void setLocalityCounter(DAGCounter localityCounter) { if (!localitySet.get()) { localitySet.set(true); if (counters == null) { counters = new TezCounters(); } if (localityCounter != null) { counters.findCounter(localityCounter).increment(1); // TODO Maybe validate that the correct value is being set. } } } }
@Override public void readFields(DataInput in) throws IOException { progress = in.readFloat(); progressNotified = in.readBoolean(); if (in.readBoolean()) { tezCounters = new TezCounters(); tezCounters.readFields(in); } if (in.readBoolean()) { statistics = new TaskStatistics(); statistics.readFields(in); } }
protected RuntimeTask(TaskSpec taskSpec, Configuration tezConf, TezUmbilical tezUmbilical, String pid, boolean setupSysCounterUpdater) { this.taskSpec = taskSpec; this.tezConf = tezConf; this.tezUmbilical = tezUmbilical; this.tezCounters = new TezCounters(); this.eventCounter = new AtomicInteger(0); this.nextFromEventId = new AtomicInteger(0); this.nextPreRoutedEventId = new AtomicInteger(0); this.progress = 0.0f; this.taskDone = new AtomicBoolean(false); this.statistics = new TaskStatistics(); if (setupSysCounterUpdater) { this.counterUpdater = new TaskCounterUpdater(tezCounters, tezConf, pid); } else { this.counterUpdater = null; } }
@Override public TezCounters getCachedCounters() { readLock.lock(); try { // FIXME a better lightweight approach for counters is needed if (fullCounters == null && cachedCounters != null && ((cachedCountersTimestamp+10000) > System.currentTimeMillis())) { LOG.info("Asked for counters" + ", cachedCountersTimestamp=" + cachedCountersTimestamp + ", currentTime=" + System.currentTimeMillis()); return cachedCounters; } cachedCountersTimestamp = System.currentTimeMillis(); if (inTerminalState()) { this.mayBeConstructFinalFullCounters(); return fullCounters; } TezCounters counters = new TezCounters(); counters.incrAllCounters(this.counters); cachedCounters = incrTaskCounters(counters, tasks.values()); return cachedCounters; } finally { readLock.unlock(); } }
@Override public TezCounters getAllCounters() { readLock.lock(); try { if (inTerminalState()) { this.mayBeConstructFinalFullCounters(); return fullCounters; } TezCounters counters = new TezCounters(); counters.incrAllCounters(this.counters); return incrTaskCounters(counters, tasks.values()); } finally { readLock.unlock(); } }
@Private public void constructFinalFullcounters() { this.fullCounters = new TezCounters(); this.fullCounters.incrAllCounters(dagCounters); for (Vertex v : this.vertices.values()) { this.fullCounters.incrAllCounters(v.getAllCounters()); } }
@Override public TezCounters getCounters() { TezCounters counters = new TezCounters(); counters.incrAllCounters(this.counters); readLock.lock(); try { TaskAttempt bestAttempt = selectBestAttempt(); if (bestAttempt != null) { counters.incrAllCounters(bestAttempt.getCounters()); } return counters; } finally { readLock.unlock(); } }
@Override public TezCounters getCachedCounters() { readLock.lock(); try { // FIXME a better lightweight approach for counters is needed if (fullCounters == null && cachedCounters != null && ((cachedCountersTimestamp+10000) > System.currentTimeMillis())) { LOG.info("Asked for counters" + ", cachedCountersTimestamp=" + cachedCountersTimestamp + ", currentTime=" + System.currentTimeMillis()); return cachedCounters; } cachedCountersTimestamp = System.currentTimeMillis(); if (inTerminalState()) { this.mayBeConstructFinalFullCounters(); return fullCounters; } // dag not yet finished. update cpu time counters updateCpuCounters(); TezCounters counters = new TezCounters(); counters.incrAllCounters(dagCounters); return incrTaskCounters(counters, vertices.values()); } finally { readLock.unlock(); } }
@Override public TezCounters getAllCounters() { readLock.lock(); try { DAGState state = getInternalState(); if (state == DAGState.ERROR || state == DAGState.FAILED || state == DAGState.KILLED || state == DAGState.SUCCEEDED) { this.mayBeConstructFinalFullCounters(); return fullCounters; } // dag not yet finished. update cpu time counters updateCpuCounters(); TezCounters counters = new TezCounters(); counters.incrAllCounters(dagCounters); return incrTaskCounters(counters, vertices.values()); } finally { readLock.unlock(); } }
@Private public void constructFinalFullcounters() { this.fullCounters = new TezCounters(); this.fullCounters.incrAllCounters(counters); this.vertexStats = new VertexStats(); for (Task t : this.tasks.values()) { vertexStats.updateStats(t.getReport()); TezCounters counters = t.getCounters(); this.fullCounters.incrAllCounters(counters); } }
public static TezCounters convertTezCountersFromProto(TezCountersProto proto) { TezCounters counters = new TezCounters(); for (TezCounterGroupProto counterGroupProto : proto.getCounterGroupsList()) { CounterGroup group = counters.addGroup(counterGroupProto.getName(), counterGroupProto.getDisplayName()); for (TezCounterProto counterProto : counterGroupProto.getCountersList()) { TezCounter counter = group.findCounter( counterProto.getName(), counterProto.getDisplayName()); counter.setValue(counterProto.getValue()); } } return counters; }
TezCounters counters = new TezCounters();