private TezCounters vertexCounter(Set<StatusGetOpts> statusOptions, String vertexName) { try { return dagClient.getVertexStatus(vertexName, statusOptions).getVertexCounters(); } catch (IOException | TezException e) { // best attempt, shouldn't really kill DAG for this } return null; }
private TezCounters vertexCounter(Set<StatusGetOpts> statusOptions, String vertexName) { try { return dagClient.getVertexStatus(vertexName, statusOptions).getVertexCounters(); } catch (IOException | TezException e) { // best attempt, shouldn't really kill DAG for this } return null; }
private TezCounters vertexCounters(Set<StatusGetOpts> statusOptions, String vertexName) { try { return dagClient.getVertexStatus(vertexName, statusOptions).getVertexCounters(); } catch (IOException | TezException e) { // best attempt, shouldn't really kill DAG for this } return null; } }
private TezCounters vertexCounters(Set<StatusGetOpts> statusOptions, String vertexName) { try { return dagClient.getVertexStatus(vertexName, statusOptions).getVertexCounters(); } catch (IOException | TezException e) { // best attempt, shouldn't really kill DAG for this } return null; } }
private TezCounters vertexCounter(Set<StatusGetOpts> statusOptions, String vertexName) { try { return dagClient.getVertexStatus(vertexName, statusOptions).getVertexCounters(); } catch (IOException | TezException e) { // best attempt, shouldn't really kill DAG for this } return null; }
private long hiveInputRecordsFromTezCounters(String vertexName, String inputVertexName) { // Get the counters for the input vertex. Set<StatusGetOpts> statusOptions = new HashSet<>(1); statusOptions.add(StatusGetOpts.GET_COUNTERS); VertexStatus inputVertexStatus = vertexStatus(statusOptions, inputVertexName); final TezCounters inputVertexCounters = inputVertexStatus.getVertexCounters(); // eg, group name TaskCounter_Map_7_OUTPUT_Reducer_8, counter name OUTPUT_RECORDS String groupName = formattedName("TaskCounter", inputVertexName, vertexName); String counterName = "OUTPUT_RECORDS"; // Do not create counter if it does not exist - // instead fall back to default behavior for determining input records. TezCounter tezCounter = inputVertexCounters.getGroup(groupName).findCounter(counterName, false); if (tezCounter == null) { return -1; } else { return tezCounter.getValue(); } }
private long hiveInputRecordsFromTezCounters(String vertexName, String inputVertexName) { // Get the counters for the input vertex. Set<StatusGetOpts> statusOptions = new HashSet<>(1); statusOptions.add(StatusGetOpts.GET_COUNTERS); VertexStatus inputVertexStatus = vertexStatus(statusOptions, inputVertexName); final TezCounters inputVertexCounters = inputVertexStatus.getVertexCounters(); // eg, group name TaskCounter_Map_7_OUTPUT_Reducer_8, counter name OUTPUT_RECORDS String groupName = formattedName("TaskCounter", inputVertexName, vertexName); String counterName = "OUTPUT_RECORDS"; // Do not create counter if it does not exist - // instead fall back to default behavior for determining input records. TezCounter tezCounter = inputVertexCounters.getGroup(groupName).findCounter(counterName, false); if (tezCounter == null) { return -1; } else { return tezCounter.getValue(); } }
final TezCounters vertexCounters = vertexStatus.getVertexCounters(); final double cpuTimeMillis = getCounterValueByGroupName(vertexCounters, TaskCounter.class.getName(),
final TezCounters vertexCounters = vertexStatus.getVertexCounters(); final double cpuTimeMillis = getCounterValueByGroupName(vertexCounters, TaskCounter.class.getName(),
@Override public String toString() { StringBuilder sb = new StringBuilder(); sb.append("status=" + getState() + ", progress=" + getProgress() + ", counters=" + (getVertexCounters() == null ? "null" : getVertexCounters().toString())); return sb.toString(); }
protected TezCounters getCounters( DAGClient dagClient ) throws IOException { VertexStatus vertexStatus = updateProgress( dagClient, STATUS_GET_COUNTERS ); if( vertexStatus == null ) return null; TezCounters vertexCounters = vertexStatus.getVertexCounters(); if( vertexCounters == null ) logWarn( "could not retrieve vertex counters in stats status: {}, and vertex state: {}", getStatus(), vertexStatus.getState() ); return vertexCounters; } };
protected TezCounters getCounters( DAGClient dagClient ) throws IOException { VertexStatus vertexStatus = updateProgress( dagClient, STATUS_GET_COUNTERS ); if( vertexStatus == null ) return null; TezCounters vertexCounters = vertexStatus.getVertexCounters(); if( vertexCounters == null ) logWarn( "could not retrieve vertex counters in stats status: {}, and vertex state: {}", getStatus(), vertexStatus.getState() ); return vertexCounters; } };
@Override public boolean equals(Object obj) { if (obj instanceof VertexStatus){ VertexStatus other = (VertexStatus)obj; return getState().equals(other.getState()) && getDiagnostics().equals(other.getDiagnostics()) && getProgress().equals(other.getProgress()) && ((getVertexCounters() == null && other.getVertexCounters() == null) || getVertexCounters().equals(other.getVertexCounters())); } return false; }
this.parallelism = parallelism; //compile time parallelism this.numTasks = status.getProgress().getTotalTaskCount(); //run time parallelism TezCounters tezCounters = status.getVertexCounters(); counters = Maps.newHashMap(); Iterator<CounterGroup> grpIt = tezCounters.iterator();
@Override public int hashCode() { final int prime = 46021; int result = 1; result = prime + getState().hashCode(); List<String> diagnostics = getDiagnostics(); Progress vProgress = getProgress(); TezCounters counters = getVertexCounters(); result = prime * result + ((diagnostics == null)? 0 : diagnostics.hashCode()); result = prime * result + ((vProgress == null)? 0 : vProgress.hashCode()); result = prime * result + ((counters == null)? 0 : counters.hashCode()); return result; }
TezCounters counters = vStatus.getVertexCounters(); if (counters != null) { System.out.println("Vertex Counters for " + vertexName + ": "
final TezCounters vertexCounters = vertexStatus.getVertexCounters(); final double cpuTimeMillis = getCounterValueByGroupName(vertexCounters, TaskCounter.class.getName(),
TezCounters counters = vStatus.getVertexCounters(); if (counters != null) { log("Vertex Counters for " + vertex + ":\n" + counters);