public java.lang.Object getFieldValue(_Fields field) { switch (field) { case EMITTED: return get_emitted(); case TRANSFERRED: return get_transferred(); case SPECIFIC: return get_specific(); case RATE: return get_rate(); } throw new java.lang.IllegalStateException(); }
/** * Convert Executor stats to thrift data structure. * @param stats the stats in the form of a map. * @return teh thrift structure for the stats. */ public static ExecutorStats thriftifyExecutorStats(Map stats) { ExecutorStats ret = new ExecutorStats(); ExecutorSpecificStats specificStats = thriftifySpecificStats(stats); ret.set_specific(specificStats); ret.set_emitted(ClientStatsUtil.windowSetConverter(ClientStatsUtil.getMapByKey(stats, EMITTED), TO_STRING, TO_STRING)); ret.set_transferred(ClientStatsUtil.windowSetConverter(ClientStatsUtil.getMapByKey(stats, TRANSFERRED), TO_STRING, TO_STRING)); ret.set_rate(((Number) stats.get(RATE)).doubleValue()); return ret; }
case EMITTED: if (value == null) { unset_emitted(); } else { set_emitted((java.util.Map<java.lang.String,java.util.Map<java.lang.String,java.lang.Long>>)value); unset_transferred(); } else { set_transferred((java.util.Map<java.lang.String,java.util.Map<java.lang.String,java.lang.Long>>)value); unset_specific(); } else { set_specific((ExecutorSpecificStats)value); unset_rate(); } else { set_rate((java.lang.Double)value);
private static List<Double> extractBoltValues(List<ExecutorSummary> summaries, GlobalStreamId id, Function<BoltStats, Map<String, Map<GlobalStreamId, Double>>> func) { List<Double> ret = new ArrayList<>(); if (summaries != null) { for (ExecutorSummary summ : summaries) { if (summ != null && summ.is_set_stats()) { Map<String, Map<GlobalStreamId, Double>> data = func.apply(summ.get_stats().get_specific().get_bolt()); if (data != null) { List<Double> subvalues = data.values().stream() .map((subMap) -> subMap.get(id)) .filter((value) -> value != null) .collect(Collectors.toList()); ret.addAll(subvalues); } } } } return ret; }
ExecutorSpecificStats executorSpecificStats = executorStats.get_specific(); if (executorSpecificStats == null) { Map<String, Map<String, Long>> transferred = executorStats.get_transferred(); Map<String, Long> txMap = transferred.get(":all-time"); if (txMap == null) {
/** * aggregate common stats from a spout/bolt, called in aggregateSpoutStats/aggregateBoltStats. */ public static <T> Map<String, Map<String, Map<T, Long>>> aggregateCommonStats(List<ExecutorSummary> statsSeq) { Map<String, Map<String, Map<T, Long>>> ret = new HashMap<>(); List<Map<String, Map<String, Long>>> emitted = new ArrayList<>(); List<Map<String, Map<String, Long>>> transferred = new ArrayList<>(); for (ExecutorSummary summ : statsSeq) { emitted.add(summ.get_stats().get_emitted()); transferred.add(summ.get_stats().get_transferred()); } ((Map) ret).put(EMITTED, aggregateCounts(emitted)); ((Map) ret).put(TRANSFERRED, aggregateCounts(transferred)); return ret; }
@Override public void read(org.apache.storm.thrift.protocol.TProtocol prot, ClusterWorkerHeartbeat struct) throws org.apache.storm.thrift.TException { org.apache.storm.thrift.protocol.TTupleProtocol iprot = (org.apache.storm.thrift.protocol.TTupleProtocol) prot; struct.storm_id = iprot.readString(); struct.set_storm_id_isSet(true); { org.apache.storm.thrift.protocol.TMap _map778 = new org.apache.storm.thrift.protocol.TMap(org.apache.storm.thrift.protocol.TType.STRUCT, org.apache.storm.thrift.protocol.TType.STRUCT, iprot.readI32()); struct.executor_stats = new java.util.HashMap<ExecutorInfo,ExecutorStats>(2*_map778.size); ExecutorInfo _key779; ExecutorStats _val780; for (int _i781 = 0; _i781 < _map778.size; ++_i781) { _key779 = new ExecutorInfo(); _key779.read(iprot); _val780 = new ExecutorStats(); _val780.read(iprot); struct.executor_stats.put(_key779, _val780); } } struct.set_executor_stats_isSet(true); struct.time_secs = iprot.readI32(); struct.set_time_secs_isSet(true); struct.uptime_secs = iprot.readI32(); struct.set_uptime_secs_isSet(true); } }
/** * getStatMapFromExecutorSummary. * @param executorSummary executorSummary * @return getStatMapFromExecutorSummary */ public static Map<String, Object> getStatMapFromExecutorSummary(ExecutorSummary executorSummary) { Map<String, Object> result = new HashMap(); result.put(":host", executorSummary.get_host()); result.put(":port", executorSummary.get_port()); result.put(":uptime_secs", executorSummary.get_uptime_secs()); result.put(":transferred", null); if (executorSummary.is_set_stats()) { result.put(":transferred", sanitizeTransferredStats(executorSummary.get_stats().get_transferred())); } return result; }
public ExecutorStats deepCopy() { return new ExecutorStats(this); }
LOG.debug("Getting uptime for worker {}, {}", key, uptime); for (Map.Entry<String, Map<String, Long>> statEntry : summary.get_stats().get_emitted().entrySet()) { String timeWindow = statEntry.getKey(); long timeSecs = uptime;
case EMITTED: if (value == null) { unset_emitted(); } else { set_emitted((Map<String,Map<String,Long>>)value); unset_transferred(); } else { set_transferred((Map<String,Map<String,Long>>)value); unset_specific(); } else { set_specific((ExecutorSpecificStats)value); unset_rate(); } else { set_rate((Double)value);
/** * aggregate spout stats. * * @param statsSeq a seq of ExecutorStats * @param includeSys whether to include system streams * @return aggregated spout stats: {metric -> win -> global stream id -> value} */ public static Map<String, Map> aggregateSpoutStats(List<ExecutorSummary> statsSeq, boolean includeSys) { // actually Map<String, Map<String, Map<String, Long/Double>>> Map<String, Map> ret = new HashMap<>(); Map<String, Map<String, Map<String, Long>>> commonStats = aggregateCommonStats(statsSeq); // filter sys streams if necessary commonStats = preProcessStreamSummary(commonStats, includeSys); List<Map<String, Map<String, Long>>> acked = new ArrayList<>(); List<Map<String, Map<String, Long>>> failed = new ArrayList<>(); List<Map<String, Map<String, Double>>> completeLatencies = new ArrayList<>(); for (ExecutorSummary summary : statsSeq) { ExecutorStats stats = summary.get_stats(); acked.add(stats.get_specific().get_spout().get_acked()); failed.add(stats.get_specific().get_spout().get_failed()); completeLatencies.add(stats.get_specific().get_spout().get_complete_ms_avg()); } ret.putAll(commonStats); ((Map) ret).put(ACKED, aggregateCounts(acked)); ((Map) ret).put(FAILED, aggregateCounts(failed)); ((Map) ret).put(COMP_LATENCIES, aggregateAverages(completeLatencies, acked)); return ret; }
totalExecutors++; if (stats != null) { if (stats.get_specific().is_set_spout()) { SpoutStats ss = stats.get_specific().get_spout(); Map<String, Long> failedMap = ss.get_failed().get(":all-time"); if (failedMap != null) { Map<String,Map<String,Long>> transferred = stats.get_transferred(); if ( transferred != null) { Map<String, Long> e2 = transferred.get(":all-time");
if (stats != null) { Map<String, Map<String, Long>> statted = WATCH_EMITTED.equals(_watch) ? stats.get_emitted() : stats.get_transferred(); if (statted != null) { Map<String, Long> e2 = statted.get(":all-time");
@Override public void read(org.apache.storm.thrift.protocol.TProtocol prot, ExecutorSummary struct) throws org.apache.storm.thrift.TException { org.apache.storm.thrift.protocol.TTupleProtocol iprot = (org.apache.storm.thrift.protocol.TTupleProtocol) prot; struct.executor_info = new ExecutorInfo(); struct.executor_info.read(iprot); struct.set_executor_info_isSet(true); struct.component_id = iprot.readString(); struct.set_component_id_isSet(true); struct.host = iprot.readString(); struct.set_host_isSet(true); struct.port = iprot.readI32(); struct.set_port_isSet(true); struct.uptime_secs = iprot.readI32(); struct.set_uptime_secs_isSet(true); java.util.BitSet incoming = iprot.readBitSet(1); if (incoming.get(0)) { struct.stats = new ExecutorStats(); struct.stats.read(iprot); struct.set_stats_isSet(true); } } }
/** * Performs a deep copy on <i>other</i>. */ public ClusterWorkerHeartbeat(ClusterWorkerHeartbeat other) { __isset_bitfield = other.__isset_bitfield; if (other.is_set_storm_id()) { this.storm_id = other.storm_id; } if (other.is_set_executor_stats()) { java.util.Map<ExecutorInfo,ExecutorStats> __this__executor_stats = new java.util.HashMap<ExecutorInfo,ExecutorStats>(other.executor_stats.size()); for (java.util.Map.Entry<ExecutorInfo, ExecutorStats> other_element : other.executor_stats.entrySet()) { ExecutorInfo other_element_key = other_element.getKey(); ExecutorStats other_element_value = other_element.getValue(); ExecutorInfo __this__executor_stats_copy_key = new ExecutorInfo(other_element_key); ExecutorStats __this__executor_stats_copy_value = new ExecutorStats(other_element_value); __this__executor_stats.put(__this__executor_stats_copy_key, __this__executor_stats_copy_value); } this.executor_stats = __this__executor_stats; } this.time_secs = other.time_secs; this.uptime_secs = other.uptime_secs; }
/** * convert thrift ExecutorStats structure into a java HashMap. */ public static Map<String, Object> convertExecutorStats(ExecutorStats stats) { Map<String, Object> ret = new HashMap<>(); ret.put(EMITTED, stats.get_emitted()); ret.put(TRANSFERRED, stats.get_transferred()); ret.put(RATE, stats.get_rate()); if (stats.get_specific().is_set_bolt()) { ret.putAll(convertSpecificStats(stats.get_specific().get_bolt())); ret.put(TYPE, ClientStatsUtil.BOLT); } else { ret.putAll(convertSpecificStats(stats.get_specific().get_spout())); ret.put(TYPE, ClientStatsUtil.SPOUT); } return ret; }
@Override public ExecutorStats renderStats() { ExecutorStats ret = new ExecutorStats(); // common fields ret.set_emitted(valueStat(getEmitted())); ret.set_transferred(valueStat(getTransferred())); ret.set_rate(this.rate); // spout stats SpoutStats spoutStats = new SpoutStats( valueStat(getAcked()), valueStat(getFailed()), valueStat(completeLatencyStats)); ret.set_specific(ExecutorSpecificStats.spout(spoutStats)); return ret; } }
for (ExecutorSummary summary : statsSeq) { ExecutorStats stat = summary.get_stats(); acked.add(stat.get_specific().get_bolt().get_acked()); failed.add(stat.get_specific().get_bolt().get_failed()); executed.add(stat.get_specific().get_bolt().get_executed()); processLatencies.add(stat.get_specific().get_bolt().get_process_ms_avg()); executeLatencies.add(stat.get_specific().get_bolt().get_execute_ms_avg());
if (stats != null) { Map<String,Map<String,Long>> statted = WATCH_EMITTED.equals(_watch) ? stats.get_emitted() : stats.get_transferred(); if ( statted != null) { Map<String, Long> e2 = statted.get(":all-time");