private TimeSeriesComponentMetric buildTimeSeriesComponentMetric(String name, Map<String, Map<Long, Double>> stats) { Map<String, Map<Long, Double>> misc = new HashMap<>(); misc.put(StormMappedMetric.ackedRecords.name(), stats.get(StormMappedMetric.ackedRecords.name())); if (stats.containsKey(StormMappedMetric.completeLatency.name())) { misc.put(StormMappedMetric.completeLatency.name(), stats.get(StormMappedMetric.completeLatency.name())); } if (stats.containsKey(StormMappedMetric.executeTime.name())) { misc.put(StormMappedMetric.executeTime.name(), stats.get(StormMappedMetric.executeTime.name())); } TimeSeriesComponentMetric metric = new TimeSeriesComponentMetric(name, stats.getOrDefault(StormMappedMetric.inputRecords.name(), Collections.emptyMap()), stats.getOrDefault(StormMappedMetric.outputRecords.name(), Collections.emptyMap()), stats.getOrDefault(StormMappedMetric.failedRecords.name(), Collections.emptyMap()), stats.getOrDefault(StormMappedMetric.processedTime.name(), Collections.emptyMap()), stats.getOrDefault(StormMappedMetric.recordsInWaitQueue.name(), Collections.emptyMap()), misc); return metric; }
@VisibleForTesting static double aggregateCompleteLatency(TopologyTimeSeriesMetrics.TimeSeriesComponentMetric metrics) { Map<Long, Double> keyMetrics = metrics.getMisc().getOrDefault(StormMappedMetric.completeLatency.name(), Collections.emptyMap()); Map<Long, Double> weightMetrics = metrics.getMisc().getOrDefault(METRIC_NAME_ACK_COUNT, Collections.emptyMap()); return calculateWeightedAverage(keyMetrics, weightMetrics); }
@VisibleForTesting static double aggregateExecuteLatency(TopologyTimeSeriesMetrics.TimeSeriesComponentMetric metrics) { Map<Long, Double> keyMetrics = metrics.getMisc().getOrDefault(StormMappedMetric.executeTime.name(), Collections.emptyMap()); return calculateWeightedAverage(keyMetrics, metrics.getInputRecords()); }
@Override public TimeSeriesComponentMetric getTopologyStats(TopologyLayout topology, long from, long to, String asUser) { String stormTopologyName = StormTopologyUtil.findOrGenerateTopologyName(client, topology.getId(), topology.getName(), asUser); Map<String, Map<Long, Double>> stats = ParallelStreamUtil.execute(() -> Arrays.asList(STATS_METRICS_SOURCE) .parallelStream() .collect(toMap(m -> m.name(), m -> queryTopologyMetrics(stormTopologyName, m, from, to))), FORK_JOIN_POOL); return buildTimeSeriesComponentMetric(topology.getName(), stats); }
@Override public TimeSeriesComponentMetric getComponentStats(TopologyLayout topology, Component component, long from, long to, String asUser) { String stormTopologyName = StormTopologyUtil.findOrGenerateTopologyName(client, topology.getId(), topology.getName(), asUser); String stormComponentName = getComponentName(component); StormMappedMetric[] stats; if (component instanceof Source) { stats = STATS_METRICS_SOURCE; } else { stats = STATS_METRICS; } // empty map if time-series DB is not set to the namespace Map<String, Map<Long, Double>> componentStats = ParallelStreamUtil.execute(() -> Arrays.asList(stats) .parallelStream() .collect(toMap(m -> m.name(), m -> queryComponentMetrics(stormTopologyName, stormComponentName, m, from, to))), FORK_JOIN_POOL); return buildTimeSeriesComponentMetric(component.getName(), componentStats); }
@Override public Map<String, Map<Long, Double>> getkafkaTopicOffsets(TopologyLayout topology, Component component, long from, long to, String asUser) { String stormTopologyName = StormTopologyUtil.findOrGenerateTopologyName(client, topology.getId(), topology.getName(), asUser); String stormComponentName = getComponentName(component); String topicName = findKafkaTopicName(component); if (topicName == null) { throw new IllegalStateException("Cannot find Kafka topic name from source config - topology name: " + topology.getName() + " / source : " + component.getName()); } StormMappedMetric[] metrics = { StormMappedMetric.logsize, StormMappedMetric.offset, StormMappedMetric.lag }; Map<String, Map<Long, Double>> kafkaOffsets = new HashMap<>(); for (StormMappedMetric metric : metrics) { kafkaOffsets.put(metric.name(), queryKafkaMetrics(stormTopologyName, stormComponentName, metric, topicName, from, to)); } return kafkaOffsets; }