@Override public void add(Collection<? extends MetricValues> metricValues) { List<CubeFact> facts = Lists.newArrayListWithCapacity(metricValues.size()); for (MetricValues metricValue : metricValues) { String scope = metricValue.getTags().get(Constants.Metrics.Tag.SCOPE); List<Measurement> metrics = Lists.newArrayList(); // todo improve this logic? for (MetricValue metric : metricValue.getMetrics()) { String measureName = (scope == null ? "system." : scope + ".") + metric.getName(); MeasureType type = metric.getType() == MetricType.COUNTER ? MeasureType.COUNTER : MeasureType.GAUGE; metrics.add(new Measurement(measureName, type, metric.getValue())); } CubeFact fact = new CubeFact(metricValue.getTimestamp()) .addDimensionValues(metricValue.getTags()) .addMeasurements(metrics); facts.add(fact); } cube.get().add(facts); }
@Override protected void publish(Iterator<MetricValues> metrics) throws Exception { int size = topicPayloads.size(); while (metrics.hasNext()) { encoderOutputStream.reset(); MetricValues metricValues = metrics.next(); // Encode MetricValues into bytes recordWriter.encode(metricValues, encoder); TopicPayload topicPayload = topicPayloads.get(Math.abs(metricValues.getTags().hashCode() % size)); // Calculate the topic number with the hashcode of MetricValues' tags and store the encoded payload in the // corresponding list of the topic number topicPayload.addPayload(encoderOutputStream.toByteArray(), metricValues.getTags(), metricValues.getMetrics().size()); } publishMetric(topicPayloads.values()); }
/** * Persist metrics into metric store * * @param metricValues a non-empty deque of {@link MetricValues} */ private void persistMetrics(Deque<MetricValues> metricValues, Map<TopicIdMetaKey, TopicProcessMeta> topicProcessMetaMap) { long now = System.currentTimeMillis(); long lastMetricTime = metricValues.peekLast().getTimestamp(); List<MetricValue> topicLevelDelays = new ArrayList<>(); //add topic level delay metrics for (Map.Entry<TopicIdMetaKey, TopicProcessMeta> entry : topicProcessMetaMap.entrySet()) { TopicProcessMeta topicProcessMeta = entry.getValue(); long delay = now - TimeUnit.SECONDS.toMillis(topicProcessMeta.getOldestMetricsTimestamp()); topicLevelDelays.add(new MetricValue(topicProcessMeta.getOldestMetricsTimestampMetricName(), MetricType.GAUGE, delay)); delay = now - TimeUnit.SECONDS.toMillis(topicProcessMeta.getLatestMetricsTimestamp()); topicLevelDelays.add(new MetricValue(topicProcessMeta.getLatestMetricsTimestampMetricName(), MetricType.GAUGE, delay)); } List<MetricValue> processorMetrics = new ArrayList<>(topicLevelDelays); processorMetrics.add(new MetricValue(processMetricName, MetricType.COUNTER, metricValues.size())); metricValues.add(new MetricValues(metricsContextMap, TimeUnit.MILLISECONDS.toSeconds(now), processorMetrics)); metricStore.add(metricValues); metricsProcessedCount += metricValues.size(); PROGRESS_LOG.debug("{} metrics persisted. Last metric's timestamp: {}", metricsProcessedCount, lastMetricTime); }
@Override protected MetricValues computeNext() { while (iterator.hasNext()) { Map.Entry<Map<String, String>, LoadingCache<String, AggregatedMetricsEmitter>> entry = iterator.next(); Map<String, AggregatedMetricsEmitter> metricEmitters = entry.getValue().asMap(); // +1 because we add extra metric about how many metric values did we emit in this context (see below) List<MetricValue> metricValues = Lists.newArrayListWithCapacity(metricEmitters.size() + 1); for (Map.Entry<String, AggregatedMetricsEmitter> emitterEntry : metricEmitters.entrySet()) { MetricValue metricValue = emitterEntry.getValue().emit(); // skip increment by 0 if (metricValue.getType() == MetricType.COUNTER && metricValue.getValue() == 0) { continue; } metricValues.add(metricValue); } if (metricValues.isEmpty()) { // skip if there are no metric values to send continue; } // number of emitted metrics metricValues.add(new MetricValue("metrics.emitted.count", MetricType.COUNTER, metricValues.size() + 1)); LOG.trace("Emit metric {}", metricValues); return new MetricValues(entry.getKey(), timestamp, metricValues); } return endOfData(); } };
break; lastMetricTimeSecs = metricValues.getTimestamp(); currentMessageId = input.getId(); if (LOG.isTraceEnabled()) {
/** * Persist metrics into metric store * * @param metricValues a non-empty deque of {@link MetricValues} */ private void persistMetrics(Deque<MetricValues> metricValues, Map<TopicIdMetaKey, TopicProcessMeta> topicProcessMetaMap) { long now = System.currentTimeMillis(); long lastMetricTime = metricValues.peekLast().getTimestamp(); List<MetricValue> topicLevelDelays = new ArrayList<>(); //add topic level delay metrics for (Map.Entry<TopicIdMetaKey, TopicProcessMeta> entry : topicProcessMetaMap.entrySet()) { TopicProcessMeta topicProcessMeta = entry.getValue(); long delay = now - TimeUnit.SECONDS.toMillis(topicProcessMeta.getOldestMetricsTimestamp()); topicLevelDelays.add(new MetricValue(topicProcessMeta.getOldestMetricsTimestampMetricName(), MetricType.GAUGE, delay)); delay = now - TimeUnit.SECONDS.toMillis(topicProcessMeta.getLatestMetricsTimestamp()); topicLevelDelays.add(new MetricValue(topicProcessMeta.getLatestMetricsTimestampMetricName(), MetricType.GAUGE, delay)); } List<MetricValue> processorMetrics = new ArrayList<>(topicLevelDelays); processorMetrics.add(new MetricValue(processMetricName, MetricType.COUNTER, metricValues.size())); metricValues.add(new MetricValues(metricsContextMap, TimeUnit.MILLISECONDS.toSeconds(now), processorMetrics)); metricStore.add(metricValues); metricsProcessedCount += metricValues.size(); PROGRESS_LOG.debug("{} metrics persisted. Last metric's timestamp: {}", metricsProcessedCount, lastMetricTime); }
@Override protected MetricValues computeNext() { while (iterator.hasNext()) { Map.Entry<Map<String, String>, LoadingCache<String, AggregatedMetricsEmitter>> entry = iterator.next(); Map<String, AggregatedMetricsEmitter> metricEmitters = entry.getValue().asMap(); // +1 because we add extra metric about how many metric values did we emit in this context (see below) List<MetricValue> metricValues = Lists.newArrayListWithCapacity(metricEmitters.size() + 1); for (Map.Entry<String, AggregatedMetricsEmitter> emitterEntry : metricEmitters.entrySet()) { MetricValue metricValue = emitterEntry.getValue().emit(); // skip increment by 0 if (metricValue.getType() == MetricType.COUNTER && metricValue.getValue() == 0) { continue; } metricValues.add(metricValue); } if (metricValues.isEmpty()) { // skip if there are no metric values to send continue; } // number of emitted metrics metricValues.add(new MetricValue("metrics.emitted.count", MetricType.COUNTER, metricValues.size() + 1)); LOG.trace("Emit metric {}", metricValues); return new MetricValues(entry.getKey(), timestamp, metricValues); } return endOfData(); } };
@Override protected void publish(Iterator<MetricValues> metrics) throws Exception { int size = topicPayloads.size(); while (metrics.hasNext()) { encoderOutputStream.reset(); MetricValues metricValues = metrics.next(); // Encode MetricValues into bytes recordWriter.encode(metricValues, encoder); TopicPayload topicPayload = topicPayloads.get(Math.abs(metricValues.getTags().hashCode() % size)); // Calculate the topic number with the hashcode of MetricValues' tags and store the encoded payload in the // corresponding list of the topic number topicPayload.addPayload(encoderOutputStream.toByteArray(), metricValues.getTags(), metricValues.getMetrics().size()); } publishMetric(topicPayloads.values()); }
break; lastMetricTimeSecs = metricValues.getTimestamp(); currentMessageId = input.getId(); if (LOG.isTraceEnabled()) {
@Override public void add(Collection<? extends MetricValues> metricValues) { List<CubeFact> facts = Lists.newArrayListWithCapacity(metricValues.size()); for (MetricValues metricValue : metricValues) { String scope = metricValue.getTags().get(Constants.Metrics.Tag.SCOPE); List<Measurement> metrics = Lists.newArrayList(); // todo improve this logic? for (MetricValue metric : metricValue.getMetrics()) { String measureName = (scope == null ? "system." : scope + ".") + metric.getName(); MeasureType type = metric.getType() == MetricType.COUNTER ? MeasureType.COUNTER : MeasureType.GAUGE; metrics.add(new Measurement(measureName, type, metric.getValue())); } CubeFact fact = new CubeFact(metricValue.getTimestamp()) .addDimensionValues(metricValue.getTags()) .addMeasurements(metrics); facts.add(fact); } cube.get().add(facts); }
private void gauge(Map<String, String> context, String metric, long timestamp, Long value) throws Exception { metricStore.add(new MetricValues(context, metric, timestamp, value, MetricType.GAUGE)); }