/** * query the metric store and return the Collection<MetricTimeSeries> * @param query * @return Collection<MetricTimeSeries> */ public Collection<MetricTimeSeries> query(MetricDataQuery query) { return metricStore.query(query); }
/** * Search the metric store and return the collection of metric names available for the tag-values in search query * @param query * @return Collection of metric names */ public Collection<String> searchMetricNames(MetricSearchQuery query) { return metricStore.findMetricNames(query); }
@Override protected void publish(Iterator<MetricValues> metrics) { while (metrics.hasNext()) { metricStore.add(metrics.next()); } }
private Map<String, Long> getSparkDetails(ProgramId sparkProgram, String runId) { Map<String, String> context = new HashMap<>(); context.put(Constants.Metrics.Tag.NAMESPACE, sparkProgram.getNamespace()); context.put(Constants.Metrics.Tag.APP, sparkProgram.getApplication()); context.put(Constants.Metrics.Tag.SPARK, sparkProgram.getProgram()); context.put(Constants.Metrics.Tag.RUN_ID, runId); List<TagValue> tags = new ArrayList<>(); for (Map.Entry<String, String> entry : context.entrySet()) { tags.add(new TagValue(entry.getKey(), entry.getValue())); } MetricSearchQuery metricSearchQuery = new MetricSearchQuery(0, 0, Integer.MAX_VALUE, tags); Collection<String> metricNames = metricStore.findMetricNames(metricSearchQuery); Map<String, Long> overallResult = new HashMap<>(); for (String metricName : metricNames) { Collection<MetricTimeSeries> resultPerQuery = metricStore.query( new MetricDataQuery(0, 0, Integer.MAX_VALUE, metricName, AggregationFunction.SUM, context, new ArrayList<String>())); for (MetricTimeSeries metricTimeSeries : resultPerQuery) { overallResult.put(metricTimeSeries.getMetricName(), metricTimeSeries.getTimeValues().get(0).getValue()); } } return overallResult; }
/** * Search the metric store and return the collection of next available tags for the search query * @param query * @return Collection of tag values */ public Collection<TagValue> searchTags(MetricSearchQuery query) { return metricStore.findNextAvailableTags(query); }
/** * deletes all metrics */ public void resetAll() { metricStore.deleteAll(); }
private void deleteMetrics(NamespaceId namespaceId) { long endTs = System.currentTimeMillis() / 1000; Map<String, String> tags = new LinkedHashMap<>(); tags.put(Constants.Metrics.Tag.NAMESPACE, namespaceId.getNamespace()); MetricDeleteQuery deleteQuery = new MetricDeleteQuery(0, endTs, Collections.emptySet(), tags, new ArrayList<>(tags.keySet())); metricStore.delete(deleteQuery); } }
private Map<String, Long> getSparkDetails(ProgramId sparkProgram, String runId) { Map<String, String> context = new HashMap<>(); context.put(Constants.Metrics.Tag.NAMESPACE, sparkProgram.getNamespace()); context.put(Constants.Metrics.Tag.APP, sparkProgram.getApplication()); context.put(Constants.Metrics.Tag.SPARK, sparkProgram.getProgram()); context.put(Constants.Metrics.Tag.RUN_ID, runId); List<TagValue> tags = new ArrayList<>(); for (Map.Entry<String, String> entry : context.entrySet()) { tags.add(new TagValue(entry.getKey(), entry.getValue())); } MetricSearchQuery metricSearchQuery = new MetricSearchQuery(0, 0, Integer.MAX_VALUE, tags); Collection<String> metricNames = metricStore.findMetricNames(metricSearchQuery); Map<String, Long> overallResult = new HashMap<>(); for (String metricName : metricNames) { Collection<MetricTimeSeries> resultPerQuery = metricStore.query( new MetricDataQuery(0, 0, Integer.MAX_VALUE, metricName, AggregationFunction.SUM, context, new ArrayList<String>())); for (MetricTimeSeries metricTimeSeries : resultPerQuery) { overallResult.put(metricTimeSeries.getMetricName(), metricTimeSeries.getTimeValues().get(0).getValue()); } } return overallResult; }
/** * Search the metric store and return the collection of next available tags for the search query * @param query * @return Collection of tag values */ public Collection<TagValue> searchTags(MetricSearchQuery query) { return metricStore.findNextAvailableTags(query); }
/** * deletes all metrics */ public void resetAll() { metricStore.deleteAll(); }
private void deleteMetrics(NamespaceId namespaceId) { long endTs = System.currentTimeMillis() / 1000; Map<String, String> tags = new LinkedHashMap<>(); tags.put(Constants.Metrics.Tag.NAMESPACE, namespaceId.getNamespace()); MetricDeleteQuery deleteQuery = new MetricDeleteQuery(0, endTs, Collections.emptySet(), tags, new ArrayList<>(tags.keySet())); metricStore.delete(deleteQuery); } }
/** * query the metric store and return the Collection<MetricTimeSeries> * @param query * @return Collection<MetricTimeSeries> */ public Collection<MetricTimeSeries> query(MetricDataQuery query) { return metricStore.query(query); }
/** * Search the metric store and return the collection of metric names available for the tag-values in search query * @param query * @return Collection of metric names */ public Collection<String> searchMetricNames(MetricSearchQuery query) { return metricStore.findMetricNames(query); }
@Override protected void publish(Iterator<MetricValues> metrics) { List<MetricValues> metricValues = new ArrayList<>(); while (metrics.hasNext()) { metricValues.add(metrics.next()); } metricStore.add(metricValues); }
public List<MetricTagValue> searchTags(List<String> tags) { // we want to search the entire range, so startTimestamp is '0' and end Timestamp is Integer.MAX_VALUE and // limit is -1 , to include the entire search result. MetricSearchQuery searchQuery = new MetricSearchQuery(0, Integer.MAX_VALUE, -1, toTagValues(humanToTagNames(parseTagValues(tags)))); return tagValuesToHuman(metricStore.findNextAvailableTags(searchQuery)); }
Assert.assertEquals(0L, getMetric(metricStore, programRunId, myProfile, SYSTEM_METRIC_PREFIX + Constants.Metrics.Program.PROGRAM_FAILED_RUNS)); metricStore.deleteAll();
private void deleteMetrics(ApplicationId applicationId, Iterable<String> flows) { LOG.debug("Deleting metrics for application {}", applicationId); for (String flow : flows) { long endTs = System.currentTimeMillis() / 1000; Map<String, String> tags = new LinkedHashMap<>(); tags.put(Constants.Metrics.Tag.NAMESPACE, applicationId.getNamespace()); tags.put(Constants.Metrics.Tag.APP, applicationId.getApplication()); tags.put(Constants.Metrics.Tag.FLOW, flow); MetricDeleteQuery deleteQuery = new MetricDeleteQuery(0, endTs, Collections.emptySet(), tags, new ArrayList<>(tags.keySet())); metricStore.delete(deleteQuery); } } }
private Iterator<TimeValue> queryTimeSeries(MetricDataQuery query) throws Exception { Collection<MetricTimeSeries> result = metricStore.query(query); if (result.size() == 0) { return new ArrayList<TimeValue>().iterator(); } // since there's no group by condition, it'll return single time series always MetricTimeSeries timeSeries = result.iterator().next(); return Iterables.transform(timeSeries.getTimeValues(), new Function<TimeValue, TimeValue>() { @Override public TimeValue apply(TimeValue input) { return new TimeValue(input.getTimestamp(), input.getValue()); } }).iterator(); }
private Collection<String> getMetrics(List<MetricTagValue> tagValues) { // we want to search the entire range, so startTimestamp is '0' and end Timestamp is Integer.MAX_VALUE and // limit is -1 , to include the entire search result. MetricSearchQuery searchQuery = new MetricSearchQuery(0, Integer.MAX_VALUE, -1, toTagValues(tagValues)); Collection<String> metricNames = metricStore.findMetricNames(searchQuery); return Lists.newArrayList(Iterables.filter(metricNames, Predicates.notNull())); }
/** * Persist metrics into metric store * * @param metricValues a non-empty deque of {@link MetricValues} */ private void persistMetrics(Deque<MetricValues> metricValues, Map<TopicIdMetaKey, TopicProcessMeta> topicProcessMetaMap) { long now = System.currentTimeMillis(); long lastMetricTime = metricValues.peekLast().getTimestamp(); List<MetricValue> topicLevelDelays = new ArrayList<>(); //add topic level delay metrics for (Map.Entry<TopicIdMetaKey, TopicProcessMeta> entry : topicProcessMetaMap.entrySet()) { TopicProcessMeta topicProcessMeta = entry.getValue(); long delay = now - TimeUnit.SECONDS.toMillis(topicProcessMeta.getOldestMetricsTimestamp()); topicLevelDelays.add(new MetricValue(topicProcessMeta.getOldestMetricsTimestampMetricName(), MetricType.GAUGE, delay)); delay = now - TimeUnit.SECONDS.toMillis(topicProcessMeta.getLatestMetricsTimestamp()); topicLevelDelays.add(new MetricValue(topicProcessMeta.getLatestMetricsTimestampMetricName(), MetricType.GAUGE, delay)); } List<MetricValue> processorMetrics = new ArrayList<>(topicLevelDelays); processorMetrics.add(new MetricValue(processMetricName, MetricType.COUNTER, metricValues.size())); metricValues.add(new MetricValues(metricsContextMap, TimeUnit.MILLISECONDS.toSeconds(now), processorMetrics)); metricStore.add(metricValues); metricsProcessedCount += metricValues.size(); PROGRESS_LOG.debug("{} metrics persisted. Last metric's timestamp: {}", metricsProcessedCount, lastMetricTime); }