/** * query the metric store and return the Collection<MetricTimeSeries> * @param query * @return Collection<MetricTimeSeries> */ public Collection<MetricTimeSeries> query(MetricDataQuery query) { return metricStore.query(query); }
/** * query the metric store and return the Collection<MetricTimeSeries> * @param query * @return Collection<MetricTimeSeries> */ public Collection<MetricTimeSeries> query(MetricDataQuery query) { return metricStore.query(query); }
private Iterator<TimeValue> queryTimeSeries(MetricDataQuery query) throws Exception { Collection<MetricTimeSeries> result = metricStore.query(query); if (result.size() == 0) { return new ArrayList<TimeValue>().iterator(); } // since there's no group by condition, it'll return single time series always MetricTimeSeries timeSeries = result.iterator().next(); return Iterables.transform(timeSeries.getTimeValues(), new Function<TimeValue, TimeValue>() { @Override public TimeValue apply(TimeValue input) { return new TimeValue(input.getTimestamp(), input.getValue()); } }).iterator(); }
private Map<String, Long> getTotalsWithSingleGroupByTag(MetricDataQuery query) throws Exception { // query must have resolution set to Integer.MAX_VALUE (i.e. "totals") Collection<MetricTimeSeries> result = metricStore.query(query); Map<String, Long> map = Maps.newHashMap(); for (MetricTimeSeries timeSeries : result) { // we know there's only ony group by tag String groupByTagValue = timeSeries.getTagValues().values().iterator().next(); // since it is totals, it will have only one TimeValue map.put(groupByTagValue, timeSeries.getTimeValues().get(0).getValue()); } return map; } }
private long getSingleValueFromTotals(MetricDataQuery query) { try { Collection<MetricTimeSeries> result = metricStore.query(query); if (result.isEmpty()) { return 0; } // since it is totals query and not groupBy specified, we know there's one time series List<TimeValue> timeValues = result.iterator().next().getTimeValues(); if (timeValues.isEmpty()) { return 0; } // since it is totals, we know there's one value only return timeValues.get(0).getValue(); } catch (Exception e) { throw Throwables.propagate(e); } } }
private long getTotals(MetricDataQuery query) throws Exception { // query must have resolution set to Integer.MAX_VALUE (i.e. "totals") Collection<MetricTimeSeries> result = metricStore.query(query); if (result.size() == 0) { return 0; } // since there's no group by condition, it'll return single time series always MetricTimeSeries timeSeries = result.iterator().next(); if (timeSeries.getTimeValues().isEmpty()) { return 0; } // since it is totals, it will have only one TimeValue or none return timeSeries.getTimeValues().get(0).getValue(); }
private long getSingleValueFromTotals(MetricDataQuery query) { try { Collection<MetricTimeSeries> result = metricStore.query(query); if (result.isEmpty()) { return 0; } // since it is totals query and not groupBy specified, we know there's one time series List<TimeValue> timeValues = result.iterator().next().getTimeValues(); if (timeValues.isEmpty()) { return 0; } // since it is totals, we know there's one value only return timeValues.get(0).getValue(); } catch (Exception e) { throw Throwables.propagate(e); } } }
private void getAggregates(Map<String, String> tags, Map<String, String> metricsToCounters, Map<String, Long> result) { Map<String, AggregationFunction> metrics = Maps.newHashMap(); // all map-reduce metrics are gauges for (String metric : metricsToCounters.keySet()) { metrics.put(metric, AggregationFunction.LATEST); } MetricDataQuery metricDataQuery = new MetricDataQuery(0, Integer.MAX_VALUE, Integer.MAX_VALUE, metrics, tags, ImmutableList.<String>of()); Collection<MetricTimeSeries> query = metricStore.query(metricDataQuery); // initialize elements to zero for (String counterName : metricsToCounters.values()) { result.put(counterName, 0L); } for (MetricTimeSeries metricTimeSeries : query) { List<TimeValue> timeValues = metricTimeSeries.getTimeValues(); TimeValue timeValue = Iterables.getOnlyElement(timeValues); result.put(metricsToCounters.get(metricTimeSeries.getMetricName()), timeValue.getValue()); } }
private Collection<MetricTimeSeries> getMetricTimeSeries() { return metricStore.query(new MetricDataQuery( 0, System.currentTimeMillis() / 1000L, Integer.MAX_VALUE, "user.beforeSubmit", AggregationFunction.SUM, ImmutableMap.of(Constants.Metrics.Tag.NAMESPACE, "metrics_ns", Constants.Metrics.Tag.APP, "AppWithMapReduce", Constants.Metrics.Tag.MAPREDUCE, "AggregateTimeseriesByTag"), Collections.<String>emptyList())); }
private void getAggregates(Map<String, String> tags, Map<String, String> metricsToCounters, Map<String, Long> result) { Map<String, AggregationFunction> metrics = Maps.newHashMap(); // all map-reduce metrics are gauges for (String metric : metricsToCounters.keySet()) { metrics.put(metric, AggregationFunction.LATEST); } MetricDataQuery metricDataQuery = new MetricDataQuery(0, Integer.MAX_VALUE, Integer.MAX_VALUE, metrics, tags, ImmutableList.<String>of()); Collection<MetricTimeSeries> query = metricStore.query(metricDataQuery); // initialize elements to zero for (String counterName : metricsToCounters.values()) { result.put(counterName, 0L); } for (MetricTimeSeries metricTimeSeries : query) { List<TimeValue> timeValues = metricTimeSeries.getTimeValues(); TimeValue timeValue = Iterables.getOnlyElement(timeValues); result.put(metricsToCounters.get(metricTimeSeries.getMetricName()), timeValue.getValue()); } }
private void queryGroupedAggregates(Map<String, String> tags, Table<String, String, Long> allTaskMetrics, Map<String, String> metricsToCounters) { Map<String, AggregationFunction> metrics = Maps.newHashMap(); // all map-reduce metrics are gauges for (String metric : metricsToCounters.keySet()) { metrics.put(metric, AggregationFunction.LATEST); } MetricDataQuery metricDataQuery = new MetricDataQuery(0, Integer.MAX_VALUE, Integer.MAX_VALUE, metrics, tags, ImmutableList.of(Constants.Metrics.Tag.INSTANCE_ID)); Collection<MetricTimeSeries> query = metricStore.query(metricDataQuery); for (MetricTimeSeries metricTimeSeries : query) { List<TimeValue> timeValues = metricTimeSeries.getTimeValues(); TimeValue timeValue = Iterables.getOnlyElement(timeValues); String taskId = metricTimeSeries.getTagValues().get(Constants.Metrics.Tag.INSTANCE_ID); allTaskMetrics.put(taskId, metricsToCounters.get(metricTimeSeries.getMetricName()), timeValue.getValue()); } } }
private void queryGroupedAggregates(Map<String, String> tags, Table<String, String, Long> allTaskMetrics, Map<String, String> metricsToCounters) { Map<String, AggregationFunction> metrics = Maps.newHashMap(); // all map-reduce metrics are gauges for (String metric : metricsToCounters.keySet()) { metrics.put(metric, AggregationFunction.LATEST); } MetricDataQuery metricDataQuery = new MetricDataQuery(0, Integer.MAX_VALUE, Integer.MAX_VALUE, metrics, tags, ImmutableList.of(Constants.Metrics.Tag.INSTANCE_ID)); Collection<MetricTimeSeries> query = metricStore.query(metricDataQuery); for (MetricTimeSeries metricTimeSeries : query) { List<TimeValue> timeValues = metricTimeSeries.getTimeValues(); TimeValue timeValue = Iterables.getOnlyElement(timeValues); String taskId = metricTimeSeries.getTagValues().get(Constants.Metrics.Tag.INSTANCE_ID); allTaskMetrics.put(taskId, metricsToCounters.get(metricTimeSeries.getMetricName()), timeValue.getValue()); } } }
private Map<String, Long> getSparkDetails(ProgramId sparkProgram, String runId) { Map<String, String> context = new HashMap<>(); context.put(Constants.Metrics.Tag.NAMESPACE, sparkProgram.getNamespace()); context.put(Constants.Metrics.Tag.APP, sparkProgram.getApplication()); context.put(Constants.Metrics.Tag.SPARK, sparkProgram.getProgram()); context.put(Constants.Metrics.Tag.RUN_ID, runId); List<TagValue> tags = new ArrayList<>(); for (Map.Entry<String, String> entry : context.entrySet()) { tags.add(new TagValue(entry.getKey(), entry.getValue())); } MetricSearchQuery metricSearchQuery = new MetricSearchQuery(0, 0, Integer.MAX_VALUE, tags); Collection<String> metricNames = metricStore.findMetricNames(metricSearchQuery); Map<String, Long> overallResult = new HashMap<>(); for (String metricName : metricNames) { Collection<MetricTimeSeries> resultPerQuery = metricStore.query( new MetricDataQuery(0, 0, Integer.MAX_VALUE, metricName, AggregationFunction.SUM, context, new ArrayList<String>())); for (MetricTimeSeries metricTimeSeries : resultPerQuery) { overallResult.put(metricTimeSeries.getMetricName(), metricTimeSeries.getTimeValues().get(0).getValue()); } } return overallResult; }
private long getMetric(MetricStore metricStore, ProgramRunId programRunId, ProfileId profileId, String metricName) { Map<String, String> tags = getMetricsTags(programRunId, profileId); MetricDataQuery query = new MetricDataQuery(0, 0, Integer.MAX_VALUE, metricName, AggregationFunction.SUM, tags, new ArrayList<>()); Collection<MetricTimeSeries> result = metricStore.query(query); if (result.isEmpty()) { return 0; } List<TimeValue> timeValues = result.iterator().next().getTimeValues(); if (timeValues.isEmpty()) { return 0; } return timeValues.get(0).getValue(); } }
protected long getProfileTotalMetric(String metricName) { Map<String, String> tags = new HashMap<>(); tags.put(Constants.Metrics.Tag.PROFILE, Profile.NATIVE_NAME); tags.put(Constants.Metrics.Tag.PROFILE_SCOPE, EntityScope.SYSTEM.name()); MetricDataQuery query = new MetricDataQuery(0, 0, Integer.MAX_VALUE, "system." + metricName, AggregationFunction.SUM, tags, Collections.emptyList()); Collection<MetricTimeSeries> results = metricStore.query(query); if (results.isEmpty()) { return 0; } // since it is totals query and not groupBy specified, we know there's one time series List<TimeValue> timeValues = results.iterator().next().getTimeValues(); if (timeValues.isEmpty()) { return 0; } // since it is totals, we know there's one value only return timeValues.get(0).getValue(); }
@Override public void collect() throws Exception { Collection<MetricTimeSeries> collection = metricStore.query(new MetricDataQuery(0, 0, Integer.MAX_VALUE, Integer.MAX_VALUE, METRICS, Constants.Metrics.TRANSACTION_MANAGER_CONTEXT, Collections.<String>emptyList(), null)); for (MetricTimeSeries metricTimeSeries : collection) { if (metricTimeSeries.getMetricName().equals("system.committing.size")) { numCommittingChangeSets = (int) aggregateMetricValue(metricTimeSeries); } if (metricTimeSeries.getMetricName().equals("system.committed.size")) { numCommittedChangeSets = (int) aggregateMetricValue(metricTimeSeries); } } Transaction transaction = txClient.startShort(); readPointer = transaction.getReadPointer(); writePointer = transaction.getWritePointer(); numInProgressTx = transaction.getInProgress().length; numInvalidTx = transaction.getInvalids().length; txClient.abort(transaction); } }
private long getMetric(MetricStore metricStore, ProgramRunId programRunId, ProfileId profileId, String metricName) { Map<String, String> tags = ImmutableMap.<String, String>builder() .put(Constants.Metrics.Tag.PROFILE_SCOPE, profileId.getScope().name()) .put(Constants.Metrics.Tag.PROFILE, profileId.getProfile()) .put(Constants.Metrics.Tag.NAMESPACE, programRunId.getNamespace()) .put(Constants.Metrics.Tag.PROGRAM_TYPE, programRunId.getType().getPrettyName()) .put(Constants.Metrics.Tag.APP, programRunId.getApplication()) .put(Constants.Metrics.Tag.PROGRAM, programRunId.getProgram()) .build(); MetricDataQuery query = new MetricDataQuery(0, 0, Integer.MAX_VALUE, metricName, AggregationFunction.SUM, tags, new ArrayList<>()); Collection<MetricTimeSeries> result = metricStore.query(query); if (result.isEmpty()) { return 0; } List<TimeValue> timeValues = result.iterator().next().getTimeValues(); if (timeValues.isEmpty()) { return 0; } return timeValues.get(0).getValue(); } }
private long getMetric(MetricStore metricStore, ProgramRunId programRunId, ProfileId profileId, String metricName) { Map<String, String> tags = ImmutableMap.<String, String>builder() .put(Constants.Metrics.Tag.PROFILE_SCOPE, profileId.getScope().name()) .put(Constants.Metrics.Tag.PROFILE, profileId.getProfile()) .put(Constants.Metrics.Tag.NAMESPACE, programRunId.getNamespace()) .put(Constants.Metrics.Tag.PROGRAM_TYPE, programRunId.getType().getPrettyName()) .put(Constants.Metrics.Tag.APP, programRunId.getApplication()) .put(Constants.Metrics.Tag.PROGRAM, programRunId.getProgram()) .build(); MetricDataQuery query = new MetricDataQuery(0, 0, Integer.MAX_VALUE, metricName, AggregationFunction.SUM, tags, new ArrayList<>()); Collection<MetricTimeSeries> result = metricStore.query(query); if (result.isEmpty()) { return 0; } List<TimeValue> timeValues = result.iterator().next().getTimeValues(); if (timeValues.isEmpty()) { return 0; } return timeValues.get(0).getValue(); } }
private MetricQueryResult executeQuery(MetricQueryRequest queryRequest) throws Exception { if (queryRequest.getMetrics().size() == 0) { throw new IllegalArgumentException("Missing metrics parameter in the query"); } Map<String, String> tagsSliceBy = humanToTagNames(transformTagMap(queryRequest.getTags())); MetricQueryRequest.TimeRange timeRange = queryRequest.getTimeRange(); MetricDataQuery query = new MetricDataQuery(timeRange.getStart(), timeRange.getEnd(), timeRange.getResolutionInSeconds(), timeRange.getCount(), toMetrics(queryRequest.getMetrics()), tagsSliceBy, transformGroupByTags(queryRequest.getGroupBy()), timeRange.getInterpolate()); Collection<MetricTimeSeries> queryResult = metricStore.query(query); long endTime = timeRange.getEnd(); if (timeRange.getResolutionInSeconds() == Integer.MAX_VALUE && endTime == 0) { // for aggregate query, we set the end time to be query time (current time) endTime = TimeUnit.MILLISECONDS.toSeconds(System.currentTimeMillis()); } return decorate(queryResult, timeRange.getStart(), endTime, timeRange.getResolutionInSeconds()); }
private MetricQueryResult executeQuery(MetricQueryRequest queryRequest) throws Exception { if (queryRequest.getMetrics().size() == 0) { throw new IllegalArgumentException("Missing metrics parameter in the query"); } Map<String, String> tagsSliceBy = humanToTagNames(transformTagMap(queryRequest.getTags())); MetricQueryRequest.TimeRange timeRange = queryRequest.getTimeRange(); MetricDataQuery query = new MetricDataQuery(timeRange.getStart(), timeRange.getEnd(), timeRange.getResolutionInSeconds(), timeRange.getCount(), toMetrics(queryRequest.getMetrics()), tagsSliceBy, transformGroupByTags(queryRequest.getGroupBy()), timeRange.getInterpolate()); Collection<MetricTimeSeries> queryResult = metricStore.query(query); long endTime = timeRange.getEnd(); if (timeRange.getResolutionInSeconds() == Integer.MAX_VALUE && endTime == 0) { // for aggregate query, we set the end time to be query time (current time) endTime = TimeUnit.MILLISECONDS.toSeconds(System.currentTimeMillis()); } return decorate(queryResult, timeRange.getStart(), endTime, timeRange.getResolutionInSeconds()); }