protected long aggregateMetricValue(MetricTimeSeries metricTimeSery) { long aggregateValue = 0L; for (TimeValue timeValue : metricTimeSery.getTimeValues()) { aggregateValue += timeValue.getValue(); } return aggregateValue; } }
private void queryGroupedAggregates(Map<String, String> tags, Table<String, String, Long> allTaskMetrics, Map<String, String> metricsToCounters) { Map<String, AggregationFunction> metrics = Maps.newHashMap(); // all map-reduce metrics are gauges for (String metric : metricsToCounters.keySet()) { metrics.put(metric, AggregationFunction.LATEST); } MetricDataQuery metricDataQuery = new MetricDataQuery(0, Integer.MAX_VALUE, Integer.MAX_VALUE, metrics, tags, ImmutableList.of(Constants.Metrics.Tag.INSTANCE_ID)); Collection<MetricTimeSeries> query = metricStore.query(metricDataQuery); for (MetricTimeSeries metricTimeSeries : query) { List<TimeValue> timeValues = metricTimeSeries.getTimeValues(); TimeValue timeValue = Iterables.getOnlyElement(timeValues); String taskId = metricTimeSeries.getTagValues().get(Constants.Metrics.Tag.INSTANCE_ID); allTaskMetrics.put(taskId, metricsToCounters.get(metricTimeSeries.getMetricName()), timeValue.getValue()); } } }
private Map<String, Long> getTotalsWithSingleGroupByTag(MetricDataQuery query) throws Exception { // query must have resolution set to Integer.MAX_VALUE (i.e. "totals") Collection<MetricTimeSeries> result = metricStore.query(query); Map<String, Long> map = Maps.newHashMap(); for (MetricTimeSeries timeSeries : result) { // we know there's only ony group by tag String groupByTagValue = timeSeries.getTagValues().values().iterator().next(); // since it is totals, it will have only one TimeValue map.put(groupByTagValue, timeSeries.getTimeValues().get(0).getValue()); } return map; } }
private void getAggregates(Map<String, String> tags, Map<String, String> metricsToCounters, Map<String, Long> result) { Map<String, AggregationFunction> metrics = Maps.newHashMap(); // all map-reduce metrics are gauges for (String metric : metricsToCounters.keySet()) { metrics.put(metric, AggregationFunction.LATEST); } MetricDataQuery metricDataQuery = new MetricDataQuery(0, Integer.MAX_VALUE, Integer.MAX_VALUE, metrics, tags, ImmutableList.<String>of()); Collection<MetricTimeSeries> query = metricStore.query(metricDataQuery); // initialize elements to zero for (String counterName : metricsToCounters.values()) { result.put(counterName, 0L); } for (MetricTimeSeries metricTimeSeries : query) { List<TimeValue> timeValues = metricTimeSeries.getTimeValues(); TimeValue timeValue = Iterables.getOnlyElement(timeValues); result.put(metricsToCounters.get(metricTimeSeries.getMetricName()), timeValue.getValue()); } }
@Override public void collect() throws Exception { Collection<MetricTimeSeries> collection = metricStore.query(new MetricDataQuery(0, 0, Integer.MAX_VALUE, Integer.MAX_VALUE, METRICS, Constants.Metrics.TRANSACTION_MANAGER_CONTEXT, Collections.<String>emptyList(), null)); for (MetricTimeSeries metricTimeSeries : collection) { if (metricTimeSeries.getMetricName().equals("system.committing.size")) { numCommittingChangeSets = (int) aggregateMetricValue(metricTimeSeries); } if (metricTimeSeries.getMetricName().equals("system.committed.size")) { numCommittedChangeSets = (int) aggregateMetricValue(metricTimeSeries); } } Transaction transaction = txClient.startShort(); readPointer = transaction.getReadPointer(); writePointer = transaction.getWritePointer(); numInProgressTx = transaction.getInProgress().length; numInvalidTx = transaction.getInvalids().length; txClient.abort(transaction); } }
@Override public Collection<MetricTimeSeries> query(MetricDataQuery query) { Collection<TimeSeries> cubeResult = cube.get().query(buildCubeQuery(query)); List<MetricTimeSeries> result = Lists.newArrayList(); for (TimeSeries timeSeries : cubeResult) { result.add(new MetricTimeSeries(timeSeries.getMeasureName(), timeSeries.getDimensionValues(), timeSeries.getTimeValues())); } return result; }
private void getAggregates(Map<String, String> tags, Map<String, String> metricsToCounters, Map<String, Long> result) { Map<String, AggregationFunction> metrics = Maps.newHashMap(); // all map-reduce metrics are gauges for (String metric : metricsToCounters.keySet()) { metrics.put(metric, AggregationFunction.LATEST); } MetricDataQuery metricDataQuery = new MetricDataQuery(0, Integer.MAX_VALUE, Integer.MAX_VALUE, metrics, tags, ImmutableList.<String>of()); Collection<MetricTimeSeries> query = metricStore.query(metricDataQuery); // initialize elements to zero for (String counterName : metricsToCounters.values()) { result.put(counterName, 0L); } for (MetricTimeSeries metricTimeSeries : query) { List<TimeValue> timeValues = metricTimeSeries.getTimeValues(); TimeValue timeValue = Iterables.getOnlyElement(timeValues); result.put(metricsToCounters.get(metricTimeSeries.getMetricName()), timeValue.getValue()); } }
switch (metricTimeSery.getMetricName()) { case "system.request.received": totalRequests = aggregateMetricValue(metricTimeSery);
@Override public Collection<MetricTimeSeries> query(MetricDataQuery query) { Collection<TimeSeries> cubeResult = cube.get().query(buildCubeQuery(query)); List<MetricTimeSeries> result = Lists.newArrayList(); for (TimeSeries timeSeries : cubeResult) { result.add(new MetricTimeSeries(timeSeries.getMeasureName(), timeSeries.getDimensionValues(), timeSeries.getTimeValues())); } return result; }
private long getTotals(MetricDataQuery query) throws Exception { // query must have resolution set to Integer.MAX_VALUE (i.e. "totals") Collection<MetricTimeSeries> result = metricStore.query(query); if (result.size() == 0) { return 0; } // since there's no group by condition, it'll return single time series always MetricTimeSeries timeSeries = result.iterator().next(); if (timeSeries.getTimeValues().isEmpty()) { return 0; } // since it is totals, it will have only one TimeValue or none return timeSeries.getTimeValues().get(0).getValue(); }
private MetricQueryResult decorate(Collection<MetricTimeSeries> series, long startTs, long endTs, int resolution) { MetricQueryResult.TimeSeries[] serieses = new MetricQueryResult.TimeSeries[series.size()]; int i = 0; for (MetricTimeSeries timeSeries : series) { MetricQueryResult.TimeValue[] timeValues = decorate(timeSeries.getTimeValues()); serieses[i++] = new MetricQueryResult.TimeSeries(timeSeries.getMetricName(), tagNamesToHuman(timeSeries.getTagValues()), timeValues); } return new MetricQueryResult(startTs, endTs, serieses, resolution); }
private Map<String, Long> getSparkDetails(ProgramId sparkProgram, String runId) { Map<String, String> context = new HashMap<>(); context.put(Constants.Metrics.Tag.NAMESPACE, sparkProgram.getNamespace()); context.put(Constants.Metrics.Tag.APP, sparkProgram.getApplication()); context.put(Constants.Metrics.Tag.SPARK, sparkProgram.getProgram()); context.put(Constants.Metrics.Tag.RUN_ID, runId); List<TagValue> tags = new ArrayList<>(); for (Map.Entry<String, String> entry : context.entrySet()) { tags.add(new TagValue(entry.getKey(), entry.getValue())); } MetricSearchQuery metricSearchQuery = new MetricSearchQuery(0, 0, Integer.MAX_VALUE, tags); Collection<String> metricNames = metricStore.findMetricNames(metricSearchQuery); Map<String, Long> overallResult = new HashMap<>(); for (String metricName : metricNames) { Collection<MetricTimeSeries> resultPerQuery = metricStore.query( new MetricDataQuery(0, 0, Integer.MAX_VALUE, metricName, AggregationFunction.SUM, context, new ArrayList<String>())); for (MetricTimeSeries metricTimeSeries : resultPerQuery) { overallResult.put(metricTimeSeries.getMetricName(), metricTimeSeries.getTimeValues().get(0).getValue()); } } return overallResult; }
private Iterator<TimeValue> queryTimeSeries(MetricDataQuery query) throws Exception { Collection<MetricTimeSeries> result = metricStore.query(query); if (result.size() == 0) { return new ArrayList<TimeValue>().iterator(); } // since there's no group by condition, it'll return single time series always MetricTimeSeries timeSeries = result.iterator().next(); return Iterables.transform(timeSeries.getTimeValues(), new Function<TimeValue, TimeValue>() { @Override public TimeValue apply(TimeValue input) { return new TimeValue(input.getTimestamp(), input.getValue()); } }).iterator(); }
private MetricQueryResult decorate(Collection<MetricTimeSeries> series, long startTs, long endTs, int resolution) { MetricQueryResult.TimeSeries[] serieses = new MetricQueryResult.TimeSeries[series.size()]; int i = 0; for (MetricTimeSeries timeSeries : series) { MetricQueryResult.TimeValue[] timeValues = decorate(timeSeries.getTimeValues()); serieses[i++] = new MetricQueryResult.TimeSeries(timeSeries.getMetricName(), tagNamesToHuman(timeSeries.getTagValues()), timeValues); } return new MetricQueryResult(startTs, endTs, serieses, resolution); }
private Map<String, Long> getSparkDetails(ProgramId sparkProgram, String runId) { Map<String, String> context = new HashMap<>(); context.put(Constants.Metrics.Tag.NAMESPACE, sparkProgram.getNamespace()); context.put(Constants.Metrics.Tag.APP, sparkProgram.getApplication()); context.put(Constants.Metrics.Tag.SPARK, sparkProgram.getProgram()); context.put(Constants.Metrics.Tag.RUN_ID, runId); List<TagValue> tags = new ArrayList<>(); for (Map.Entry<String, String> entry : context.entrySet()) { tags.add(new TagValue(entry.getKey(), entry.getValue())); } MetricSearchQuery metricSearchQuery = new MetricSearchQuery(0, 0, Integer.MAX_VALUE, tags); Collection<String> metricNames = metricStore.findMetricNames(metricSearchQuery); Map<String, Long> overallResult = new HashMap<>(); for (String metricName : metricNames) { Collection<MetricTimeSeries> resultPerQuery = metricStore.query( new MetricDataQuery(0, 0, Integer.MAX_VALUE, metricName, AggregationFunction.SUM, context, new ArrayList<String>())); for (MetricTimeSeries metricTimeSeries : resultPerQuery) { overallResult.put(metricTimeSeries.getMetricName(), metricTimeSeries.getTimeValues().get(0).getValue()); } } return overallResult; }
private long getSingleValueFromTotals(MetricDataQuery query) { try { Collection<MetricTimeSeries> result = metricStore.query(query); if (result.isEmpty()) { return 0; } // since it is totals query and not groupBy specified, we know there's one time series List<TimeValue> timeValues = result.iterator().next().getTimeValues(); if (timeValues.isEmpty()) { return 0; } // since it is totals, we know there's one value only return timeValues.get(0).getValue(); } catch (Exception e) { throw Throwables.propagate(e); } } }
private void queryGroupedAggregates(Map<String, String> tags, Table<String, String, Long> allTaskMetrics, Map<String, String> metricsToCounters) { Map<String, AggregationFunction> metrics = Maps.newHashMap(); // all map-reduce metrics are gauges for (String metric : metricsToCounters.keySet()) { metrics.put(metric, AggregationFunction.LATEST); } MetricDataQuery metricDataQuery = new MetricDataQuery(0, Integer.MAX_VALUE, Integer.MAX_VALUE, metrics, tags, ImmutableList.of(Constants.Metrics.Tag.INSTANCE_ID)); Collection<MetricTimeSeries> query = metricStore.query(metricDataQuery); for (MetricTimeSeries metricTimeSeries : query) { List<TimeValue> timeValues = metricTimeSeries.getTimeValues(); TimeValue timeValue = Iterables.getOnlyElement(timeValues); String taskId = metricTimeSeries.getTagValues().get(Constants.Metrics.Tag.INSTANCE_ID); allTaskMetrics.put(taskId, metricsToCounters.get(metricTimeSeries.getMetricName()), timeValue.getValue()); } } }
private long getSingleValueFromTotals(MetricDataQuery query) { try { Collection<MetricTimeSeries> result = metricStore.query(query); if (result.isEmpty()) { return 0; } // since it is totals query and not groupBy specified, we know there's one time series List<TimeValue> timeValues = result.iterator().next().getTimeValues(); if (timeValues.isEmpty()) { return 0; } // since it is totals, we know there's one value only return timeValues.get(0).getValue(); } catch (Exception e) { throw Throwables.propagate(e); } } }
List<TimeValue> timeValues = metric.getTimeValues(); if (timeValues == null || timeValues.size() != 1) { throw new IOException("Should collect exactly one time value");
private long getMetric(MetricStore metricStore, ProgramRunId programRunId, ProfileId profileId, String metricName) { Map<String, String> tags = getMetricsTags(programRunId, profileId); MetricDataQuery query = new MetricDataQuery(0, 0, Integer.MAX_VALUE, metricName, AggregationFunction.SUM, tags, new ArrayList<>()); Collection<MetricTimeSeries> result = metricStore.query(query); if (result.isEmpty()) { return 0; } List<TimeValue> timeValues = result.iterator().next().getTimeValues(); if (timeValues.isEmpty()) { return 0; } return timeValues.get(0).getValue(); } }