public static Map<String, Long> getMetrics(PipelineResult result) { final MetricQueryResults metricQueryResults = result.metrics().queryMetrics(MetricsFilter.builder().build()); final Map<String, Long> gauges = StreamSupport.stream(metricQueryResults.getGauges().spliterator(), false) .collect(Collectors.groupingBy( MetricResult::getName, Collectors.reducing(GaugeResult.empty(), GET_COMMITTED_GAUGE, BinaryOperator.maxBy( Comparator.comparing(GaugeResult::getTimestamp))))) .entrySet().stream() .collect(Collectors.toMap(e -> e.getKey().getName(), e -> e.getValue().getValue())); final Map<String, Long> counters = StreamSupport.stream( metricQueryResults.getCounters().spliterator(), false) .collect(Collectors.groupingBy(m -> m.getName().getName(), Collectors.summingLong(GET_COMMITTED_COUNTER))); Map<String, Long> ret = new HashMap<>(); ret.putAll(gauges); ret.putAll(counters); return Collections.unmodifiableMap(ret); }
/** * Return the current value for a long counter, or a default value if can't be retrieved. Note * this uses only attempted metrics because some runners don't support committed metrics. */ public long getCounterMetric(String name, long defaultValue) { MetricQueryResults metrics = result .metrics() .queryMetrics( MetricsFilter.builder() .addNameFilter(MetricNameFilter.named(namespace, name)) .build()); Iterable<MetricResult<Long>> counters = metrics.getCounters(); checkIfMetricResultIsUnique(name, counters); try { MetricResult<Long> metricResult = counters.iterator().next(); return metricResult.getAttempted(); } catch (NoSuchElementException e) { LOG.error("Failed to get metric {}, from namespace {}", name, namespace); } return defaultValue; }
public static void queryMetrics(PipelineResult result) { MetricQueryResults metrics = result.metrics().queryMetrics( MetricsFilter.builder().addNameFilter(MetricNameFilter.inNamespace("PollingExample")).build()); Iterable<MetricResult<Long>> counters = metrics.getCounters(); for (MetricResult<Long> counter : counters) { System.out.println(counter.getName().name() + " : " + counter.getAttempted() + " " + Instant.now()); } }
/** * Return the current value for a long counter, or a default value if can't be retrieved. Note * this uses only attempted metrics because some runners don't support committed metrics. */ private long getDistributionMetric(String name, DistributionType distType, long defaultValue) { MetricQueryResults metrics = result .metrics() .queryMetrics( MetricsFilter.builder() .addNameFilter(MetricNameFilter.named(namespace, name)) .build()); Iterable<MetricResult<DistributionResult>> distributions = metrics.getDistributions(); checkIfMetricResultIsUnique(name, distributions); try { MetricResult<DistributionResult> distributionResult = distributions.iterator().next(); switch (distType) { case MIN: return distributionResult.getAttempted().getMin(); case MAX: return distributionResult.getAttempted().getMax(); default: return defaultValue; } } catch (NoSuchElementException e) { LOG.error("Failed to get distribution metric {} for namespace {}", name, namespace); } return defaultValue; }
Map<String, ?> renderAll() { Map<String, Object> metrics = new HashMap<>(); MetricResults metricResults = asAttemptedOnlyMetricResults(MetricsAccumulator.getInstance().value()); MetricQueryResults metricQueryResults = metricResults.queryMetrics(MetricsFilter.builder().build()); for (MetricResult<Long> metricResult : metricQueryResults.getCounters()) { metrics.put(renderName(metricResult), metricResult.getAttempted()); } for (MetricResult<DistributionResult> metricResult : metricQueryResults.getDistributions()) { DistributionResult result = metricResult.getAttempted(); metrics.put(renderName(metricResult) + ".count", result.getCount()); metrics.put(renderName(metricResult) + ".sum", result.getSum()); metrics.put(renderName(metricResult) + ".min", result.getMin()); metrics.put(renderName(metricResult) + ".max", result.getMax()); metrics.put(renderName(metricResult) + ".mean", result.getMean()); } for (MetricResult<GaugeResult> metricResult : metricQueryResults.getGauges()) { metrics.put(renderName(metricResult), metricResult.getAttempted().getValue()); } return metrics; }
private static Enumerable<Object> count(PipelineOptions options, BeamRelNode node) { Pipeline pipeline = Pipeline.create(options); BeamSqlRelUtils.toPCollection(pipeline, node).apply(ParDo.of(new RowCounter())); PipelineResult result = pipeline.run(); long count = 0; if (!containsUnboundedPCollection(pipeline)) { result.waitUntilFinish(); MetricQueryResults metrics = result .metrics() .queryMetrics( MetricsFilter.builder() .addNameFilter(MetricNameFilter.named(BeamEnumerableConverter.class, "rows")) .build()); Iterator<MetricResult<Long>> iterator = metrics.getCounters().iterator(); if (iterator.hasNext()) { count = iterator.next().getAttempted(); } } return Linq4j.singletonEnumerable(count); }
void updateMetrics() { MetricResults metricResults = asAttemptedOnlyMetricResults(metricsAccumulator.getLocalValue()); MetricQueryResults metricQueryResults = metricResults.queryMetrics(MetricsFilter.builder().build()); updateCounters(metricQueryResults.counters()); updateDistributions(metricQueryResults.distributions()); updateGauge(metricQueryResults.gauges()); }
void updateMetrics(String stepName) { MetricResults metricResults = asAttemptedOnlyMetricResults(metricsAccumulator.getLocalValue()); MetricQueryResults metricQueryResults = metricResults.queryMetrics(MetricsFilter.builder().addStep(stepName).build()); updateCounters(metricQueryResults.getCounters()); updateDistributions(metricQueryResults.getDistributions()); updateGauge(metricQueryResults.getGauges()); }
void updateMetrics(String stepName) { MetricResults metricResults = asAttemptedOnlyMetricResults(metricsAccumulator.getLocalValue()); MetricQueryResults metricQueryResults = metricResults.queryMetrics(MetricsFilter.builder().addStep(stepName).build()); updateCounters(metricQueryResults.getCounters()); updateDistributions(metricQueryResults.getDistributions()); updateGauge(metricQueryResults.getGauges()); }
private static MetricQueryResults queryTestMetrics(PipelineResult result) { return result .metrics() .queryMetrics( MetricsFilter.builder() .addNameFilter(MetricNameFilter.inNamespace(MetricsTest.class)) .build()); }
private void pushMetrics() { if (!(metricsSink instanceof NoOpMetricsSink)) { try { // merge metrics MetricResults metricResults = asAttemptedOnlyMetricResults(metricsContainerStepMap); MetricQueryResults metricQueryResults = metricResults.queryMetrics(MetricsFilter.builder().build()); if ((Iterables.size(metricQueryResults.getDistributions()) != 0) || (Iterables.size(metricQueryResults.getGauges()) != 0) || (Iterables.size(metricQueryResults.getCounters()) != 0)) { metricsSink.writeMetrics(metricQueryResults); } } catch (Exception e) { MetricsPushException metricsPushException = new MetricsPushException(e); metricsPushException.printStackTrace(); } } }
@Test public void testMatchClassNamespaceFilters() { // MetricsFilter with a Class-namespace + name filter. Without step filter. // Successful match. assertTrue( MetricFiltering.matches( MetricsFilter.builder() .addNameFilter(MetricNameFilter.named(MetricFilteringTest.class, "myMetricName")) .build(), MetricKey.create( "anyStep", MetricName.named(MetricFilteringTest.class, "myMetricName")))); // Unsuccessful match. assertFalse( MetricFiltering.matches( MetricsFilter.builder() .addNameFilter(MetricNameFilter.named(MetricFilteringTest.class, "myMetricName")) .build(), MetricKey.create("anyStep", MetricName.named(MetricFiltering.class, "myMetricName")))); }
@Test public void testMatchStringNamespaceFilters() { // MetricsFilter with a String-namespace + name filter. Without step filter. // Successful match. assertTrue( MetricFiltering.matches( MetricsFilter.builder() .addNameFilter(MetricNameFilter.named("myNamespace", "myMetricName")) .build(), MetricKey.create("anyStep", MetricName.named("myNamespace", "myMetricName")))); // Unsuccessful match. assertFalse( MetricFiltering.matches( MetricsFilter.builder() .addNameFilter(MetricNameFilter.named("myOtherNamespace", "myMetricName")) .build(), MetricKey.create("anyStep", MetricName.named("myNamespace", "myMetricname")))); }
@Test public void testCounterCommittedUnsupportedInAttemptedAccumulatedMetricResults() { MetricsContainerStepMap attemptedMetrics = new MetricsContainerStepMap(); attemptedMetrics.update(STEP1, metricsContainer); MetricResults metricResults = asAttemptedOnlyMetricResults(attemptedMetrics); MetricQueryResults step1res = metricResults.queryMetrics(MetricsFilter.builder().addStep(STEP1).build()); thrown.expect(UnsupportedOperationException.class); thrown.expectMessage("This runner does not currently support committed metrics results."); assertCounter(COUNTER_NAME, step1res, STEP1, VALUE, true); }
@Test public void testDistributionCommittedUnsupportedInAttemptedAccumulatedMetricResults() { MetricsContainerStepMap attemptedMetrics = new MetricsContainerStepMap(); attemptedMetrics.update(STEP1, metricsContainer); MetricResults metricResults = asAttemptedOnlyMetricResults(attemptedMetrics); MetricQueryResults step1res = metricResults.queryMetrics(MetricsFilter.builder().addStep(STEP1).build()); thrown.expect(UnsupportedOperationException.class); thrown.expectMessage("This runner does not currently support committed metrics results."); assertDistribution( DISTRIBUTION_NAME, step1res, STEP1, DistributionResult.IDENTITY_ELEMENT, true); }
.metrics() .queryMetrics( MetricsFilter.builder() .addNameFilter(MetricNameFilter.inNamespace(namespace)) .build())
@Test public void testGaugeCommittedUnsupportedInAttemptedAccumulatedMetricResults() { MetricsContainerStepMap attemptedMetrics = new MetricsContainerStepMap(); attemptedMetrics.update(STEP1, metricsContainer); MetricResults metricResults = asAttemptedOnlyMetricResults(attemptedMetrics); MetricQueryResults step1res = metricResults.queryMetrics(MetricsFilter.builder().addStep(STEP1).build()); thrown.expect(UnsupportedOperationException.class); thrown.expectMessage("This runner does not currently support committed metrics results."); assertGauge(GAUGE_NAME, step1res, STEP1, GaugeResult.empty(), true); }
@Test public void testMatchStepNameFilters() { // MetricsFilter with a Class-namespace + name filter + step filter. // Successful match. assertTrue( MetricFiltering.matches( MetricsFilter.builder() .addNameFilter(MetricNameFilter.named(MetricFilteringTest.class, "myMetricName")) .addStep("myStep") .build(), MetricKey.create( "myStep", MetricName.named(MetricFilteringTest.class, "myMetricName")))); // Unsuccessful match. assertFalse( MetricFiltering.matches( MetricsFilter.builder() .addNameFilter(MetricNameFilter.named(MetricFilteringTest.class, "myMetricName")) .addStep("myOtherStep") .build(), MetricKey.create( "myStep", MetricName.named(MetricFilteringTest.class, "myMetricName")))); }
@Test public void testMatchCompositeStepNameFilters() { // MetricsFilter with a Class-namespace + name filter + step filter. // Successful match. assertTrue( MetricFiltering.matches( MetricsFilter.builder() .addNameFilter(MetricNameFilter.named(MetricFilteringTest.class, "myMetricName")) .addStep("myStep") .build(), MetricKey.create( "myBigStep/myStep", MetricName.named(MetricFilteringTest.class, "myMetricName")))); // Unsuccessful match. assertFalse( MetricFiltering.matches( MetricsFilter.builder() .addNameFilter(MetricNameFilter.named(MetricFilteringTest.class, "myMetricName")) .addStep("myOtherStep") .build(), MetricKey.create( "myOtherStepNoMatch/myStep", MetricName.named(MetricFilteringTest.class, "myMetricName")))); }
@Test @Category({NeedsRunner.class, UsesAttemptedMetrics.class, UsesCounterMetrics.class}) public void testBoundedSourceMetrics() { long numElements = 1000; pipeline.apply(GenerateSequence.from(0).to(numElements)); PipelineResult pipelineResult = pipeline.run(); MetricQueryResults metrics = pipelineResult .metrics() .queryMetrics( MetricsFilter.builder() .addNameFilter( MetricNameFilter.named( ELEMENTS_READ.getNamespace(), ELEMENTS_READ.getName())) .build()); assertThat( metrics.getCounters(), hasItem( attemptedMetricsResult( ELEMENTS_READ.getNamespace(), ELEMENTS_READ.getName(), "Read(BoundedCountingSource)", 1000L))); }