@VisibleForTesting String renderName(MetricResult<?> metricResult) { String renderedStepName = metricResult.getStep().replaceAll(ILLEGAL_CHARACTERS_AND_PERIOD, "_"); if (renderedStepName.endsWith("_")) { renderedStepName = renderedStepName.substring(0, renderedStepName.length() - 1); } MetricName metricName = metricResult.getName(); return (renderedStepName + "." + metricName.getNamespace() + "." + metricName.getName()) .replaceAll(ILLEGAL_CHARACTERS, "_"); } }
/** * Create a metric that can have its new value set, and is aggregated by taking the last reported * value. */ public static Gauge gauge(Class<?> namespace, String name) { return new DelegatingGauge(MetricName.named(namespace, name)); }
/** * The name of this metric. * * @deprecated to be removed once Dataflow no longer requires this method. */ @Deprecated public String name() { return getName(); }
/** * The namespace associated with this metric. * * @deprecated to be removed once Dataflow no longer requires this method. */ @Deprecated public String namespace() { return getNamespace(); }
public static void queryMetrics(PipelineResult result) { MetricQueryResults metrics = result.metrics().queryMetrics( MetricsFilter.builder().addNameFilter(MetricNameFilter.inNamespace("PollingExample")).build()); Iterable<MetricResult<Long>> counters = metrics.getCounters(); for (MetricResult<Long> counter : counters) { System.out.println(counter.getName().name() + " : " + counter.getAttempted() + " " + Instant.now()); } }
public static Map<String, Long> getMetrics(PipelineResult result) { final MetricQueryResults metricQueryResults = result.metrics().queryMetrics(MetricsFilter.builder().build()); final Map<String, Long> gauges = StreamSupport.stream(metricQueryResults.getGauges().spliterator(), false) .collect(Collectors.groupingBy( MetricResult::getName, Collectors.reducing(GaugeResult.empty(), GET_COMMITTED_GAUGE, BinaryOperator.maxBy( Comparator.comparing(GaugeResult::getTimestamp))))) .entrySet().stream() .collect(Collectors.toMap(e -> e.getKey().getName(), e -> e.getValue().getValue())); final Map<String, Long> counters = StreamSupport.stream( metricQueryResults.getCounters().spliterator(), false) .collect(Collectors.groupingBy(m -> m.getName().getName(), Collectors.summingLong(GET_COMMITTED_COUNTER))); Map<String, Long> ret = new HashMap<>(); ret.putAll(gauges); ret.putAll(counters); return Collections.unmodifiableMap(ret); }
@VisibleForTesting static String getFlinkMetricNameString(MetricResult<?> metricResult) { MetricName metricName = metricResult.getName(); // We use only the MetricName here, the step name is already contained // in the operator name which is passed to Flink's MetricGroup to which // the metric with the following name will be added. return metricName.getNamespace() + METRIC_KEY_SEPARATOR + metricName.getName(); }
/** * Create a metric that can have its new value set, and is aggregated by taking the last reported * value. */ public static Gauge gauge(String namespace, String name) { return new DelegatingGauge(MetricName.named(namespace, name)); }
public MetricQueryResults build() { buildMetricsIndex(); DataflowMetricResultExtractor extractor = new DataflowMetricResultExtractor(dataflowPipelineJob.getDataflowOptions().isStreaming()); for (MetricKey metricKey : metricHashKeys) { String metricName = metricKey.metricName().getName(); if (metricName.endsWith("[MIN]") || metricName.endsWith("[MAX]") || metricName.endsWith("[MEAN]") || metricName.endsWith("[COUNT]")) { // Skip distribution metrics, as these are not yet properly supported. // TODO: remove this when distributions stop being broken up for the UI. continue; } extractor.addMetricResult( metricKey, committedByName.get(metricKey), tentativeByName.get(metricKey)); } return DataflowMetricQueryResults.create( extractor.getCounterResults(), extractor.getDistributionResults(), extractor.getGaugeResults()); } }
private static String getFlinkMetricNameString(String prefix, MetricResult<?> metricResult) { return prefix + METRIC_KEY_SEPARATOR + metricResult.getStep() + METRIC_KEY_SEPARATOR + metricResult.getName().getNamespace() + METRIC_KEY_SEPARATOR + metricResult.getName().getName(); }
/** * Create a metric that can be incremented and decremented, and is aggregated by taking the sum. */ public static Counter counter(String namespace, String name) { return new DelegatingCounter(MetricName.named(namespace, name)); }
@Override protected boolean matchesSafely(MetricUpdate<T> item) { return Objects.equals(name, item.getKey().metricName().getName()) && Objects.equals(update, item.getUpdate()); }
private static boolean matchesName(MetricName metricName, Set<MetricNameFilter> nameFilters) { if (nameFilters.isEmpty()) { return true; } for (MetricNameFilter nameFilter : nameFilters) { if ((nameFilter.getName() == null || nameFilter.getName().equals(metricName.getName())) && Objects.equal(metricName.getNamespace(), nameFilter.getNamespace())) { return true; } } return false; } }
/** * Create a metric that can be incremented and decremented, and is aggregated by taking the sum. */ public static Counter counter(Class<?> namespace, String name) { return new DelegatingCounter(MetricName.named(namespace, name)); }
public static BeamFnApi.Metrics.User.MetricName metricNameToProto(MetricName metricName) { return BeamFnApi.Metrics.User.MetricName.newBuilder() .setNamespace(metricName.getNamespace()) .setName(metricName.getName()) .build(); } }
/** Create a metric that records various statistics about the distribution of reported values. */ public static Distribution distribution(String namespace, String name) { return new DelegatingDistribution(MetricName.named(namespace, name)); }
@Override protected boolean matchesSafely(MetricResult<T> item) { final T metricValue = isCommitted ? item.getCommitted() : item.getAttempted(); return Objects.equals(namespace, item.getName().getNamespace()) && Objects.equals(name, item.getName().getName()) && item.getStep().contains(step) && metricResultsEqual(value, metricValue); }
/** Create a metric that records various statistics about the distribution of reported values. */ public static Distribution distribution(Class<?> namespace, String name) { return new DelegatingDistribution(MetricName.named(namespace, name)); }
String name, String step) { if (!Objects.equals(namespace, item.getName().getNamespace())) { mismatchDescription .appendText("inNamespace: ") .appendValue(namespace) .appendText(" != ") .appendValue(item.getName().getNamespace()); if (!Objects.equals(name, item.getName().getName())) { mismatchDescription .appendText("name: ") .appendValue(name) .appendText(" != ") .appendValue(item.getName().getName());