@Test public void testEmptyMetricUpdates() throws IOException { Job modelJob = new Job(); modelJob.setCurrentState(State.RUNNING.toString()); DataflowPipelineJob job = mock(DataflowPipelineJob.class); DataflowPipelineOptions options = mock(DataflowPipelineOptions.class); when(options.isStreaming()).thenReturn(false); when(job.getDataflowOptions()).thenReturn(options); when(job.getState()).thenReturn(State.RUNNING); job.jobId = JOB_ID; JobMetrics jobMetrics = new JobMetrics(); jobMetrics.setMetrics(null /* this is how the APIs represent empty metrics */); DataflowClient dataflowClient = mock(DataflowClient.class); when(dataflowClient.getJobMetrics(JOB_ID)).thenReturn(jobMetrics); DataflowMetrics dataflowMetrics = new DataflowMetrics(job, dataflowClient); MetricQueryResults result = dataflowMetrics.queryMetrics(null); assertThat(ImmutableList.copyOf(result.getCounters()), is(empty())); assertThat(ImmutableList.copyOf(result.getDistributions()), is(empty())); }
@Override public MetricQueryResults queryMetrics(@Nullable MetricsFilter filter) { List<MetricUpdate> metricUpdates; ImmutableList<MetricResult<Long>> counters = ImmutableList.of(); ImmutableList<MetricResult<DistributionResult>> distributions = ImmutableList.of(); ImmutableList<MetricResult<GaugeResult>> gauges = ImmutableList.of(); JobMetrics jobMetrics; try { jobMetrics = getJobMetrics(); } catch (IOException e) { LOG.warn("Unable to query job metrics.\n"); return DataflowMetricQueryResults.create(counters, distributions, gauges); } metricUpdates = firstNonNull(jobMetrics.getMetrics(), Collections.<MetricUpdate>emptyList()); return populateMetricQueryResults(metricUpdates, filter); }
/** * Constructs the job. * * @param jobId the job id * @param dataflowOptions used to configure the client for the Dataflow Service * @param transformStepNames a mapping from AppliedPTransforms to Step Names */ public DataflowPipelineJob( DataflowClient dataflowClient, String jobId, DataflowPipelineOptions dataflowOptions, Map<AppliedPTransform<?, ?, ?>, String> transformStepNames) { this.dataflowClient = dataflowClient; this.jobId = jobId; this.dataflowOptions = dataflowOptions; this.transformStepNames = HashBiMap.create(firstNonNull(transformStepNames, ImmutableMap.of())); this.dataflowMetrics = new DataflowMetrics(this, this.dataflowClient); }
@Test public void testCachingMetricUpdates() throws IOException { Job modelJob = new Job(); modelJob.setCurrentState(State.RUNNING.toString()); DataflowPipelineJob job = mock(DataflowPipelineJob.class); DataflowPipelineOptions options = mock(DataflowPipelineOptions.class); when(options.isStreaming()).thenReturn(false); when(job.getDataflowOptions()).thenReturn(options); when(job.getState()).thenReturn(State.DONE); job.jobId = JOB_ID; JobMetrics jobMetrics = new JobMetrics(); jobMetrics.setMetrics(ImmutableList.of()); DataflowClient dataflowClient = mock(DataflowClient.class); when(dataflowClient.getJobMetrics(JOB_ID)).thenReturn(jobMetrics); DataflowMetrics dataflowMetrics = new DataflowMetrics(job, dataflowClient); verify(dataflowClient, times(0)).getJobMetrics(JOB_ID); dataflowMetrics.queryMetrics(null); verify(dataflowClient, times(1)).getJobMetrics(JOB_ID); dataflowMetrics.queryMetrics(null); verify(dataflowClient, times(1)).getJobMetrics(JOB_ID); }
when(dataflowClient.getJobMetrics(JOB_ID)).thenReturn(jobMetrics); DataflowMetrics dataflowMetrics = new DataflowMetrics(job, dataflowClient); MetricQueryResults result = dataflowMetrics.queryMetrics(null); assertThat( result.getCounters(),
"distributionName", "distributionNamespace", "s2", 18L, 2L, 2L, 16L, true))); DataflowMetrics dataflowMetrics = new DataflowMetrics(job, dataflowClient); MetricQueryResults result = dataflowMetrics.queryMetrics(null); try { result.getDistributions().iterator().next().getCommitted();
makeCounterMetricUpdate("otherCounter[MIN]", "otherNamespace", "s2", 0L, true))); DataflowMetrics dataflowMetrics = new DataflowMetrics(job, dataflowClient); MetricQueryResults result = dataflowMetrics.queryMetrics(null); assertThat( result.getCounters(),
"distributionName", "distributionNamespace", "s2", 18L, 2L, 2L, 16L, true))); DataflowMetrics dataflowMetrics = new DataflowMetrics(job, dataflowClient); MetricQueryResults result = dataflowMetrics.queryMetrics(null); assertThat( result.getDistributions(),
makeCounterMetricUpdate("counterName", "otherNamespace", "s4", 1233L, true))); DataflowMetrics dataflowMetrics = new DataflowMetrics(job, dataflowClient); MetricQueryResults result = dataflowMetrics.queryMetrics(null); try { result.getCounters().iterator().next().getCommitted();
makeCounterMetricUpdate("lostName", "otherNamespace", "s5", 1200L, true))); DataflowMetrics dataflowMetrics = new DataflowMetrics(job, dataflowClient); MetricQueryResults result = dataflowMetrics.queryMetrics(null); assertThat( result.getCounters(),