@VisibleForTesting static String getContainerImageForJob(DataflowPipelineOptions options) { String workerHarnessContainerImage = options.getWorkerHarnessContainerImage(); if (!workerHarnessContainerImage.contains("IMAGE")) { return workerHarnessContainerImage; } else if (hasExperiment(options, "beam_fn_api")) { return workerHarnessContainerImage.replace("IMAGE", "java"); } else if (options.isStreaming()) { return workerHarnessContainerImage.replace("IMAGE", "beam-java-streaming"); } else { return workerHarnessContainerImage.replace("IMAGE", "beam-java-batch"); } }
@VisibleForTesting protected void replaceTransforms(Pipeline pipeline) { boolean streaming = options.isStreaming() || containsUnboundedPCollection(pipeline); // Ensure all outputs of all reads are consumed before potentially replacing any // Read PTransforms UnconsumedReads.ensureAllReadsConsumed(pipeline); pipeline.replaceAll(getOverrides(streaming)); }
/** Helper to configure the Dataflow Job Environment based on the user's job options. */ private static Map<String, Object> getEnvironmentVersion(DataflowPipelineOptions options) { DataflowRunnerInfo runnerInfo = DataflowRunnerInfo.getDataflowRunnerInfo(); String majorVersion; String jobType; if (hasExperiment(options, "beam_fn_api")) { majorVersion = runnerInfo.getFnApiEnvironmentMajorVersion(); jobType = options.isStreaming() ? "FNAPI_STREAMING" : "FNAPI_BATCH"; } else { majorVersion = runnerInfo.getLegacyEnvironmentMajorVersion(); jobType = options.isStreaming() ? "STREAMING" : "JAVA_BATCH_AUTOSCALING"; } return ImmutableMap.of( PropertyNames.ENVIRONMENT_VERSION_MAJOR_KEY, majorVersion, PropertyNames.ENVIRONMENT_VERSION_JOB_TYPE_KEY, jobType); }
public MetricQueryResults build() { buildMetricsIndex(); DataflowMetricResultExtractor extractor = new DataflowMetricResultExtractor(dataflowPipelineJob.getDataflowOptions().isStreaming()); for (MetricKey metricKey : metricHashKeys) { String metricName = metricKey.metricName().getName(); if (metricName.endsWith("[MIN]") || metricName.endsWith("[MAX]") || metricName.endsWith("[MEAN]") || metricName.endsWith("[COUNT]")) { // Skip distribution metrics, as these are not yet properly supported. // TODO: remove this when distributions stop being broken up for the UI. continue; } extractor.addMetricResult( metricKey, committedByName.get(metricKey), tentativeByName.get(metricKey)); } return DataflowMetricQueryResults.create( extractor.getCounterResults(), extractor.getDistributionResults(), extractor.getGaugeResults()); } }
@Test public void testEmptyMetricUpdates() throws IOException { Job modelJob = new Job(); modelJob.setCurrentState(State.RUNNING.toString()); DataflowPipelineJob job = mock(DataflowPipelineJob.class); DataflowPipelineOptions options = mock(DataflowPipelineOptions.class); when(options.isStreaming()).thenReturn(false); when(job.getDataflowOptions()).thenReturn(options); when(job.getState()).thenReturn(State.RUNNING); job.jobId = JOB_ID; JobMetrics jobMetrics = new JobMetrics(); jobMetrics.setMetrics(null /* this is how the APIs represent empty metrics */); DataflowClient dataflowClient = mock(DataflowClient.class); when(dataflowClient.getJobMetrics(JOB_ID)).thenReturn(jobMetrics); DataflowMetrics dataflowMetrics = new DataflowMetrics(job, dataflowClient); MetricQueryResults result = dataflowMetrics.queryMetrics(null); assertThat(ImmutableList.copyOf(result.getCounters()), is(empty())); assertThat(ImmutableList.copyOf(result.getDistributions()), is(empty())); }
@Test public void testCachingMetricUpdates() throws IOException { Job modelJob = new Job(); modelJob.setCurrentState(State.RUNNING.toString()); DataflowPipelineJob job = mock(DataflowPipelineJob.class); DataflowPipelineOptions options = mock(DataflowPipelineOptions.class); when(options.isStreaming()).thenReturn(false); when(job.getDataflowOptions()).thenReturn(options); when(job.getState()).thenReturn(State.DONE); job.jobId = JOB_ID; JobMetrics jobMetrics = new JobMetrics(); jobMetrics.setMetrics(ImmutableList.of()); DataflowClient dataflowClient = mock(DataflowClient.class); when(dataflowClient.getJobMetrics(JOB_ID)).thenReturn(jobMetrics); DataflowMetrics dataflowMetrics = new DataflowMetrics(job, dataflowClient); verify(dataflowClient, times(0)).getJobMetrics(JOB_ID); dataflowMetrics.queryMetrics(null); verify(dataflowClient, times(1)).getJobMetrics(JOB_ID); dataflowMetrics.queryMetrics(null); verify(dataflowClient, times(1)).getJobMetrics(JOB_ID); }
if (dataflowOptions.isStreaming() && dataflowOptions.getGcsUploadBufferSizeBytes() == null) { dataflowOptions.setGcsUploadBufferSizeBytes(GCS_UPLOAD_BUFFER_SIZE_BYTES_DEFAULT);
DataflowPipelineJob job = mock(DataflowPipelineJob.class); DataflowPipelineOptions options = mock(DataflowPipelineOptions.class); when(options.isStreaming()).thenReturn(false); when(job.getDataflowOptions()).thenReturn(options); when(job.getState()).thenReturn(State.RUNNING);
DataflowPipelineJob job = mock(DataflowPipelineJob.class); DataflowPipelineOptions options = mock(DataflowPipelineOptions.class); when(options.isStreaming()).thenReturn(false); when(job.getDataflowOptions()).thenReturn(options); when(job.getState()).thenReturn(State.RUNNING);
DataflowPipelineJob job = mock(DataflowPipelineJob.class); DataflowPipelineOptions options = mock(DataflowPipelineOptions.class); when(options.isStreaming()).thenReturn(true); when(job.getDataflowOptions()).thenReturn(options); when(job.getState()).thenReturn(State.RUNNING);
if (context.getPipelineOptions().isStreaming() && (signature.usesState() || signature.usesTimers())) { stepContext.addInput(PropertyNames.USES_KEYED_STATE, "true");
DataflowPipelineJob job = mock(DataflowPipelineJob.class); DataflowPipelineOptions options = mock(DataflowPipelineOptions.class); when(options.isStreaming()).thenReturn(false); when(job.getDataflowOptions()).thenReturn(options); when(job.getState()).thenReturn(State.RUNNING);
DataflowPipelineJob job = mock(DataflowPipelineJob.class); DataflowPipelineOptions options = mock(DataflowPipelineOptions.class); when(options.isStreaming()).thenReturn(true); when(job.getDataflowOptions()).thenReturn(options); when(job.getState()).thenReturn(State.RUNNING);
DataflowPipelineJob job = mock(DataflowPipelineJob.class); DataflowPipelineOptions options = mock(DataflowPipelineOptions.class); when(options.isStreaming()).thenReturn(false); when(job.getDataflowOptions()).thenReturn(options); when(job.getState()).thenReturn(State.RUNNING);
@Override public void translate(Impulse transform, TranslationContext context) { if (context.getPipelineOptions().isStreaming()) { StepTranslationContext stepContext = context.addStep(transform, "ParallelRead"); stepContext.addInput(PropertyNames.FORMAT, "pubsub"); stepContext.addInput(PropertyNames.PUBSUB_SUBSCRIPTION, "_starting_signal/"); stepContext.addOutput(PropertyNames.OUTPUT, context.getOutput(transform)); } else { StepTranslationContext stepContext = context.addStep(transform, "ParallelRead"); stepContext.addInput(PropertyNames.FORMAT, "impulse"); WindowedValue.FullWindowedValueCoder<byte[]> coder = WindowedValue.getFullCoder( context.getOutput(transform).getCoder(), GlobalWindow.Coder.INSTANCE); byte[] encodedImpulse; try { encodedImpulse = encodeToByteArray(coder, WindowedValue.valueInGlobalWindow(new byte[0])); } catch (Exception e) { throw new RuntimeException(e); } stepContext.addInput(PropertyNames.IMPULSE_ELEMENT, byteArrayToJsonString(encodedImpulse)); stepContext.addOutput(PropertyNames.OUTPUT, context.getOutput(transform)); } } }
optionsMap, hasEntry("stableUniqueNames", (Object) options.getStableUniqueNames().toString())); assertThat(optionsMap, hasEntry("streaming", (Object) options.isStreaming())); assertThat( optionsMap,
if (options.isStreaming()) { job.setType("JOB_TYPE_STREAMING"); } else { job.setLabels(options.getLabels()); if (options.isStreaming() && !hasExperiment(options, "enable_windmill_service")) {
@Override public void translate(StreamingPubsubIOWrite transform, TranslationContext context) { checkArgument( context.getPipelineOptions().isStreaming(), "StreamingPubsubIOWrite is only for streaming pipelines."); PubsubUnboundedSink overriddenTransform = transform.getOverriddenTransform();
@Override public void translate(StreamingPubsubIORead transform, TranslationContext context) { checkArgument( context.getPipelineOptions().isStreaming(), "StreamingPubsubIORead is only for streaming pipelines."); PubsubUnboundedSource overriddenTransform = transform.getOverriddenTransform();