@Override public String create(PipelineOptions options) { String containerVersion = DataflowRunnerInfo.getDataflowRunnerInfo().getContainerVersion(); return String.format("gcr.io/cloud-dataflow/v1beta3/IMAGE:%s", containerVersion); } }
@Test public void testDataflowDistributionOverride() throws Exception { try (InputStream in = DataflowRunnerInfo.class.getResourceAsStream(DATAFLOW_DISTRIBUTION_PROPERTIES_PATH)) { Properties properties = new Properties(); properties.load(in); assertEquals(properties.getProperty(FNAPI_ENVIRONMENT_MAJOR_VERSION_KEY), DataflowRunnerInfo.getDataflowRunnerInfo().getFnApiEnvironmentMajorVersion()); assertEquals(properties.getProperty(LEGACY_ENVIRONMENT_MAJOR_VERSION_KEY), DataflowRunnerInfo.getDataflowRunnerInfo().getLegacyEnvironmentMajorVersion()); assertEquals(properties.getProperty(CONTAINER_VERSION_KEY), DataflowRunnerInfo.getDataflowRunnerInfo().getContainerVersion()); } } }
@Test public void testFromOptionsUserAgentFromPipelineInfo() throws Exception { DataflowPipelineOptions options = buildPipelineOptions(); DataflowRunner.fromOptions(options); String expectedName = DataflowRunnerInfo.getDataflowRunnerInfo().getName().replace(" ", "_"); assertThat(options.getUserAgent(), containsString(expectedName)); String expectedVersion = DataflowRunnerInfo.getDataflowRunnerInfo().getVersion(); assertThat(options.getUserAgent(), containsString(expectedVersion)); }
/** Helper to configure the Dataflow Job Environment based on the user's job options. */ private static Map<String, Object> getEnvironmentVersion(DataflowPipelineOptions options) { DataflowRunnerInfo runnerInfo = DataflowRunnerInfo.getDataflowRunnerInfo(); String majorVersion; String jobType; if (hasExperiment(options, "beam_fn_api")) { majorVersion = runnerInfo.getFnApiEnvironmentMajorVersion(); jobType = options.isStreaming() ? "FNAPI_STREAMING" : "FNAPI_BATCH"; } else { majorVersion = runnerInfo.getLegacyEnvironmentMajorVersion(); jobType = options.isStreaming() ? "STREAMING" : "JAVA_BATCH_AUTOSCALING"; } return ImmutableMap.of( PropertyNames.ENVIRONMENT_VERSION_MAJOR_KEY, majorVersion, PropertyNames.ENVIRONMENT_VERSION_JOB_TYPE_KEY, jobType); }
@Test public void getDataflowRunnerInfo() throws Exception { DataflowRunnerInfo info = DataflowRunnerInfo.getDataflowRunnerInfo(); String version = info.getLegacyEnvironmentMajorVersion(); // Validate major version is a number assertTrue( String.format("Legacy environment major version number %s is not a number", version), version.matches("\\d+")); version = info.getFnApiEnvironmentMajorVersion(); // Validate major version is a number assertTrue( String.format("FnAPI environment major version number %s is not a number", version), version.matches("\\d+")); // Validate container version does not contain a $ (indicating it was not filled in). assertThat("container version invalid", info.getContainerVersion(), not(containsString("$"))); for (String property : new String[] {"java.vendor", "java.version", "os.arch", "os.name", "os.version"}) { assertEquals(System.getProperty(property), info.getProperties().get(property)); } } }
DataflowRunnerInfo dataflowRunnerInfo = DataflowRunnerInfo.getDataflowRunnerInfo(); String userAgent = String.format("%s/%s", dataflowRunnerInfo.getName(), dataflowRunnerInfo.getVersion())
DataflowRunnerInfo dataflowRunnerInfo = DataflowRunnerInfo.getDataflowRunnerInfo(); String version = dataflowRunnerInfo.getVersion(); checkState(
assertEquals(cloudDataflowDataset, workflowJob.getEnvironment().getDataset()); assertEquals( DataflowRunnerInfo.getDataflowRunnerInfo().getName(), workflowJob.getEnvironment().getUserAgent().get("name")); assertEquals( DataflowRunnerInfo.getDataflowRunnerInfo().getVersion(), workflowJob.getEnvironment().getUserAgent().get("version"));