/** Returns true if the specified experiment is enabled, handling null experiments. */ public static boolean hasExperiment(DataflowPipelineDebugOptions options, String experiment) { List<String> experiments = firstNonNull(options.getExperiments(), Collections.<String>emptyList()); return experiments.contains(experiment); }
/** * Stages {@link DataflowPipelineOptions#getFilesToStage()}, which defaults to every file on the * classpath unless overridden, as well as {@link * DataflowPipelineDebugOptions#getOverrideWindmillBinary()} if specified. * * @see #stageFiles(List) */ @Override public List<DataflowPackage> stageDefaultFiles() { checkNotNull(options.getStagingLocation()); String windmillBinary = options.as(DataflowPipelineDebugOptions.class).getOverrideWindmillBinary(); String dataflowWorkerJar = options.getDataflowWorkerJar(); List<String> filesToStage = options.getFilesToStage(); if (windmillBinary != null) { filesToStage.add("windmill_main=" + windmillBinary); } if (dataflowWorkerJar != null && !dataflowWorkerJar.isEmpty()) { filesToStage.add("dataflow-worker.jar=" + dataflowWorkerJar); } return stageFiles(filesToStage); }
@Test public void testTransformNameMapping() throws Exception { DataflowPipelineDebugOptions options = PipelineOptionsFactory.fromArgs( "--transformNameMapping={\"a\":\"b\",\"foo\":\"\",\"bar\":\"baz\"}") .as(DataflowPipelineDebugOptions.class); assertEquals(3, options.getTransformNameMapping().size()); assertThat(options.getTransformNameMapping(), hasEntry("a", "b")); assertThat(options.getTransformNameMapping(), hasEntry("foo", "")); assertThat(options.getTransformNameMapping(), hasEntry("bar", "baz")); } }
@Override public Stager create(PipelineOptions options) { DataflowPipelineDebugOptions debugOptions = options.as(DataflowPipelineDebugOptions.class); return InstanceBuilder.ofType(Stager.class) .fromClass(debugOptions.getStagerClass()) .fromFactoryMethod("fromOptions") .withArg(PipelineOptions.class, options) .build(); } }
dataflowOptions.as(DataflowPipelineDebugOptions.class); if (debugOptions.getNumberOfWorkerHarnessThreads() < 0) { throw new IllegalArgumentException( "Number of worker harness threads '" + debugOptions.getNumberOfWorkerHarnessThreads() + "' invalid. Please make sure the value is non-negative.");
@Test public void testInvalidNumberOfWorkerHarnessThreads() throws IOException { DataflowPipelineOptions options = PipelineOptionsFactory.as(DataflowPipelineOptions.class); FileSystems.setDefaultPipelineOptions(options); options.setRunner(DataflowRunner.class); options.setProject("foo-12345"); options.setGcpTempLocation(VALID_TEMP_BUCKET); options.setGcsUtil(mockGcsUtil); options.as(DataflowPipelineDebugOptions.class).setNumberOfWorkerHarnessThreads(-1); thrown.expect(IllegalArgumentException.class); thrown.expectMessage("Number of worker harness threads"); thrown.expectMessage("Please make sure the value is non-negative."); DataflowRunner.fromOptions(options); }