public static TestFlinkRunner fromOptions(PipelineOptions options) { options.setRunner(TestFlinkRunner.class); FlinkPipelineOptions flinkOptions = PipelineOptionsValidator.validate(FlinkPipelineOptions.class, options); return new TestFlinkRunner(flinkOptions); }
private static PipelineOptions getOpts(String exec, String[] args) { switch (exec) { case "flink": PipelineOptions ret1 = PipelineOptionsFactory.fromArgs(args).as(FlinkPipelineOptions.class); ret1.setRunner(FlinkRunner.class); return ret1; case "flink-test": PipelineOptions ret2 = PipelineOptionsFactory.fromArgs(args).as(FlinkPipelineOptions.class); ret2.setRunner(TestFlinkRunner.class); return ret2; } throw new IllegalArgumentException("Unknown executor: " + exec); } }
options.setRunner(CrashingRunner.class);
/** * Construct a runner from the provided options. * * @param options Properties which configure the runner. * @return The newly created runner. */ public static FlinkRunner fromOptions(PipelineOptions options) { options.setRunner(FlinkRunner.class); FlinkPipelineOptions flinkOptions = PipelineOptionsValidator.validate(FlinkPipelineOptions.class, options); ArrayList<String> missing = new ArrayList<>(); if (flinkOptions.getAppName() == null) { missing.add("appName"); } if (missing.size() > 0) { throw new IllegalArgumentException( "Missing required values: " + Joiner.on(',').join(missing)); } if (flinkOptions.getFilesToStage() == null) { flinkOptions.setFilesToStage( detectClassPathResourcesToStage(FlinkRunner.class.getClassLoader())); LOG.info( "PipelineOptions.filesToStage was not specified. " + "Defaulting to files from the classpath: will stage {} files. " + "Enable logging at DEBUG level to see which files will be staged.", flinkOptions.getFilesToStage().size()); LOG.debug("Classpath elements: {}", flinkOptions.getFilesToStage()); } return new FlinkRunner(flinkOptions); }
private Pipeline getPipeline() { PipelineOptions opts = PipelineOptionsFactory.create(); opts.setRunner(DirectRunner.class); return Pipeline.create(opts); }
@Test public void testToString() { PipelineOptions options = PipelineOptionsFactory.as(PipelineOptions.class); options.setRunner(CrashingRunner.class); Pipeline pipeline = Pipeline.create(options); assertEquals("Pipeline#" + pipeline.hashCode(), pipeline.toString()); }
final String outputFilePath = args[1]; final PipelineOptions options = PipelineOptionsFactory.create().as(NemoPipelineOptions.class); options.setRunner(NemoRunner.class); options.setJobName("PerKeyMedian");
@Test public void testPipelineSDKExceptionHandling() { PipelineOptions options = TestPipeline.testingPipelineOptions(); options.setRunner(TestPipelineRunnerThrowingSdkException.class); Pipeline p = Pipeline.create(options); // Check pipeline runner correctly catches SDK errors. try { p.run(); fail("Should have thrown an exception."); } catch (RuntimeException exn) { // Make sure the exception isn't a UserCodeException. assertThat(exn, not(instanceOf(UserCodeException.class))); // Assert that the message is correct. assertThat(exn.getMessage(), containsString("SDK exception")); // RuntimeException should be IllegalStateException. assertThat(exn, instanceOf(IllegalStateException.class)); } }
@Test public void testInstantiation() { PipelineOptions options = PipelineOptionsFactory.create(); options.setRunner(CrashingRunner.class); PipelineRunner<?> runner = PipelineRunner.fromOptions(options); assertTrue(runner instanceof CrashingRunner); }
@Test public void fromOptionsCreatesInstance() { PipelineOptions opts = PipelineOptionsFactory.create(); opts.setRunner(CrashingRunner.class); PipelineRunner<? extends PipelineResult> runner = PipelineRunner.fromOptions(opts); assertTrue("Should have created a CrashingRunner", runner instanceof CrashingRunner); }
PipelineOptions opts = TestPipeline.testingPipelineOptions(); opts.as(DirectOptions.class).setBlockOnRun(false); opts.setRunner(DirectRunner.class);
@Test public void applySucceeds() { PipelineOptions opts = PipelineOptionsFactory.create(); opts.setRunner(CrashingRunner.class); Pipeline p = Pipeline.create(opts); p.apply(Create.of(1, 2, 3)); }
final String outputFilePath = args[1]; final PipelineOptions options = PipelineOptionsFactory.create(); options.setRunner(NemoRunner.class);
options.setRunner(NemoRunner.class); options.setJobName("SimpleSumSQL"); final Pipeline p = Pipeline.create(options);
@Test public void testPipelineUserExceptionHandling() { PipelineOptions options = TestPipeline.testingPipelineOptions(); options.setRunner(TestPipelineRunnerThrowingUserException.class); Pipeline p = Pipeline.create(options); // Check pipeline runner correctly catches user errors. thrown.expect(PipelineExecutionException.class); thrown.expectCause(isA(IllegalStateException.class)); thrown.expectMessage("user code exception"); p.run(); }
@Test public void runThrows() { PipelineOptions opts = PipelineOptionsFactory.create(); opts.setRunner(CrashingRunner.class); Pipeline p = Pipeline.create(opts); p.apply(Create.of(1, 2, 3)); thrown.expect(IllegalArgumentException.class); thrown.expectMessage("Cannot call #run"); thrown.expectMessage(TestPipeline.PROPERTY_BEAM_TEST_PIPELINE_OPTIONS); p.run(); } }
final String outputFilePath = args[1]; final PipelineOptions options = PipelineOptionsFactory.create().as(NemoPipelineOptions.class); options.setRunner(NemoRunner.class); options.setJobName("WordCount");
options.setRunner(NemoRunner.class); options.setJobName("WindowedWordCount");
/** * Tests that a {@link DoFn} that mutates an output with a good equals() fails in the {@link * DirectRunner}. */ @Test public void testMutatingOutputWithEnforcementDisabledSucceeds() throws Exception { PipelineOptions options = PipelineOptionsFactory.create(); options.setRunner(DirectRunner.class); options.as(DirectOptions.class).setEnforceImmutability(false); Pipeline pipeline = Pipeline.create(options); pipeline .apply(Create.of(42)) .apply( ParDo.of( new DoFn<Integer, List<Integer>>() { @ProcessElement public void processElement(ProcessContext c) { List<Integer> outputList = Arrays.asList(1, 2, 3, 4); c.output(outputList); outputList.set(0, 37); c.output(outputList); } })); pipeline.run(); }
options.setRunner(NemoRunner.class); options.setJobName("WindowedBroadcast");