/** * @param p pipeline. * @return source. */ private static PCollection<Long> getSource(final Pipeline p) { return p.apply(GenerateSequence .from(1) .withRate(2, Duration.standardSeconds(1)) .withTimestampFn(num -> new Instant(num * 500))); // 0.5 second between subsequent elements } /**
.from(0) .to(Long.MAX_VALUE) .withRate(10, Duration.millis(100)) //Duration.standardSeconds(2))
.withRate(2, Duration.standardSeconds(1)) .withTimestampFn(num -> new Instant(num * 500))) // 0.5 second between subsequent elements .apply(MapElements.via(new SimpleFunction<Long, KV<String, Long>>() {
p.apply(GenerateSequence.from(0).withRate(1L, Duration.standardSeconds(1)));
@Test @Category(NeedsRunner.class) public void testUnboundedInputRate() { long numElements = 5000; long elemsPerPeriod = 10L; Duration periodLength = Duration.millis(8); PCollection<Long> input = p.apply(GenerateSequence.from(0).to(numElements).withRate(elemsPerPeriod, periodLength)); addCountingAsserts(input, 0, numElements); long expectedRuntimeMillis = (periodLength.getMillis() * numElements) / elemsPerPeriod; Instant startTime = Instant.now(); p.run(); Instant endTime = Instant.now(); assertThat(endTime.isAfter(startTime.plus(expectedRuntimeMillis)), is(true)); }
@Test @Category(NeedsRunner.class) public void testReadWatchForNewFiles() throws IOException, InterruptedException { final Path basePath = tempFolder.getRoot().toPath().resolve("readWatch"); basePath.toFile().mkdir(); p.apply(GenerateSequence.from(0).to(10).withRate(1, Duration.millis(100))) .apply( Window.<Long>into(FixedWindows.of(Duration.millis(150))) .withAllowedLateness(Duration.ZERO) .triggering(Repeatedly.forever(AfterPane.elementCountAtLeast(1))) .discardingFiredPanes()) .apply(ToString.elements()) .apply( TextIO.write() .to(basePath.resolve("data").toString()) .withNumShards(1) .withWindowedWrites()); PCollection<String> lines = p.apply( TextIO.read() .from(basePath.resolve("*").toString()) .watchForNewFiles( Duration.millis(100), Watch.Growth.afterTimeSinceNewOutput(Duration.standardSeconds(3)))); PAssert.that(lines).containsInAnyOrder("0", "1", "2", "3", "4", "5", "6", "7", "8", "9"); p.run(); } }
@Test public void shouldRecognizeAndTranslateStreamingPipeline() { FlinkPipelineOptions options = PipelineOptionsFactory.as(FlinkPipelineOptions.class); options.setRunner(TestFlinkRunner.class); options.setFlinkMaster("[auto]"); FlinkPipelineExecutionEnvironment flinkEnv = new FlinkPipelineExecutionEnvironment(options); Pipeline pipeline = Pipeline.create(); pipeline .apply(GenerateSequence.from(0).withRate(1, Duration.standardSeconds(1))) .apply( ParDo.of( new DoFn<Long, String>() { @ProcessElement public void processElement(ProcessContext c) throws Exception { c.output(Long.toString(c.element())); } })) .apply(Window.into(FixedWindows.of(Duration.standardHours(1)))) .apply(TextIO.write().withNumShards(1).withWindowedWrites().to("/dummy/path")); flinkEnv.translate(pipeline); // no exception should be thrown }
@Test public void shouldRecognizeAndTranslateStreamingPipeline() { FlinkPipelineOptions options = PipelineOptionsFactory.as(FlinkPipelineOptions.class); options.setRunner(TestFlinkRunner.class); options.setFlinkMaster("[auto]"); FlinkPipelineExecutionEnvironment flinkEnv = new FlinkPipelineExecutionEnvironment(options); Pipeline pipeline = Pipeline.create(); pipeline .apply(GenerateSequence.from(0).withRate(1, Duration.standardSeconds(1))) .apply( ParDo.of( new DoFn<Long, String>() { @ProcessElement public void processElement(ProcessContext c) throws Exception { c.output(Long.toString(c.element())); } })) .apply(Window.into(FixedWindows.of(Duration.standardHours(1)))) .apply(TextIO.write().withNumShards(1).withWindowedWrites().to("/dummy/path")); flinkEnv.translate(pipeline); // no exception should be thrown }
.discardingFiredPanes(); readPipeline .apply("Sequence first", GenerateSequence.from(0).to(3).withRate(1, Duration.millis(300))) .apply("Window first", window) .apply("Map first", MapElements.via(mapFn)) .withWindowedWrites()); readPipeline .apply("Sequence second", GenerateSequence.from(3).to(7).withRate(1, Duration.millis(300))) .apply("Window second", window) .apply("Map second", MapElements.via(mapFn))