@Test public void testUnboundedDisplayData() { Duration maxReadTime = Duration.standardHours(5); SerializableFunction<Long, Instant> timestampFn = input -> Instant.now(); PTransform<?, ?> input = GenerateSequence.from(0).to(1234).withMaxReadTime(maxReadTime).withTimestampFn(timestampFn); DisplayData displayData = DisplayData.from(input); assertThat(displayData, hasDisplayItem("maxReadTime", maxReadTime)); assertThat(displayData, hasDisplayItem("timestampFn", timestampFn.getClass())); }
/** * @param p pipeline. * @return source. */ private static PCollection<Long> getSource(final Pipeline p) { return p.apply(GenerateSequence .from(1) .withRate(2, Duration.standardSeconds(1)) .withTimestampFn(num -> new Instant(num * 500))); // 0.5 second between subsequent elements } /**
@Test public void nonAdditionalInputsWithOneMainInputSucceeds() { PCollection<Long> input = pipeline.apply(GenerateSequence.from(1L)); AppliedPTransform<PInput, POutput, TestTransform> transform = AppliedPTransform.of( "input-single", Collections.singletonMap(new TupleTag<Long>() {}, input), Collections.emptyMap(), new TestTransform(), pipeline); assertThat(TransformInputs.nonAdditionalInputs(transform), Matchers.containsInAnyOrder(input)); }
@Category({ValidatesRunner.class, UsesAttemptedMetrics.class, UsesCounterMetrics.class}) @Test public void test() throws Exception { pipeline .apply( // Use maxReadTime to force unbounded mode. GenerateSequence.from(0).to(NUM_ELEMENTS).withMaxReadTime(Duration.standardDays(1))) .apply(ParDo.of(new CountingDoFn())); pipeline.run(); // give metrics pusher time to push Thread.sleep((pipeline.getOptions().getMetricsPushPeriod() + 1L) * 1000); assertThat(TestMetricsSink.getCounterValue(), is(NUM_ELEMENTS)); }
@Test @Category(NeedsRunner.class) public void testUnboundedInputTimestamps() { long numElements = 1000; PCollection<Long> input = p.apply(GenerateSequence.from(0).to(numElements).withTimestampFn(new ValueAsTimestampFn())); addCountingAsserts(input, 0, numElements); PCollection<Long> diffs = input .apply("TimestampDiff", ParDo.of(new ElementValueDiff())) .apply("DistinctTimestamps", Distinct.create()); // This assert also confirms that diffs only has one unique value. PAssert.thatSingleton(diffs).isEqualTo(0L); p.run(); }
@Test public void registerPCollectionExistingNameCollision() throws IOException { PCollection<Long> pCollection = pipeline.apply("FirstCount", GenerateSequence.from(0)).setName("foo"); String firstId = components.registerPCollection(pCollection); PCollection<Long> duplicate = pipeline.apply("SecondCount", GenerateSequence.from(0)).setName("foo"); String secondId = components.registerPCollection(duplicate); assertThat(firstId, equalTo("foo")); assertThat(secondId, containsString("foo")); assertThat(secondId, not(equalTo("foo"))); components.toComponents().getPcollectionsOrThrow(firstId); components.toComponents().getPcollectionsOrThrow(secondId); }
Pipeline p = TestPipeline.create(); PCollection<Long> createOne = p.apply("CreateOne", Create.of(1L, 2L, 3L)); PCollection<Long> boundedCount = p.apply("CountBounded", GenerateSequence.from(0).to(23)); PCollection<Long> unboundedCount = p.apply("CountUnbounded", GenerateSequence.from(0)); PCollection<Long> createTwo = p.apply("CreateTwo", Create.of(-1L, -2L)); PCollection<Long> maxReadTimeCount = p.apply( "CountLimited", GenerateSequence.from(0).withMaxReadTime(Duration.standardSeconds(5)));
pipeline .apply( GenerateSequence.from(0) .to(100) .withTimestampFn( new SerializableFunction<Long, Instant>() { @Override
@Test public void shouldRecognizeAndTranslateStreamingPipeline() { FlinkPipelineOptions options = PipelineOptionsFactory.as(FlinkPipelineOptions.class); options.setRunner(TestFlinkRunner.class); options.setFlinkMaster("[auto]"); FlinkPipelineExecutionEnvironment flinkEnv = new FlinkPipelineExecutionEnvironment(options); Pipeline pipeline = Pipeline.create(); pipeline .apply(GenerateSequence.from(0).withRate(1, Duration.standardSeconds(1))) .apply( ParDo.of( new DoFn<Long, String>() { @ProcessElement public void processElement(ProcessContext c) throws Exception { c.output(Long.toString(c.element())); } })) .apply(Window.into(FixedWindows.of(Duration.standardHours(1)))) .apply(TextIO.write().withNumShards(1).withWindowedWrites().to("/dummy/path")); flinkEnv.translate(pipeline); // no exception should be thrown }
.from(1) .withRate(2, Duration.standardSeconds(1)) .withTimestampFn(num -> new Instant(num * 500))) // 0.5 second between subsequent elements .apply(MapElements.via(new SimpleFunction<Long, KV<String, Long>>() { @Override
@Test public void nonAdditionalInputsWithAdditionalInputsSucceeds() { Map<TupleTag<?>, PValue> additionalInputs = new HashMap<>(); additionalInputs.put(new TupleTag<String>() {}, pipeline.apply(Create.of("1, 2", "3"))); additionalInputs.put(new TupleTag<Long>() {}, pipeline.apply(GenerateSequence.from(3L))); Map<TupleTag<?>, PValue> allInputs = new HashMap<>(); PCollection<Integer> mainInts = pipeline.apply("MainInput", Create.of(12, 3)); allInputs.put(new TupleTag<Integer>() {}, mainInts); PCollection<Void> voids = pipeline.apply("VoidInput", Create.empty(VoidCoder.of())); allInputs.put(new TupleTag<Void>() {}, voids); allInputs.putAll(additionalInputs); AppliedPTransform<PInput, POutput, TestTransform> transform = AppliedPTransform.of( "additional", allInputs, Collections.emptyMap(), new TestTransform(additionalInputs), pipeline); assertThat( TransformInputs.nonAdditionalInputs(transform), Matchers.containsInAnyOrder(mainInts, voids)); }
@Test @Category(NeedsRunner.class) public void testUnboundedInputRate() { long numElements = 5000; long elemsPerPeriod = 10L; Duration periodLength = Duration.millis(8); PCollection<Long> input = p.apply(GenerateSequence.from(0).to(numElements).withRate(elemsPerPeriod, periodLength)); addCountingAsserts(input, 0, numElements); long expectedRuntimeMillis = (periodLength.getMillis() * numElements) / elemsPerPeriod; Instant startTime = Instant.now(); p.run(); Instant endTime = Instant.now(); assertThat(endTime.isAfter(startTime.plus(expectedRuntimeMillis)), is(true)); }
@Test @Category({NeedsRunner.class, UsesAttemptedMetrics.class, UsesCounterMetrics.class}) public void testUnboundedSourceMetrics() { long numElements = 1000; // Use withMaxReadTime to force unbounded mode. pipeline.apply( GenerateSequence.from(0).to(numElements).withMaxReadTime(Duration.standardDays(1))); PipelineResult pipelineResult = pipeline.run(); MetricQueryResults metrics = pipelineResult .metrics() .queryMetrics( MetricsFilter.builder() .addNameFilter( MetricNameFilter.named( ELEMENTS_READ.getNamespace(), ELEMENTS_READ.getName())) .build()); assertThat( metrics.getCounters(), hasItem( attemptedMetricsResult( ELEMENTS_READ.getNamespace(), ELEMENTS_READ.getName(), "Read(UnboundedCountingSource)", 1000L))); } }
@Test public void shouldRecognizeAndTranslateStreamingPipeline() { FlinkPipelineOptions options = PipelineOptionsFactory.as(FlinkPipelineOptions.class); options.setRunner(TestFlinkRunner.class); options.setFlinkMaster("[auto]"); FlinkPipelineExecutionEnvironment flinkEnv = new FlinkPipelineExecutionEnvironment(options); Pipeline pipeline = Pipeline.create(); pipeline .apply(GenerateSequence.from(0).withRate(1, Duration.standardSeconds(1))) .apply( ParDo.of( new DoFn<Long, String>() { @ProcessElement public void processElement(ProcessContext c) throws Exception { c.output(Long.toString(c.element())); } })) .apply(Window.into(FixedWindows.of(Duration.standardHours(1)))) .apply(TextIO.write().withNumShards(1).withWindowedWrites().to("/dummy/path")); flinkEnv.translate(pipeline); // no exception should be thrown }
@Test @Category({ValidatesRunner.class, UsesFailureMessage.class}) public void testEmptyFalse() throws Exception { PCollection<Long> vals = pipeline.apply(GenerateSequence.from(0).to(5)); PAssert.that("Vals should have been empty", vals).empty(); Throwable thrown = runExpectingAssertionFailure(pipeline); String message = thrown.getMessage(); assertThat(message, containsString("Vals should have been empty")); assertThat(message, containsString("Expected: iterable over [] in any order")); }
@Test public void nonAdditionalInputsWithOnlyAdditionalInputsThrows() { Map<TupleTag<?>, PValue> additionalInputs = new HashMap<>(); additionalInputs.put(new TupleTag<String>() {}, pipeline.apply(Create.of("1, 2", "3"))); additionalInputs.put(new TupleTag<Long>() {}, pipeline.apply(GenerateSequence.from(3L))); AppliedPTransform<PInput, POutput, TestTransform> transform = AppliedPTransform.of( "additional-only", additionalInputs, Collections.emptyMap(), new TestTransform(additionalInputs), pipeline); thrown.expect(IllegalArgumentException.class); thrown.expectMessage("at least one"); TransformInputs.nonAdditionalInputs(transform); }