@Test @Category({ValidatesRunner.class, UsesFailureMessage.class}) public void testAssertionSiteIsCapturedWithMessage() throws Exception { PCollection<Long> vals = pipeline.apply(GenerateSequence.from(0).to(5)); assertThatCollectionIsEmptyWithMessage(vals); Throwable thrown = runExpectingAssertionFailure(pipeline); assertThat(thrown.getMessage(), containsString("Should be empty")); assertThat(thrown.getMessage(), containsString("Expected: iterable over [] in any order")); String stacktrace = Throwables.getStackTraceAsString(thrown); assertThat(stacktrace, containsString("testAssertionSiteIsCapturedWithMessage")); assertThat(stacktrace, containsString("assertThatCollectionIsEmptyWithMessage")); }
@Test @Category({ValidatesRunner.class, UsesFailureMessage.class}) public void testAssertionSiteIsCapturedWithoutMessage() throws Exception { PCollection<Long> vals = pipeline.apply(GenerateSequence.from(0).to(5)); assertThatCollectionIsEmptyWithoutMessage(vals); Throwable thrown = runExpectingAssertionFailure(pipeline); assertThat(thrown.getMessage(), containsString("Expected: iterable over [] in any order")); String stacktrace = Throwables.getStackTraceAsString(thrown); assertThat(stacktrace, containsString("testAssertionSiteIsCapturedWithoutMessage")); assertThat(stacktrace, containsString("assertThatCollectionIsEmptyWithoutMessage")); }
@Test @Category({ValidatesRunner.class, UsesFailureMessage.class}) public void testEmptyFalse() throws Exception { PCollection<Long> vals = pipeline.apply(GenerateSequence.from(0).to(5)); PAssert.that("Vals should have been empty", vals).empty(); Throwable thrown = runExpectingAssertionFailure(pipeline); String message = thrown.getMessage(); assertThat(message, containsString("Vals should have been empty")); assertThat(message, containsString("Expected: iterable over [] in any order")); }
@Test @Category({ValidatesRunner.class, UsesFailureMessage.class}) public void testEmptyFalseDefaultReasonString() throws Exception { PCollection<Long> vals = pipeline.apply(GenerateSequence.from(0).to(5)); PAssert.that(vals).empty(); Throwable thrown = runExpectingAssertionFailure(pipeline); String message = thrown.getMessage(); assertThat(message, containsString("GenerateSequence/Read(BoundedCountingSource).out")); assertThat(message, containsString("Expected: iterable over [] in any order")); }
private static void runProgram(String resultPath) throws Exception { Pipeline p = FlinkTestPipeline.createForBatch(); PCollection<String> result = p .apply(GenerateSequence.from(0).to(10)) .apply(ParDo.of(new DoFn<Long, String>() { @ProcessElement public void processElement(ProcessContext c) throws Exception { c.output(c.element().toString()); } })); result.apply(TextIO.write().to(new URI(resultPath).getPath() + "/part")); p.run(); } }
private static void runProgram(String resultPath) throws Exception { Pipeline p = FlinkTestPipeline.createForBatch(); PCollection<String> result = p.apply(GenerateSequence.from(0).to(10)) .apply( ParDo.of( new DoFn<Long, String>() { @ProcessElement public void processElement(ProcessContext c) throws Exception { c.output(c.element().toString()); } })); result.apply(TextIO.write().to(new URI(resultPath).getPath() + "/part")); p.run(); } }
@Test @Category(NeedsRunner.class) public void testBoundedInputSubrange() { long start = 10; long end = 1000; PCollection<Long> input = p.apply(GenerateSequence.from(start).to(end)); addCountingAsserts(input, start, end); p.run(); }
@Test @Category(NeedsRunner.class) public void testBoundedInput() { long numElements = 1000; PCollection<Long> input = p.apply(GenerateSequence.from(0).to(numElements)); addCountingAsserts(input, 0, numElements); p.run(); }
@Test @Category(NeedsRunner.class) public void testEmptyBoundedInput() { PCollection<Long> input = p.apply(GenerateSequence.from(0).to(0)); PAssert.that(input).empty(); p.run(); }
@Test @Category(NeedsRunner.class) public void testEmptyBoundedInputSubrange() { PCollection<Long> input = p.apply(GenerateSequence.from(42).to(42)); PAssert.that(input).empty(); p.run(); }
@Test public void testBoundedDisplayData() { PTransform<?, ?> input = GenerateSequence.from(0).to(1234); DisplayData displayData = DisplayData.from(input); assertThat(displayData, hasDisplayItem("from", 0)); assertThat(displayData, hasDisplayItem("to", 1234)); }
@Test public void testBoundedDisplayDataSubrange() { PTransform<?, ?> input = GenerateSequence.from(12).to(1234); DisplayData displayData = DisplayData.from(input); assertThat(displayData, hasDisplayItem("from", 12)); assertThat(displayData, hasDisplayItem("to", 1234)); }
@Category({ValidatesRunner.class, UsesAttemptedMetrics.class, UsesCounterMetrics.class}) @Test public void test() throws Exception { pipeline .apply( // Use maxReadTime to force unbounded mode. GenerateSequence.from(0).to(NUM_ELEMENTS).withMaxReadTime(Duration.standardDays(1))) .apply(ParDo.of(new CountingDoFn())); pipeline.run(); // give metrics pusher time to push Thread.sleep((pipeline.getOptions().getMetricsPushPeriod() + 1L) * 1000); assertThat(TestMetricsSink.getCounterValue(), is(NUM_ELEMENTS)); }
@Test public void testUnboundedDisplayData() { Duration maxReadTime = Duration.standardHours(5); SerializableFunction<Long, Instant> timestampFn = input -> Instant.now(); PTransform<?, ?> input = GenerateSequence.from(0).to(1234).withMaxReadTime(maxReadTime).withTimestampFn(timestampFn); DisplayData displayData = DisplayData.from(input); assertThat(displayData, hasDisplayItem("maxReadTime", maxReadTime)); assertThat(displayData, hasDisplayItem("timestampFn", timestampFn.getClass())); }
@Test @Ignore("https://issues.apache.org/jira/browse/BEAM-4088 Test reliably fails.") public void testNoThreadsLeakInPipelineExecution() { pipeline.apply(GenerateSequence.from(0).to(NUM_ELEMENTS)).apply(ParDo.of(new CountingDoFn())); pipeline.run(); }
private static void runProgram(String resultPath) { Pipeline p = FlinkTestPipeline.createForStreaming(); p.apply(GenerateSequence.from(0).to(10)) .apply( ParDo.of( new DoFn<Long, String>() { @ProcessElement public void processElement(ProcessContext c) throws Exception { c.output(c.element().toString()); } })) .apply(TextIO.write().to(resultPath)); p.run(); } }
@Test public void testUnencodableOutputFromBoundedRead() throws Exception { Pipeline p = getPipeline(); p.apply(GenerateSequence.from(0).to(10)).setCoder(new LongNoDecodeCoder()); thrown.expectCause(isA(CoderException.class)); thrown.expectMessage("Cannot decode a long"); p.run(); }
public static void main(String[] args) { Options options = PipelineOptionsFactory.fromArgs(args).withValidation().as(Options.class); Pipeline pipeline = Pipeline.create(options); pipeline .apply("Generate sequence", GenerateSequence.from(0).to(10)) .apply("Produce text lines", ParDo.of(new DeterministicallyConstructTestTextLineFn())) .apply("Produce Avro records", ParDo.of(new DeterministicallyConstructAvroRecordsFn())) .setCoder(AvroCoder.of(SCHEMA)) .apply( "Write Parquet files", FileIO.<GenericRecord>write().via(ParquetIO.sink(SCHEMA)).to(options.getOutput())); pipeline.run(); }
@Test @Category(NeedsRunner.class) public void testUnboundedInputTimestamps() { long numElements = 1000; PCollection<Long> input = p.apply(GenerateSequence.from(0).to(numElements).withTimestampFn(new ValueAsTimestampFn())); addCountingAsserts(input, 0, numElements); PCollection<Long> diffs = input .apply("TimestampDiff", ParDo.of(new ElementValueDiff())) .apply("DistinctTimestamps", Distinct.create()); // This assert also confirms that diffs only has one unique value. PAssert.thatSingleton(diffs).isEqualTo(0L); p.run(); }
private void runWrite() { pipelineWrite .apply("GenSequence", GenerateSequence.from(0).to((long) options.getNumberOfRecords())) .apply("PrepareTestRows", ParDo.of(new TestRow.DeterministicallyConstructTestRowFn())) .apply("MapToEntity", ParDo.of(new CreateScientistFn())) .apply( "WriteToCassandra", CassandraIO.<Scientist>write() .withHosts(options.getCassandraHost()) .withPort(options.getCassandraPort()) .withKeyspace(KEYSPACE) .withEntity(Scientist.class)); pipelineWrite.run().waitUntilFinish(); }