@Test public void testRuntimeValueProviderSubscription() { TestPipeline pipeline = TestPipeline.create(); ValueProvider<String> subscription = pipeline.newProvider("projects/project/subscriptions/subscription"); Read<String> pubsubRead = PubsubIO.readStrings().fromSubscription(subscription); pipeline.apply(pubsubRead); assertThat(pubsubRead.getSubscriptionProvider(), not(nullValue())); assertThat(pubsubRead.getSubscriptionProvider().isAccessible(), is(false)); }
@Test public void testRuntimeOptionsNotCalledInApplyInputTable() { BigQueryIO.Read read = BigQueryIO.read().from(p.newProvider("")).withoutValidation(); // Test that this doesn't throw. DisplayData.from(read); }
@Test public void testRuntimeValueProviderTopic() { TestPipeline pipeline = TestPipeline.create(); ValueProvider<String> topic = pipeline.newProvider("projects/project/topics/topic"); Read<String> pubsubRead = PubsubIO.readStrings().fromTopic(topic); pipeline.apply(pubsubRead); assertThat(pubsubRead.getTopicProvider(), not(nullValue())); assertThat(pubsubRead.getTopicProvider().isAccessible(), is(false)); }
@Test public void testRuntimeOptionsNotCalledInApplyInputQuery() { BigQueryIO.Read read = BigQueryIO.read().fromQuery(p.newProvider("")).withoutValidation(); // Test that this doesn't throw. DisplayData.from(read); }
@Test public void testRuntimeOptionsNotCalledInApplyOutput() { p.enableAbandonedNodeEnforcement(false); BigQueryIO.Write<TableRow> write = BigQueryIO.writeTableRows() .to(p.newProvider("some-table")) .withSchema( ValueProvider.NestedValueProvider.of( p.newProvider("some-schema"), new BigQueryHelpers.JsonSchemaToTableSchema())) .withoutValidation(); p.apply(Create.empty(TableRowJsonCoder.of())).apply(write); // Test that this doesn't throw. DisplayData.from(write); }
private <T extends GenericRecord> void testWriteThenReadGeneratedClass( AvroIO.Write<T> writeTransform, AvroIO.Read<T> readTransform) throws Exception { File outputFile = tmpFolder.newFile("output.avro"); List<T> values = ImmutableList.of( (T) new AvroGeneratedUser("Bob", 256, null), (T) new AvroGeneratedUser("Alice", 128, null), (T) new AvroGeneratedUser("Ted", null, "white")); writePipeline .apply(Create.of(values)) .apply( writeTransform .to(writePipeline.newProvider(outputFile.getAbsolutePath())) .withoutSharding()); writePipeline.run(); PAssert.that( readPipeline.apply( "Read", readTransform.from(readPipeline.newProvider(outputFile.getAbsolutePath())))) .containsInAnyOrder(values); readPipeline.run(); }
@Test @Category(NeedsRunner.class) public void testWriteThenReadJavaClass() throws Throwable { List<GenericClass> values = ImmutableList.of(new GenericClass(3, "hi"), new GenericClass(5, "bar")); File outputFile = tmpFolder.newFile("output.avro"); writePipeline .apply(Create.of(values)) .apply( AvroIO.write(GenericClass.class) .to(writePipeline.newProvider(outputFile.getAbsolutePath())) .withoutSharding()); writePipeline.run(); PAssert.that( readPipeline.apply( "Read", AvroIO.read(GenericClass.class) .from(readPipeline.newProvider(outputFile.getAbsolutePath())))) .containsInAnyOrder(values); readPipeline.run(); }
.apply( AvroIO.<Long, GenericClass>writeCustomType() .to(writePipeline.newProvider(outputFile.getAbsolutePath())) .withFormatFunction(new CreateGenericClass()) .withSchema(ReflectData.get().getSchema(GenericClass.class)) "Read", AvroIO.read(GenericClass.class) .from(readPipeline.newProvider(outputFile.getAbsolutePath()))) .apply( MapElements.via(
"Match existing with provider", FileIO.match() .filepattern(p.newProvider(tmpFolder.getRoot().getAbsolutePath() + "/*")))) .containsInAnyOrder(metadata(firstPath, firstSize), metadata(secondPath, secondSize)); PAssert.that(
@Test @Category(NeedsRunner.class) public void testNewProvider() { ValueProvider<String> foo = pipeline.newProvider("foo"); ValueProvider<String> foobar = ValueProvider.NestedValueProvider.of(foo, input -> input + "bar"); assertFalse(foo.isAccessible()); assertFalse(foobar.isAccessible()); PAssert.that(pipeline.apply("create foo", Create.ofProvider(foo, StringUtf8Coder.of()))) .containsInAnyOrder("foo"); PAssert.that(pipeline.apply("create foobar", Create.ofProvider(foobar, StringUtf8Coder.of()))) .containsInAnyOrder("foobar"); pipeline.run(); } }
@Test @Category(NeedsRunner.class) public void testCreateOfProvider() throws Exception { PAssert.that( p.apply( "Static", Create.ofProvider(StaticValueProvider.of("foo"), StringUtf8Coder.of()))) .containsInAnyOrder("foo"); PAssert.that( p.apply( "Static nested", Create.ofProvider( NestedValueProvider.of(StaticValueProvider.of("foo"), input -> input + "bar"), StringUtf8Coder.of()))) .containsInAnyOrder("foobar"); PAssert.that( p.apply( "Runtime", Create.ofProvider(p.newProvider("runtimeFoo"), StringUtf8Coder.of()))) .containsInAnyOrder("runtimeFoo"); p.run(); }