@Test public void testQuerySourcePrimitiveDisplayData() throws IOException, InterruptedException { DisplayDataEvaluator evaluator = DisplayDataEvaluator.create(); BigQueryIO.Read read = BigQueryIO.read() .fromQuery("foobar") .withTestServices( new FakeBigQueryServices() .withDatasetService(new FakeDatasetService()) .withJobService(new FakeJobService())) .withoutValidation(); Set<DisplayData> displayData = evaluator.displayDataForPrimitiveSourceTransforms(read); assertThat( "BigQueryIO.Read should include the query in its primitive display data", displayData, hasItem(hasDisplayItem("query"))); }
.apply(BigQueryIO.read().from("ReadSourceTable").fromQuery(options.getBqQuery()) .usingStandardSql()) .apply(ParDo.of(MUTATION_TRANSFORM))
private void runBigQueryToTablePipeline() { Pipeline p = Pipeline.create(options); BigQueryIO.Read bigQueryRead = BigQueryIO.read().fromQuery(options.getQuery()); if (options.getUsingStandardSql()) { bigQueryRead = bigQueryRead.usingStandardSql(); } PCollection<TableRow> input = p.apply(bigQueryRead); if (options.getReshuffle()) { input = input .apply(WithKeys.<Void, TableRow>of((Void) null)) .setCoder(KvCoder.of(VoidCoder.of(), TableRowJsonCoder.of())) .apply(Reshuffle.<Void, TableRow>of()) .apply(Values.<TableRow>create()); } input.apply( BigQueryIO.writeTableRows() .to(options.getOutput()) .withSchema(options.getOutputSchema()) .withCreateDisposition(BigQueryIO.Write.CreateDisposition.CREATE_IF_NEEDED)); p.run().waitUntilFinish(); }
@Test public void testBuildSourceDisplayDataQuery() { BigQueryIO.Read read = BigQueryIO.read() .fromQuery("myQuery") .withoutResultFlattening() .usingStandardSql() .withoutValidation(); DisplayData displayData = DisplayData.from(read); assertThat(displayData, hasDisplayItem("query", "myQuery")); assertThat(displayData, hasDisplayItem("flattenResults", false)); assertThat(displayData, hasDisplayItem("useLegacySql", false)); assertThat(displayData, hasDisplayItem("validation", false)); }
@Test public void testRuntimeOptionsNotCalledInApplyInputQuery() { BigQueryIO.Read read = BigQueryIO.read().fromQuery(p.newProvider("")).withoutValidation(); // Test that this doesn't throw. DisplayData.from(read); }
@Test public void testBuildQueryBasedSource() { BigQueryIO.Read read = BigQueryIO.read().fromQuery("foo_query"); checkReadQueryObject(read, "foo_query"); }
@Test public void testBuildQueryBasedSourceWithoutValidation() { // This test just checks that using withoutValidation will not trigger object // construction errors. BigQueryIO.Read read = BigQueryIO.read().fromQuery("some_query").withoutValidation(); checkReadQueryObjectWithValidate(read, "some_query", false); }