@Test public void testTableSourcePrimitiveDisplayData() throws IOException, InterruptedException { DisplayDataEvaluator evaluator = DisplayDataEvaluator.create(); BigQueryIO.Read read = BigQueryIO.read() .from("project:dataset.tableId") .withTestServices( new FakeBigQueryServices() .withDatasetService(new FakeDatasetService()) .withJobService(new FakeJobService())) .withoutValidation(); Set<DisplayData> displayData = evaluator.displayDataForPrimitiveSourceTransforms(read); assertThat( "BigQueryIO.Read should include the table spec in its primitive display data", displayData, hasItem(hasDisplayItem("table"))); }
BigQueryIO.Read read = BigQueryIO.read() .from("non-executing-project:somedataset.sometable") .withTestServices(fakeBqServices) .withoutValidation();
p.apply(BigQueryIO.read().from(tableRef).withTestServices(fakeBqServices)) .apply( ParDo.of(
.apply(BigQueryIO.read().from("ReadSourceTable").fromQuery(options.getBqQuery()) .usingStandardSql()) .apply(ParDo.of(MUTATION_TRANSFORM))
public static void main(String[] args) { Options options = getOptions(args); String projectId = options.getProject(); String datasetId = options.getInputBigQueryDataset(); String tableId = options.getInputBigQueryTable(); String namespace = options.getOutputDatastoreNamespace(); String kind = options.getOutputDatastoreKind(); String keyColumn = options.getKeyColumn(); LinkedHashMap<String, String> parents = parseParentPaths(options.getParentPaths()); List<String> indexedColumns = parseIndexedColumns(options.getIndexedColumns()); // Input TableReference tableRef = new TableReference().setDatasetId(datasetId).setTableId(tableId); BigQueryIO.Read reader = BigQueryIO.read().from(tableRef); // Output DatastoreV1.Write writer = DatastoreIO.v1().write().withProjectId(projectId); // Build and run pipeline TableRow2EntityFn fn = new TableRow2EntityFn(projectId, namespace, parents, kind, keyColumn, indexedColumns); Pipeline pipeline = Pipeline.create(options); pipeline .apply(reader) .apply(ParDo.of(fn)) .apply(writer); pipeline.run(); }
@Test public void testBigQueryIOGetName() { assertEquals("BigQueryIO.Read", BigQueryIO.read().from("somedataset.sometable").getName()); }
@Test public void testBuildSourceWithTableAndFlattenWithoutValidation() { thrown.expect(IllegalArgumentException.class); thrown.expectMessage( "Invalid BigQueryIO.Read: Specifies a table with a result flattening preference," + " which only applies to queries"); p.apply( BigQueryIO.read() .from("foo.com:project:somedataset.sometable") .withoutValidation() .withoutResultFlattening()); p.run(); }
@Test public void testBuildSourceDisplayDataTable() { String tableSpec = "project:dataset.tableid"; BigQueryIO.Read read = BigQueryIO.read() .from(tableSpec) .withoutResultFlattening() .usingStandardSql() .withoutValidation(); DisplayData displayData = DisplayData.from(read); assertThat(displayData, hasDisplayItem("table", tableSpec)); assertThat(displayData, hasDisplayItem("flattenResults", false)); assertThat(displayData, hasDisplayItem("useLegacySql", false)); assertThat(displayData, hasDisplayItem("validation", false)); }
@Test public void testBuildSourceWithTableAndSqlDialect() { thrown.expect(IllegalArgumentException.class); thrown.expectMessage( "Invalid BigQueryIO.Read: Specifies a table with a SQL dialect preference," + " which only applies to queries"); p.apply(BigQueryIO.read().from("foo.com:project:somedataset.sometable").usingStandardSql()); p.run(); }
@Test public void testBuildTableBasedSourceWithDefaultProject() { BigQueryIO.Read read = BigQueryIO.read().from("somedataset.sometable"); checkReadTableObject(read, null, "somedataset", "sometable"); }
private void runBigQueryIOReadPipeline() { Pipeline p = Pipeline.create(options); PCollection<Long> count = p.apply("Read", BigQueryIO.read().from(options.getInputTable())) .apply("Count", Count.globally()); PAssert.thatSingleton(count).isEqualTo(options.getNumRecords()); p.run().waitUntilFinish(); }
@Test public void testRuntimeOptionsNotCalledInApplyInputTable() { BigQueryIO.Read read = BigQueryIO.read().from(p.newProvider("")).withoutValidation(); // Test that this doesn't throw. DisplayData.from(read); }
@Test public void testBuildSourceWithTableAndFlatten() { thrown.expect(IllegalArgumentException.class); thrown.expectMessage( "Invalid BigQueryIO.Read: Specifies a table with a result flattening preference," + " which only applies to queries"); p.apply( "ReadMyTable", BigQueryIO.read().from("foo.com:project:somedataset.sometable").withoutResultFlattening()); p.run(); }
@Test public void testBuildSourceWithTableReference() { TableReference table = new TableReference() .setProjectId("foo.com:project") .setDatasetId("somedataset") .setTableId("sometable"); BigQueryIO.Read read = BigQueryIO.read().from(table); checkReadTableObject(read, "foo.com:project", "somedataset", "sometable"); }
@Test public void testBuildTableBasedSourceWithoutValidation() { // This test just checks that using withoutValidation will not trigger object // construction errors. BigQueryIO.Read read = BigQueryIO.read().from("foo.com:project:somedataset.sometable").withoutValidation(); checkReadTableObjectWithValidate(read, "foo.com:project", "somedataset", "sometable", false); }
@Test public void testBuildTableBasedSource() { BigQueryIO.Read read = BigQueryIO.read().from("foo.com:project:somedataset.sometable"); checkReadTableObject(read, "foo.com:project", "somedataset", "sometable"); }