@Override public void populateDisplayData(DisplayData.Builder builder) { super.populateDisplayData(builder); builder.addIfNotNull( DisplayData.item("table", getJsonTableRef()).withLabel("Table Reference")); if (getJsonSchema() != null) { builder.addIfNotNull(DisplayData.item("schema", getJsonSchema()).withLabel("Table Schema")); } else { builder.add(DisplayData.item("schema", "Custom Schema Function").withLabel("Table Schema")); } if (getTableFunction() != null) { builder.add( DisplayData.item("tableFn", getTableFunction().getClass()) .withLabel("Table Reference Function")); } builder .add( DisplayData.item("createDisposition", getCreateDisposition().toString()) .withLabel("Table CreateDisposition")) .add( DisplayData.item("writeDisposition", getWriteDisposition().toString()) .withLabel("Table WriteDisposition")) .addIfNotDefault( DisplayData.item("validation", getValidate()).withLabel("Validation Enabled"), true) .addIfNotNull( DisplayData.item("tableDescription", getTableDescription()) .withLabel("Table Description")); }
@Test public void testWriteBuilderMethods() { BigQueryIO.Write<TableRow> write = BigQueryIO.writeTableRows().to("foo.com:project:somedataset.sometable"); assertEquals("foo.com:project", write.getTable().get().getProjectId()); assertEquals("somedataset", write.getTable().get().getDatasetId()); assertEquals("sometable", write.getTable().get().getTableId()); assertNull(write.getJsonSchema()); assertNull(write.getSchemaFromView()); assertEquals(BigQueryIO.Write.CreateDisposition.CREATE_IF_NEEDED, write.getCreateDisposition()); assertEquals(BigQueryIO.Write.WriteDisposition.WRITE_EMPTY, write.getWriteDisposition()); assertEquals(null, write.getTableDescription()); assertEquals(true, write.getValidate()); assertFalse(write.withoutValidation().getValidate()); TableSchema schema = new TableSchema(); assertEquals( schema, BigQueryHelpers.fromJsonString( write.withSchema(schema).getJsonSchema().get(), TableSchema.class)); }
@Override public void validate(PipelineOptions pipelineOptions) { BigQueryOptions options = pipelineOptions.as(BigQueryOptions.class); // The user specified a table. if (getJsonTableRef() != null && getJsonTableRef().isAccessible() && getValidate()) { TableReference table = getTableWithDefaultProject(options).get(); DatasetService datasetService = getBigQueryServices().getDatasetService(options); // Check for destination table presence and emptiness for early failure notification. // Note that a presence check can fail when the table or dataset is created by an earlier // stage of the pipeline. For these cases the #withoutValidation method can be used to // disable the check. BigQueryHelpers.verifyDatasetPresence(datasetService, table); if (getCreateDisposition() == BigQueryIO.Write.CreateDisposition.CREATE_NEVER) { BigQueryHelpers.verifyTablePresence(datasetService, table); } if (getWriteDisposition() == BigQueryIO.Write.WriteDisposition.WRITE_EMPTY) { BigQueryHelpers.verifyTableNotExistOrEmpty(datasetService, table); } } }