/** Convert a Beam {@link PCollection} to a BigQuery {@link TableSchema}. */ public static TableSchema toTableSchema(PCollection<Row> rows) { RowCoder coder = (RowCoder) rows.getCoder(); return toTableSchema(coder.getSchema()); }
@Test public void testToTableSchema_array() { TableSchema schema = toTableSchema(ARRAY_TYPE); assertThat(schema.getFields(), contains(IDS)); }
@Test public void testToTableSchema_flat() { TableSchema schema = toTableSchema(FLAT_TYPE); assertThat(schema.getFields(), containsInAnyOrder(ID, VALUE, NAME, TIMESTAMP, VALID)); }
@Override public POutput buildIOWriter(PCollection<Row> input) { return input.apply( BigQueryIO.<Row>write() .withSchema(BigQueryUtils.toTableSchema(getSchema())) .withFormatFunction(BigQueryUtils.toTableRow()) .to(tableSpec)); }
@Test public void testToTableSchema_row() { TableSchema schema = toTableSchema(ROW_TYPE); assertThat(schema.getFields().size(), equalTo(1)); TableFieldSchema field = schema.getFields().get(0); assertThat(field.getName(), equalTo("row")); assertThat(field.getType(), equalTo(StandardSQLTypeName.STRUCT.toString())); assertThat(field.getMode(), nullValue()); assertThat(field.getFields(), containsInAnyOrder(ID, VALUE, NAME, TIMESTAMP, VALID)); }
@Test public void testToTableSchema_array_row() { TableSchema schema = toTableSchema(ARRAY_ROW_TYPE); assertThat(schema.getFields().size(), equalTo(1)); TableFieldSchema field = schema.getFields().get(0); assertThat(field.getName(), equalTo("rows")); assertThat(field.getType(), equalTo(StandardSQLTypeName.STRUCT.toString())); assertThat(field.getMode(), equalTo(Mode.REPEATED.toString())); assertThat(field.getFields(), containsInAnyOrder(ID, VALUE, NAME, TIMESTAMP, VALID)); }
private Table createTable(Description description) throws IOException, InterruptedException { TableReference tableReference = new TableReference() .setProjectId(pipelineOptions.getProject()) .setDatasetId(pipelineOptions.getTargetDataset()) .setTableId(createRandomizedName(description)); table = new Table() .setTableReference(tableReference) .setSchema(BigQueryUtils.toTableSchema(schema)) .setDescription( "Table created for " + description.getDisplayName() + " by TestBigQueryRule. " + "Should be automatically cleaned up after test completion."); if (datasetService.getTable(tableReference) != null) { throw new IllegalStateException( "Table '" + tableReference + "' already exists. " + "It should have been cleaned up by the test rule."); } datasetService.createTable(table); return table; }