.to(String.format("%1$s.%2$s",bqDataSet, bqTable)) .withSchema(schema) .withWriteDisposition(BigQueryIO.Write.WriteDisposition.WRITE_APPEND));
private BigQueryIO.Write setWriteOperation(BigQueryIO.Write bigQueryIOPTransform) { if (properties.tableOperation.getValue() == BigQueryOutputProperties.TableOperation.NONE || properties.tableOperation .getValue() == BigQueryOutputProperties.TableOperation.CREATE_IF_NOT_EXISTS) { switch (properties.writeOperation.getValue()) { case APPEND: bigQueryIOPTransform = bigQueryIOPTransform.withWriteDisposition(BigQueryIO.Write.WriteDisposition.WRITE_APPEND); break; case WRITE_TO_EMPTY: bigQueryIOPTransform = bigQueryIOPTransform.withWriteDisposition(BigQueryIO.Write.WriteDisposition.WRITE_EMPTY); break; default: throw new RuntimeException("To be implemented: " + properties.writeOperation.getValue()); } } else { if (properties.writeOperation.getValue() != null) { LOG.info("Write operation " + properties.writeOperation.getValue() + " be ignored when Table operation is " + properties.tableOperation.getValue()); } } return bigQueryIOPTransform; }
private BigQueryIO.Write setWriteOperation(BigQueryIO.Write bigQueryIOPTransform) { if (properties.tableOperation.getValue() == BigQueryOutputProperties.TableOperation.NONE || properties.tableOperation .getValue() == BigQueryOutputProperties.TableOperation.CREATE_IF_NOT_EXISTS) { switch (properties.writeOperation.getValue()) { case APPEND: bigQueryIOPTransform = bigQueryIOPTransform.withWriteDisposition(BigQueryIO.Write.WriteDisposition.WRITE_APPEND); break; case WRITE_TO_EMPTY: bigQueryIOPTransform = bigQueryIOPTransform.withWriteDisposition(BigQueryIO.Write.WriteDisposition.WRITE_EMPTY); break; default: throw new RuntimeException("To be implemented: " + properties.writeOperation.getValue()); } } else { if (properties.writeOperation.getValue() != null) { LOG.info("Write operation " + properties.writeOperation.getValue() + " be ignored when Table operation is " + properties.tableOperation.getValue()); } } return bigQueryIOPTransform; }
.withFormatFunction(toTableRowDoFn::toTableRow) .withCreateDisposition(CreateDisposition.CREATE_NEVER) .withWriteDisposition(WriteDisposition.WRITE_APPEND) .withMethod(Method.FILE_LOADS); if (!Strings.isNullOrEmpty(bigQueryOptions.tempLocation)) {
.to(options.getOutputTable()) .withCreateDisposition(CreateDisposition.CREATE_IF_NEEDED) .withWriteDisposition(WriteDisposition.WRITE_TRUNCATE) .withCustomGcsTempLocation(options.getBigQueryLoadingTemporaryDirectory()));
.withoutValidation() .withCreateDisposition(BigQueryIO.Write.CreateDisposition.CREATE_NEVER) .withWriteDisposition(BigQueryIO.Write.WriteDisposition.WRITE_APPEND) .withCustomGcsTempLocation(options.getBigQueryLoadingTemporaryDirectory()) .to(options.getOutputTable()));
case DROP_IF_EXISTS_AND_CREATE: bigQueryIOPTransform = bigQueryIOPTransform .withWriteDisposition(BigQueryIO.Write.WriteDisposition.WRITE_TRUNCATE) .withCreateDisposition(BigQueryIO.Write.CreateDisposition.CREATE_IF_NEEDED) .withSchema(bqSchema); case TRUNCATE: bigQueryIOPTransform = bigQueryIOPTransform .withWriteDisposition(BigQueryIO.Write.WriteDisposition.WRITE_TRUNCATE) .withCreateDisposition(BigQueryIO.Write.CreateDisposition.CREATE_NEVER); break;
case DROP_IF_EXISTS_AND_CREATE: bigQueryIOPTransform = bigQueryIOPTransform .withWriteDisposition(BigQueryIO.Write.WriteDisposition.WRITE_TRUNCATE) .withCreateDisposition(BigQueryIO.Write.CreateDisposition.CREATE_IF_NEEDED) .withSchema(bqSchema); case TRUNCATE: bigQueryIOPTransform = bigQueryIOPTransform .withWriteDisposition(BigQueryIO.Write.WriteDisposition.WRITE_TRUNCATE) .withCreateDisposition(BigQueryIO.Write.CreateDisposition.CREATE_NEVER); break;
.to(options.getOutputTableSpec()) .withCreateDisposition(CreateDisposition.CREATE_IF_NEEDED) .withWriteDisposition(WriteDisposition.WRITE_TRUNCATE) .withCustomGcsTempLocation(options.getBigQueryLoadingTemporaryDirectory()));
@Test public void testBuildWriteDisplayData() { String tableSpec = "project:dataset.table"; TableSchema schema = new TableSchema().set("col1", "type1").set("col2", "type2"); final String tblDescription = "foo bar table"; BigQueryIO.Write<TableRow> write = BigQueryIO.writeTableRows() .to(tableSpec) .withSchema(schema) .withCreateDisposition(BigQueryIO.Write.CreateDisposition.CREATE_IF_NEEDED) .withWriteDisposition(BigQueryIO.Write.WriteDisposition.WRITE_APPEND) .withTableDescription(tblDescription) .withoutValidation(); DisplayData displayData = DisplayData.from(write); assertThat(displayData, hasDisplayItem("table")); assertThat(displayData, hasDisplayItem("schema")); assertThat( displayData, hasDisplayItem( "createDisposition", BigQueryIO.Write.CreateDisposition.CREATE_IF_NEEDED.toString())); assertThat( displayData, hasDisplayItem( "writeDisposition", BigQueryIO.Write.WriteDisposition.WRITE_APPEND.toString())); assertThat(displayData, hasDisplayItem("tableDescription", tblDescription)); assertThat(displayData, hasDisplayItem("validation", false)); }
public static void main(String[] args) throws Exception { Options options = PipelineOptionsFactory.fromArgs(args).withValidation().as(Options.class); Pipeline p = Pipeline.create(options); // Build the table schema for the output table. List<TableFieldSchema> fields = new ArrayList<>(); fields.add(new TableFieldSchema().setName("word").setType("STRING")); fields.add(new TableFieldSchema().setName("all_plays").setType("STRING")); TableSchema schema = new TableSchema().setFields(fields); p.apply(BigQueryIO.readTableRows().from(options.getInput())) .apply(new PlaysForWord()) .apply( BigQueryIO.writeTableRows() .to(options.getOutput()) .withSchema(schema) .withCreateDisposition(BigQueryIO.Write.CreateDisposition.CREATE_IF_NEEDED) .withWriteDisposition(BigQueryIO.Write.WriteDisposition.WRITE_TRUNCATE)); p.run().waitUntilFinish(); } }
public static void main(String[] args) throws Exception { Options options = PipelineOptionsFactory.fromArgs(args).withValidation().as(Options.class); Pipeline p = Pipeline.create(options); // Build the table schema for the output table. List<TableFieldSchema> fields = new ArrayList<>(); fields.add(new TableFieldSchema().setName("month").setType("INTEGER")); fields.add(new TableFieldSchema().setName("max_mean_temp").setType("FLOAT")); TableSchema schema = new TableSchema().setFields(fields); p.apply(BigQueryIO.readTableRows().from(options.getInput())) .apply(new MaxMeanTemp()) .apply( BigQueryIO.writeTableRows() .to(options.getOutput()) .withSchema(schema) .withCreateDisposition(BigQueryIO.Write.CreateDisposition.CREATE_IF_NEEDED) .withWriteDisposition(BigQueryIO.Write.WriteDisposition.WRITE_TRUNCATE)); p.run().waitUntilFinish(); } }
@Test public void testWriteEmptyPCollection() throws Exception { TableSchema schema = new TableSchema() .setFields( ImmutableList.of(new TableFieldSchema().setName("number").setType("INTEGER"))); p.apply(Create.empty(TableRowJsonCoder.of())) .apply( BigQueryIO.writeTableRows() .to("project-id:dataset-id.table-id") .withTestServices(fakeBqServices) .withWriteDisposition(BigQueryIO.Write.WriteDisposition.WRITE_APPEND) .withCreateDisposition(BigQueryIO.Write.CreateDisposition.CREATE_IF_NEEDED) .withSchema(schema) .withoutValidation()); p.run(); checkNotNull( fakeDatasetService.getTable( BigQueryHelpers.parseTableSpec("project-id:dataset-id.table-id"))); }
static void runBigQueryTornadoes(Options options) { Pipeline p = Pipeline.create(options); // Build the table schema for the output table. List<TableFieldSchema> fields = new ArrayList<>(); fields.add(new TableFieldSchema().setName("month").setType("INTEGER")); fields.add(new TableFieldSchema().setName("tornado_count").setType("INTEGER")); TableSchema schema = new TableSchema().setFields(fields); p.apply(BigQueryIO.readTableRows().from(options.getInput())) .apply(new CountTornadoes()) .apply( BigQueryIO.writeTableRows() .to(options.getOutput()) .withSchema(schema) .withCreateDisposition(BigQueryIO.Write.CreateDisposition.CREATE_IF_NEEDED) .withWriteDisposition(BigQueryIO.Write.WriteDisposition.WRITE_TRUNCATE)); p.run().waitUntilFinish(); }
public static void main(String[] args) throws Exception { Options options = PipelineOptionsFactory.fromArgs(args).withValidation().as(Options.class); Pipeline p = Pipeline.create(options); TableSchema schema = buildWeatherSchemaProjection(); p.apply(BigQueryIO.readTableRows().from(options.getInput())) .apply(ParDo.of(new ProjectionFn())) .apply(new BelowGlobalMean(options.getMonthFilter())) .apply( BigQueryIO.writeTableRows() .to(options.getOutput()) .withSchema(schema) .withCreateDisposition(BigQueryIO.Write.CreateDisposition.CREATE_IF_NEEDED) .withWriteDisposition(BigQueryIO.Write.WriteDisposition.WRITE_TRUNCATE)); p.run().waitUntilFinish(); } }
@Override public WriteResult expand( PCollection<FailsafeElement<KV<String, String>, String>> failedRecords) { return failedRecords .apply("FailedRecordToTableRow", ParDo.of(new FailedMessageToTableRowFn())) .apply( "WriteFailedRecordsToBigQuery", BigQueryIO.writeTableRows() .to(getErrorRecordsTable()) .withJsonSchema(getErrorRecordsTableSchema()) .withCreateDisposition(CreateDisposition.CREATE_IF_NEEDED) .withWriteDisposition(WriteDisposition.WRITE_APPEND)); } }
@Override public PDone expand(PCollection<T> teamAndScore) { teamAndScore .apply("ConvertToRow", ParDo.of(new BuildRowFn())) .apply( BigQueryIO.writeTableRows() .to(getTable(projectId, datasetId, tableName)) .withSchema(getSchema()) .withCreateDisposition(CreateDisposition.CREATE_IF_NEEDED) .withWriteDisposition(WriteDisposition.WRITE_APPEND)); return PDone.in(teamAndScore.getPipeline()); } }
@Override public WriteResult expand(PCollection<FailsafeElement<String, String>> failedRecords) { return failedRecords .apply("FailedRecordToTableRow", ParDo.of(new FailedStringToTableRowFn())) .apply( "WriteFailedRecordsToBigQuery", BigQueryIO.writeTableRows() .to(getErrorRecordsTable()) .withJsonSchema(getErrorRecordsTableSchema()) .withCreateDisposition(CreateDisposition.CREATE_IF_NEEDED) .withWriteDisposition(WriteDisposition.WRITE_APPEND)); }
@Override public PDone expand(PCollection<InputT> teamAndScore) { teamAndScore .apply("ConvertToRow", ParDo.of(new BuildRowFn())) .apply( BigQueryIO.writeTableRows() .to(getTable(projectId, datasetId, tableName)) .withSchema(getSchema()) .withCreateDisposition(CreateDisposition.CREATE_IF_NEEDED) .withWriteDisposition(WriteDisposition.WRITE_APPEND)); return PDone.in(teamAndScore.getPipeline()); }
@Override public WriteResult expand(PCollection<FailsafeElement<PubsubMessage, String>> failedRecords) { return failedRecords .apply("FailedRecordToTableRow", ParDo.of(new FailedPubsubMessageToTableRowFn())) .apply( "WriteFailedRecordsToBigQuery", BigQueryIO.writeTableRows() .to(getErrorRecordsTable()) .withJsonSchema(getErrorRecordsTableSchema()) .withCreateDisposition(CreateDisposition.CREATE_IF_NEEDED) .withWriteDisposition(WriteDisposition.WRITE_APPEND)); }