"WriteBigQuery", BigQueryIO.<String>write() .withTestServices(fakeBqServices) .withMaxFilesPerBundle(5) .withMaxFileSize(10) .withCreateDisposition(BigQueryIO.Write.CreateDisposition.CREATE_IF_NEEDED) .withFormatFunction( user -> { Matcher matcher = userPattern.matcher(user); .to( new StringIntegerDestinations() { @Override .withoutValidation()); p.run();
.to(pipelineOptions.getBigQueryTablePath()) .withCreateDisposition(BigQueryIO.Write.CreateDisposition.CREATE_IF_NEEDED) .withSchema(ReadToRowConverter.getSchema()));
BigQueryIO.Write<FeatureRowExtended> write = BigQueryIO.<FeatureRowExtended>write() .to( new DynamicDestinations<FeatureRowExtended, String>() { public String getDestination(ValueInSingleWindow<FeatureRowExtended> element) { .withFormatFunction(toTableRowDoFn::toTableRow) .withCreateDisposition(CreateDisposition.CREATE_NEVER) .withWriteDisposition(WriteDisposition.WRITE_APPEND) .withMethod(Method.FILE_LOADS); if (!Strings.isNullOrEmpty(bigQueryOptions.tempLocation)) { log.info( + bigQueryOptions.tempLocation); write = write.withCustomGcsTempLocation(StaticValueProvider.of(bigQueryOptions.tempLocation)); write = write .withTriggeringFrequency(triggerFrequency) .withNumFileShards(100);
.apply( BigQueryIO.writeTableRows() .to(tableSpec) .withSchema(tableSchema) .withCreateDisposition(CreateDisposition.CREATE_IF_NEEDED) .withWriteDisposition(WriteDisposition.WRITE_TRUNCATE)); .to(tableSpec) .withSchema(tableSchema) .withFormatFunction( (Quote elem) -> new TableRow().set("source", elem.source).set("quote", elem.quote)) .withCreateDisposition(CreateDisposition.CREATE_IF_NEEDED) .withWriteDisposition(WriteDisposition.WRITE_TRUNCATE)); .to(tableSpec) .withJsonSchema(tableSchemaJson) .withFormatFunction( (Quote elem) -> new TableRow().set("source", elem.source).set("quote", elem.quote)) .withCreateDisposition(CreateDisposition.CREATE_IF_NEEDED) .withWriteDisposition(WriteDisposition.WRITE_TRUNCATE)); .to( new DynamicDestinations<WeatherData, Long>() { @Override .withFormatFunction( (WeatherData elem) ->
"InsertIntoBigQuery", BigQueryIO.writeTableRows() .withJsonSchema(getSchemaFromGCS(options.getJSONPath())) .to(options.getOutputTable()) .withExtendedErrorInfo() .withoutValidation() .withCreateDisposition(CreateDisposition.CREATE_IF_NEEDED) .withWriteDisposition(WriteDisposition.WRITE_APPEND) .withMethod(Method.STREAMING_INSERTS) .withFailedInsertRetryPolicy(InsertRetryPolicy.retryTransientErrors()) .withCustomGcsTempLocation(options.getBigQueryLoadingTemporaryDirectory()));
"WriteSuccessfulRecords", BigQueryIO.writeTableRows() .withoutValidation() .withCreateDisposition(CreateDisposition.CREATE_NEVER) .withWriteDisposition(WriteDisposition.WRITE_APPEND) .to(options.getOutputTableSpec()));
.apply( BigQueryIO.<Integer>write() .to(tableFunction) .withFormatFunction(i -> new TableRow().set("name", "number" + i).set("number", i)) .withCreateDisposition(BigQueryIO.Write.CreateDisposition.CREATE_IF_NEEDED) .withSchemaFromView(schemasView) .withTestServices(fakeBqServices) .withoutValidation()); p.run();
"WriteSuccessfulRecords", BigQueryIO.writeTableRows() .withoutValidation() .withCreateDisposition(CreateDisposition.CREATE_NEVER) .withWriteDisposition(WriteDisposition.WRITE_APPEND) .to(options.getOutputTableSpec()));
.apply( BigQueryIO.writeTableRows() .to(tableRef) .withSchema(FormatForBigquery.getSchema()) .withCreateDisposition(BigQueryIO.Write.CreateDisposition.CREATE_IF_NEEDED) .withWriteDisposition( options.isStreaming() ? BigQueryIO.Write.WriteDisposition.WRITE_APPEND
"Insert into Bigquery", BigQueryIO.writeTableRows() .withSchema( NestedValueProvider.of( options.getJSONPath(), .to(options.getOutputTable()) .withCreateDisposition(CreateDisposition.CREATE_IF_NEEDED) .withWriteDisposition(WriteDisposition.WRITE_TRUNCATE) .withCustomGcsTempLocation(options.getBigQueryLoadingTemporaryDirectory()));
"Write to BigQuery", BigQueryIO.writeTableRows() .withoutValidation() .withCreateDisposition(BigQueryIO.Write.CreateDisposition.CREATE_NEVER) .withWriteDisposition(BigQueryIO.Write.WriteDisposition.WRITE_APPEND) .withCustomGcsTempLocation(options.getBigQueryLoadingTemporaryDirectory()) .to(options.getOutputTable()));
.to(String.format("%1$s.%2$s",bqDataSet, bqTable)) .withSchema(schema) .withWriteDisposition(BigQueryIO.Write.WriteDisposition.WRITE_APPEND));
.apply( BigQueryIO.writeTableRows() .to(tableSpec) .withCreateDisposition(BigQueryIO.Write.CreateDisposition.CREATE_IF_NEEDED) .withMethod(BigQueryIO.Write.Method.STREAMING_INSERTS) .withSchema( new TableSchema() .setFields( new TableFieldSchema().setName("name").setType("STRING"), new TableFieldSchema().setName("number").setType("INTEGER")))) .withFailedInsertRetryPolicy(InsertRetryPolicy.retryTransientErrors()) .withTestServices(fakeBqServices) .withoutValidation() .withExtendedErrorInfo()) .getFailedInsertsWithErr();
case NONE: bigQueryIOPTransform = bigQueryIOPTransform.withCreateDisposition(BigQueryIO.Write.CreateDisposition.CREATE_NEVER); break; case CREATE_IF_NOT_EXISTS: bigQueryIOPTransform = bigQueryIOPTransform .withCreateDisposition(BigQueryIO.Write.CreateDisposition.CREATE_IF_NEEDED) .withSchema(bqSchema); break; case DROP_IF_EXISTS_AND_CREATE: bigQueryIOPTransform = bigQueryIOPTransform .withWriteDisposition(BigQueryIO.Write.WriteDisposition.WRITE_TRUNCATE) .withCreateDisposition(BigQueryIO.Write.CreateDisposition.CREATE_IF_NEEDED) .withSchema(bqSchema); break; case TRUNCATE: bigQueryIOPTransform = bigQueryIOPTransform .withWriteDisposition(BigQueryIO.Write.WriteDisposition.WRITE_TRUNCATE) .withCreateDisposition(BigQueryIO.Write.CreateDisposition.CREATE_NEVER); break; default:
case NONE: bigQueryIOPTransform = bigQueryIOPTransform.withCreateDisposition(BigQueryIO.Write.CreateDisposition.CREATE_NEVER); break; case CREATE_IF_NOT_EXISTS: bigQueryIOPTransform = bigQueryIOPTransform .withCreateDisposition(BigQueryIO.Write.CreateDisposition.CREATE_IF_NEEDED) .withSchema(bqSchema); break; case DROP_IF_EXISTS_AND_CREATE: bigQueryIOPTransform = bigQueryIOPTransform .withWriteDisposition(BigQueryIO.Write.WriteDisposition.WRITE_TRUNCATE) .withCreateDisposition(BigQueryIO.Write.CreateDisposition.CREATE_IF_NEEDED) .withSchema(bqSchema); break; case TRUNCATE: bigQueryIOPTransform = bigQueryIOPTransform .withWriteDisposition(BigQueryIO.Write.WriteDisposition.WRITE_TRUNCATE) .withCreateDisposition(BigQueryIO.Write.CreateDisposition.CREATE_NEVER); break; default:
.to("project-id:dataset-id.table-id") .withCreateDisposition(BigQueryIO.Write.CreateDisposition.CREATE_IF_NEEDED) .withMethod(BigQueryIO.Write.Method.STREAMING_INSERTS) .withSchema( new TableSchema() .setFields( new TableFieldSchema().setName("name").setType("STRING"), new TableFieldSchema().setName("number").setType("INTEGER")))) .withFailedInsertRetryPolicy(InsertRetryPolicy.retryTransientErrors()) .withTestServices(fakeBqServices) .withoutValidation(); .apply("Write2", bqIoWrite.withExtendedErrorInfo()) .getFailedInserts(); fail();
.apply( BigQueryIO.writeTableRows() .to("project-id:dataset-id.table-id") .withCreateDisposition(BigQueryIO.Write.CreateDisposition.CREATE_IF_NEEDED) .withMethod(BigQueryIO.Write.Method.STREAMING_INSERTS) .withSchema( new TableSchema() .setFields( new TableFieldSchema().setName("name").setType("STRING"), new TableFieldSchema().setName("number").setType("INTEGER")))) .withFailedInsertRetryPolicy(InsertRetryPolicy.retryTransientErrors()) .withTestServices(fakeBqServices) .withoutValidation()) .getFailedInserts();
"WriteBigQuery", BigQueryIO.writeTableRows() .withoutValidation() .withCreateDisposition(BigQueryIO.Write.CreateDisposition.CREATE_NEVER) .to(options.getOutputTableSpec()) .withCreateDisposition(CreateDisposition.CREATE_IF_NEEDED) .withWriteDisposition(WriteDisposition.WRITE_TRUNCATE) .withCustomGcsTempLocation(options.getBigQueryLoadingTemporaryDirectory()));
.apply( BigQueryIO.writeTableRows() .to("project-id:dataset-id.table-id") .withSchema( new TableSchema() .setFields( ImmutableList.of( new TableFieldSchema().setName("number").setType("INTEGER")))) .withTestServices(fakeBqServices) .withTriggeringFrequency(Duration.standardSeconds(30)) .withNumFileShards(2) .withMethod(BigQueryIO.Write.Method.FILE_LOADS) .withoutValidation()); p.run();
.apply(ParDo.of(new StringToRowConverter())) .apply( BigQueryIO.writeTableRows().to(tableSpec).withSchema(StringToRowConverter.getSchema()));