/** * Creates an instance of this rule. * * <p>Loads GCP configuration from {@link TestPipelineOptions}. */ public static TestBigQuery create(Schema tableSchema) { return new TestBigQuery( TestPipeline.testingPipelineOptions().as(TestBigQueryOptions.class), tableSchema); }
/** * Loads rows from BigQuery into {@link Row Rows} with given {@link Schema}. * * <p>Current implementation only supports flat {@link Row Rows} and target {@link Schema Schemas} * with {@link FieldType#STRING} fields only. */ public List<Row> getFlatJsonRows(Schema rowSchema) { Bigquery bq = newBigQueryClient(pipelineOptions); return bqRowsToBeamRows(getSchema(bq), getTableRows(bq), rowSchema); }
private void initializeBigQuery(Description description) throws IOException, InterruptedException { this.datasetService = new BigQueryServicesImpl().getDatasetService(pipelineOptions); this.table = createTable(description); }
+ "TYPE 'bigquery' \n" + "LOCATION '" + bigQuery.tableSpec() + "'"; sqlEnv.executeDdl(createTableStatement); List<Row> allJsonRows = bigQuery.getFlatJsonRows(SOURCE_SCHEMA);
+ "TYPE 'bigquery' \n" + "LOCATION '" + bigQuery.tableSpec() + "'"; sqlEnv.executeDdl(createTableStatement); .assertThatAllRows(SOURCE_SCHEMA) .eventually( containsInAnyOrder(
private static Bigquery newBigQueryClient(BigQueryOptions options) { return new Bigquery.Builder( Transport.getTransport(), Transport.getJsonFactory(), chainHttpRequestInitializer( options.getGcpCredential(), // Do not log 404. It clutters the output and is possibly even required by the // caller. new RetryHttpRequestInitializer(ImmutableList.of(404)))) .setApplicationName(options.getAppName()) .setGoogleClientRequestInitializer(options.getGoogleApiTrace()) .build(); }
private Table createTable(Description description) throws IOException, InterruptedException { TableReference tableReference = new TableReference() .setProjectId(pipelineOptions.getProject()) .setDatasetId(pipelineOptions.getTargetDataset()) .setTableId(createRandomizedName(description)); table = new Table() .setTableReference(tableReference) .setSchema(BigQueryUtils.toTableSchema(schema)) .setDescription( "Table created for " + description.getDisplayName() + " by TestBigQueryRule. " + "Should be automatically cleaned up after test completion."); if (datasetService.getTable(tableReference) != null) { throw new IllegalStateException( "Table '" + tableReference + "' already exists. " + "It should have been cleaned up by the test rule."); } datasetService.createTable(table); return table; }
+ "TYPE 'bigquery' \n" + "LOCATION '" + bigQueryTestingTypes.tableSpec() + "'"; sqlEnv.executeDdl(createTableStatement); bigQueryTestingTypes.getFlatJsonRows(SOURCE_SCHEMA_TWO), containsInAnyOrder( row(