@Override JobInfo parse(String... args) throws Exception { if (args.length >= 4) { String dataset = args[0]; String table = args[1]; String format = args[2]; TableId tableId = TableId.of(dataset, table); ExtractJobConfiguration configuration = ExtractJobConfiguration.of( tableId, Arrays.asList(args).subList(3, args.length), format); return JobInfo.of(configuration); } throw new IllegalArgumentException("Missing required arguments."); }
@Override JobInfo parse(String... args) throws Exception { if (args.length >= 4) { String dataset = args[0]; String table = args[1]; String format = args[2]; TableId tableId = TableId.of(dataset, table); LoadJobConfiguration configuration = LoadJobConfiguration.of( tableId, Arrays.asList(args).subList(3, args.length), FormatOptions.of(format)); return JobInfo.of(configuration); } throw new IllegalArgumentException("Missing required arguments."); }
/** Example of creating a query job. */ // [TARGET create(JobInfo, JobOption...)] // [VARIABLE "SELECT field FROM my_dataset_name.my_table_name"] public Job createJob(String query) { // [START ] Job job = null; JobConfiguration jobConfiguration = QueryJobConfiguration.of(query); JobInfo jobInfo = JobInfo.of(jobConfiguration); try { job = bigquery.create(jobInfo); } catch (BigQueryException e) { // the job was not created } // [END ] return job; }
@Override JobInfo parse(String... args) throws Exception { String message; if (args.length == 4) { TableId sourceTableId = TableId.of(args[0], args[1]); TableId destinationTableId = TableId.of(args[2], args[3]); return JobInfo.of(CopyJobConfiguration.of(destinationTableId, sourceTableId)); } else if (args.length < 3) { message = "Missing required source or destination table."; } else { message = "Too many arguments."; } throw new IllegalArgumentException(message); }
@Override public TableResult query(QueryJobConfiguration configuration, JobOption... options) throws InterruptedException, JobException { Job.checkNotDryRun(configuration, "query"); return create(JobInfo.of(configuration), options).getQueryResults(); }
@Override public TableResult query(QueryJobConfiguration configuration, JobId jobId, JobOption... options) throws InterruptedException, JobException { Job.checkNotDryRun(configuration, "query"); return create(JobInfo.of(jobId, configuration), options).getQueryResults(); }
private void generateTableWithDdl(String datasetId, String tableId) throws InterruptedException { String sql = String.format( "CREATE TABLE %s.%s " + "AS " + "SELECT " + "2000 + CAST(18 * RAND() as INT64) AS year, " + "IF(RAND() > 0.5,\"foo\",\"bar\") AS token " + "FROM " + "UNNEST(GENERATE_ARRAY(0,5,1)) AS r", datasetId, tableId); Job job = bigquery.create(JobInfo.of(QueryJobConfiguration.newBuilder(sql).build())); job.waitFor(); }
/** Example of copying multiple tables to a destination. */ public void copyTables(String datasetId, String destinationTableId) throws InterruptedException { generateTableWithDdl(datasetId, "table1"); generateTableWithDdl(datasetId, "table2"); // [START bigquery_copy_table_multiple_source] TableId destinationTable = TableId.of(datasetId, destinationTableId); CopyJobConfiguration configuration = CopyJobConfiguration.newBuilder( destinationTable, Arrays.asList(TableId.of(datasetId, "table1"), TableId.of(datasetId, "table2"))) .build(); // Copy the tables. Job job = bigquery.create(JobInfo.of(configuration)); job = job.waitFor(); // Check the table StandardTableDefinition table = bigquery.getTable(destinationTable).getDefinition(); System.out.println("State: " + job.getStatus().getState()); System.out.printf("Copied %d rows.\n", table.getNumRows()); // [END bigquery_copy_table_multiple_source] }
/** Example of loading a parquet file from GCS to a table. */ public void loadTableGcsParquet(String datasetName) throws InterruptedException { // [START bigquery_load_table_gcs_parquet] String sourceUri = "gs://cloud-samples-data/bigquery/us-states/us-states.parquet"; TableId tableId = TableId.of(datasetName, "us_states"); LoadJobConfiguration configuration = LoadJobConfiguration.builder(tableId, sourceUri) .setFormatOptions(FormatOptions.parquet()) .build(); // Load the table Job loadJob = bigquery.create(JobInfo.of(configuration)); loadJob = loadJob.waitFor(); // Check the table StandardTableDefinition destinationTable = bigquery.getTable(tableId).getDefinition(); System.out.println("State: " + loadJob.getStatus().getState()); System.out.printf("Loaded %d rows.\n", destinationTable.getNumRows()); // [END bigquery_load_table_gcs_parquet] }
@Test public void testToBuilderIncomplete() { JobInfo job = JobInfo.of(COPY_CONFIGURATION); compareJobInfo(job, job.toBuilder().build()); }
@Test public void testOf() { JobInfo job = JobInfo.of(COPY_CONFIGURATION); assertEquals(COPY_CONFIGURATION, job.getConfiguration()); job = JobInfo.of(EXTRACT_CONFIGURATION); assertEquals(EXTRACT_CONFIGURATION, job.getConfiguration()); job = JobInfo.of(LOAD_CONFIGURATION); assertEquals(LOAD_CONFIGURATION, job.getConfiguration()); job = JobInfo.of(QUERY_CONFIGURATION); assertEquals(QUERY_CONFIGURATION, job.getConfiguration()); job = JobInfo.of(JOB_ID, COPY_CONFIGURATION); assertEquals(JOB_ID, job.getJobId()); assertEquals(COPY_CONFIGURATION, job.getConfiguration()); job = JobInfo.of(JOB_ID, EXTRACT_CONFIGURATION); assertEquals(JOB_ID, job.getJobId()); assertEquals(EXTRACT_CONFIGURATION, job.getConfiguration()); job = JobInfo.of(JOB_ID, LOAD_CONFIGURATION); assertEquals(JOB_ID, job.getJobId()); assertEquals(LOAD_CONFIGURATION, job.getConfiguration()); job = JobInfo.of(JOB_ID, QUERY_CONFIGURATION); assertEquals(JOB_ID, job.getJobId()); assertEquals(QUERY_CONFIGURATION, job.getConfiguration()); }
@Test public void testCreateJobWithSelectedFields() { Capture<Map<BigQueryRpc.Option, Object>> capturedOptions = Capture.newInstance(); EasyMock.expect( bigqueryRpcMock.create( EasyMock.anyObject(com.google.api.services.bigquery.model.Job.class), EasyMock.capture(capturedOptions))) .andReturn(newJobPb()); EasyMock.replay(bigqueryRpcMock); BigQuery.JobOption jobOptions = BigQuery.JobOption.fields(BigQuery.JobField.USER_EMAIL); bigquery = options.getService(); bigquery.create(JobInfo.of(QueryJobConfiguration.of("SOME QUERY")), jobOptions); String selector = (String) capturedOptions.getValue().get(jobOptions.getRpcOption()); // jobReference and configuration are always sent; the RPC call won't succeed otherwise. assertThat(selector.split(",")) .asList() .containsExactly("jobReference", "configuration", "user_email"); }
@Test public void testCreateJobSuccess() { String id = "testCreateJobSuccess-id"; JobId jobId = JobId.of(id); String query = "SELECT * in FOO"; Capture<com.google.api.services.bigquery.model.Job> jobCapture = EasyMock.newCapture(); EasyMock.expect( bigqueryRpcMock.create(EasyMock.capture(jobCapture), EasyMock.eq(EMPTY_RPC_OPTIONS))) .andReturn(newJobPb()); EasyMock.replay(bigqueryRpcMock); bigquery = options.getService(); assertThat(bigquery.create(JobInfo.of(jobId, QueryJobConfiguration.of(query)))).isNotNull(); assertThat(jobCapture.getValue().getJobReference().getJobId()).isEqualTo(id); }
@Test public void testExtractJob() throws InterruptedException, TimeoutException { String tableName = "test_export_job_table"; TableId destinationTable = TableId.of(DATASET, tableName); LoadJobConfiguration configuration = LoadJobConfiguration.newBuilder(destinationTable, "gs://" + BUCKET + "/" + LOAD_FILE) .setSchema(SIMPLE_SCHEMA) .build(); Job remoteLoadJob = bigquery.create(JobInfo.of(configuration)); remoteLoadJob = remoteLoadJob.waitFor(); assertNull(remoteLoadJob.getStatus().getError()); ExtractJobConfiguration extractConfiguration = ExtractJobConfiguration.newBuilder(destinationTable, "gs://" + BUCKET + "/" + EXTRACT_FILE) .setPrintHeader(false) .build(); Job remoteExtractJob = bigquery.create(JobInfo.of(extractConfiguration)); remoteExtractJob = remoteExtractJob.waitFor(); assertNull(remoteExtractJob.getStatus().getError()); String extractedCsv = new String(storage.readAllBytes(BUCKET, EXTRACT_FILE), StandardCharsets.UTF_8); assertEquals( Sets.newHashSet(CSV_CONTENT.split("\n")), Sets.newHashSet(extractedCsv.split("\n"))); assertTrue(bigquery.delete(DATASET, tableName)); }
@Test public void testCreateJobNoGet() { String id = "testCreateJobNoGet-id"; JobId jobId = JobId.of(id); String query = "SELECT * in FOO"; Capture<com.google.api.services.bigquery.model.Job> jobCapture = EasyMock.newCapture(); EasyMock.expect( bigqueryRpcMock.create(EasyMock.capture(jobCapture), EasyMock.eq(EMPTY_RPC_OPTIONS))) .andThrow(new BigQueryException(409, "already exists, for some reason")); EasyMock.replay(bigqueryRpcMock); bigquery = options.getService(); try { bigquery.create(JobInfo.of(jobId, QueryJobConfiguration.of(query))); fail("should throw"); } catch (BigQueryException e) { assertThat(jobCapture.getValue().getJobReference().getJobId()).isEqualTo(id); } }
@Test public void testCreateJobTryGet() { final String id = "testCreateJobTryGet-id"; String query = "SELECT * in FOO"; Supplier<JobId> idProvider = new Supplier<JobId>() { @Override public JobId get() { return JobId.of(id); } }; Capture<com.google.api.services.bigquery.model.Job> jobCapture = EasyMock.newCapture(); EasyMock.expect( bigqueryRpcMock.create(EasyMock.capture(jobCapture), EasyMock.eq(EMPTY_RPC_OPTIONS))) .andThrow(new BigQueryException(409, "already exists, for some reason")); EasyMock.expect( bigqueryRpcMock.getJob( anyString(), EasyMock.eq(id), EasyMock.eq((String) null), EasyMock.eq(EMPTY_RPC_OPTIONS))) .andReturn(newJobPb()); EasyMock.replay(bigqueryRpcMock); bigquery = options.getService(); ((BigQueryImpl) bigquery).create(JobInfo.of(QueryJobConfiguration.of(query)), idProvider); assertThat(jobCapture.getValue().getJobReference().getJobId()).isEqualTo(id); }
@Test public void testCancelJob() throws InterruptedException, TimeoutException { String destinationTableName = "test_cancel_query_job_table"; String query = "SELECT TimestampField, StringField, BooleanField FROM " + TABLE_ID.getTable(); TableId destinationTable = TableId.of(DATASET, destinationTableName); QueryJobConfiguration configuration = QueryJobConfiguration.newBuilder(query) .setDefaultDataset(DatasetId.of(DATASET)) .setDestinationTable(destinationTable) .build(); Job remoteJob = bigquery.create(JobInfo.of(configuration)); assertTrue(remoteJob.cancel()); remoteJob = remoteJob.waitFor(); assertNull(remoteJob.getStatus().getError()); }
@Test public void testQueryJobWithDryRun() throws InterruptedException, TimeoutException { String tableName = "test_query_job_table"; String query = "SELECT TimestampField, StringField, BooleanField FROM " + TABLE_ID.getTable(); TableId destinationTable = TableId.of(DATASET, tableName); QueryJobConfiguration configuration = QueryJobConfiguration.newBuilder(query) .setDefaultDataset(DatasetId.of(DATASET)) .setDestinationTable(destinationTable) .setDryRun(true) .build(); Job remoteJob = bigquery.create(JobInfo.of(configuration)); assertNull(remoteJob.getJobId().getJob()); assertEquals(DONE, remoteJob.getStatus().getState()); assertNotNull(remoteJob.getConfiguration()); }
@BeforeClass public static void beforeClass() throws InterruptedException, TimeoutException { RemoteBigQueryHelper bigqueryHelper = RemoteBigQueryHelper.create(); RemoteStorageHelper storageHelper = RemoteStorageHelper.create(); bigquery = bigqueryHelper.getOptions().getService(); storage = storageHelper.getOptions().getService(); storage.create(BucketInfo.of(BUCKET)); storage.create( BlobInfo.newBuilder(BUCKET, LOAD_FILE).setContentType("text/plain").build(), CSV_CONTENT.getBytes(StandardCharsets.UTF_8)); storage.create( BlobInfo.newBuilder(BUCKET, JSON_LOAD_FILE).setContentType("application/json").build(), JSON_CONTENT.getBytes(StandardCharsets.UTF_8)); DatasetInfo info = DatasetInfo.newBuilder(DATASET).setDescription(DESCRIPTION).setLabels(LABELS).build(); bigquery.create(info); LoadJobConfiguration configuration = LoadJobConfiguration.newBuilder( TABLE_ID, "gs://" + BUCKET + "/" + JSON_LOAD_FILE, FormatOptions.json()) .setCreateDisposition(JobInfo.CreateDisposition.CREATE_IF_NEEDED) .setSchema(TABLE_SCHEMA) .build(); Job job = bigquery.create(JobInfo.of(configuration)); job = job.waitFor(); assertNull(job.getStatus().getError()); }
@Test public void testCopyJob() throws InterruptedException, TimeoutException { String sourceTableName = "test_copy_job_source_table"; String destinationTableName = "test_copy_job_destination_table"; TableId sourceTable = TableId.of(DATASET, sourceTableName); StandardTableDefinition tableDefinition = StandardTableDefinition.of(TABLE_SCHEMA); TableInfo tableInfo = TableInfo.of(sourceTable, tableDefinition); Table createdTable = bigquery.create(tableInfo); assertNotNull(createdTable); assertEquals(DATASET, createdTable.getTableId().getDataset()); assertEquals(sourceTableName, createdTable.getTableId().getTable()); TableId destinationTable = TableId.of(DATASET, destinationTableName); CopyJobConfiguration configuration = CopyJobConfiguration.of(destinationTable, sourceTable); Job remoteJob = bigquery.create(JobInfo.of(configuration)); remoteJob = remoteJob.waitFor(); assertNull(remoteJob.getStatus().getError()); Table remoteTable = bigquery.getTable(DATASET, destinationTableName); assertNotNull(remoteTable); assertEquals(destinationTable.getDataset(), remoteTable.getTableId().getDataset()); assertEquals(destinationTableName, remoteTable.getTableId().getTable()); assertEquals(TABLE_SCHEMA, remoteTable.getDefinition().getSchema()); assertTrue(createdTable.delete()); assertTrue(remoteTable.delete()); }