/** Example of reloading all fields until job status is DONE. */ // [TARGET reload(JobOption...)] public JobStatus.State reload() throws InterruptedException { // [START ] while (!JobStatus.State.DONE.equals(job.getStatus().getState())) { Thread.sleep(1000L); job = job.reload(); } // [END ] return job.getStatus().getState(); }
@Override public boolean shouldRetry(Throwable prevThrowable, Job prevResponse) { return prevResponse != null && !JobStatus.State.DONE.equals(prevResponse.getStatus().getState()); } },
/** Example of reloading status field until job status is DONE. */ // [TARGET reload(JobOption...)] public JobStatus.State reloadStatus() throws InterruptedException { // [START ] while (!JobStatus.State.DONE.equals(job.getStatus().getState())) { Thread.sleep(1000L); job = job.reload(BigQuery.JobOption.fields(BigQuery.JobField.STATUS)); } // [END ] return job.getStatus().getState(); }
@Override void run(BigQuery bigquery, JobInfo job) throws Exception { System.out.println("Creating job"); Job startedJob = bigquery.create(job); while (!startedJob.isDone()) { System.out.println("Waiting for job " + startedJob.getJobId().getJob() + " to complete"); Thread.sleep(1000L); } startedJob = startedJob.reload(); if (startedJob.getStatus().getError() == null) { System.out.println("Job " + startedJob.getJobId().getJob() + " succeeded"); } else { System.out.println("Job " + startedJob.getJobId().getJob() + " failed"); System.out.println("Error: " + startedJob.getStatus().getError()); } } }
/** Example usage of {@code waitFor()}. */ // [TARGET waitFor(RetryOption...)] public boolean waitFor() throws InterruptedException { try { // [START ] Job completedJob = job.waitFor(); if (completedJob == null) { // job no longer exists } else if (completedJob.getStatus().getError() != null) { // job failed, handle error } else { // job completed successfully } // [END ] } catch (BigQueryException e) { // Timeouts shouldn't happen without a timeout option. if (e.getCause() instanceof PollException) { return false; } throw e; } return true; }
public static void main(String... args) throws InterruptedException, TimeoutException { BigQuery bigquery = BigQueryOptions.getDefaultInstance().getService(); TableId tableId = TableId.of("dataset", "table"); Table table = bigquery.getTable(tableId); if (table == null) { System.out.println("Creating table " + tableId); Field integerField = Field.of("fieldName", LegacySQLTypeName.INTEGER); Schema schema = Schema.of(integerField); table = bigquery.create(TableInfo.of(tableId, StandardTableDefinition.of(schema))); } System.out.println("Loading data into table " + tableId); Job loadJob = table.load(FormatOptions.csv(), "gs://bucket/path"); loadJob = loadJob.waitFor(); if (loadJob.getStatus().getError() != null) { System.out.println("Job completed with errors"); } else { System.out.println("Job succeeded"); } } }
/** Example of copying multiple tables to a destination. */ public void copyTables(String datasetId, String destinationTableId) throws InterruptedException { generateTableWithDdl(datasetId, "table1"); generateTableWithDdl(datasetId, "table2"); // [START bigquery_copy_table_multiple_source] TableId destinationTable = TableId.of(datasetId, destinationTableId); CopyJobConfiguration configuration = CopyJobConfiguration.newBuilder( destinationTable, Arrays.asList(TableId.of(datasetId, "table1"), TableId.of(datasetId, "table2"))) .build(); // Copy the tables. Job job = bigquery.create(JobInfo.of(configuration)); job = job.waitFor(); // Check the table StandardTableDefinition table = bigquery.getTable(destinationTable).getDefinition(); System.out.println("State: " + job.getStatus().getState()); System.out.printf("Copied %d rows.\n", table.getNumRows()); // [END bigquery_copy_table_multiple_source] }
/** * Checks if this job has completed its execution, either failing or succeeding. If the job does * not exist this method returns {@code true}. * * <p>Example of waiting for a job until it reports that it is done. * * <pre>{@code * while (!job.isDone()) { * Thread.sleep(1000L); * } * }</pre> * * @return {@code true} if this job is in {@link JobStatus.State#DONE} state or if it does not * exist, {@code false} if the state is not {@link JobStatus.State#DONE} * @throws BigQueryException upon failure */ public boolean isDone() { checkNotDryRun("isDone"); Job job = bigquery.getJob(getJobId(), JobOption.fields(BigQuery.JobField.STATUS)); return job == null || JobStatus.State.DONE.equals(job.getStatus().getState()); } /**
/** Example of loading a parquet file from GCS to a table. */ public void loadTableGcsParquet(String datasetName) throws InterruptedException { // [START bigquery_load_table_gcs_parquet] String sourceUri = "gs://cloud-samples-data/bigquery/us-states/us-states.parquet"; TableId tableId = TableId.of(datasetName, "us_states"); LoadJobConfiguration configuration = LoadJobConfiguration.builder(tableId, sourceUri) .setFormatOptions(FormatOptions.parquet()) .build(); // Load the table Job loadJob = bigquery.create(JobInfo.of(configuration)); loadJob = loadJob.waitFor(); // Check the table StandardTableDefinition destinationTable = bigquery.getTable(tableId).getDefinition(); System.out.println("State: " + loadJob.getStatus().getState()); System.out.printf("Loaded %d rows.\n", destinationTable.getNumRows()); // [END bigquery_load_table_gcs_parquet] }
/** Example of copying the table to a destination table. */ // [TARGET copy(String, String, JobOption...)] // [VARIABLE "my_dataset"] // [VARIABLE "my_destination_table"] public Job copy(String datasetName, String tableName) { // [START ] Job job = table.copy(datasetName, tableName); // Wait for the job to complete. try { Job completedJob = job.waitFor( RetryOption.initialRetryDelay(Duration.ofSeconds(1)), RetryOption.totalTimeout(Duration.ofMinutes(3))); if (completedJob != null && completedJob.getStatus().getError() == null) { // Job completed successfully } else { // Handle error case } } catch (InterruptedException e) { // Handle interrupted wait } // [END ] return job; }
/** Example extracting data to single Google Cloud Storage file. */ // [TARGET extract(String, String, JobOption...)] // [VARIABLE "CSV"] // [VARIABLE "gs://my_bucket/filename.csv"] public Job extractSingle(String format, String gcsUrl) { // [START bigquery_extract_table] Job job = table.extract(format, gcsUrl); // Wait for the job to complete try { Job completedJob = job.waitFor( RetryOption.initialRetryDelay(Duration.ofSeconds(1)), RetryOption.totalTimeout(Duration.ofMinutes(3))); if (completedJob != null && completedJob.getStatus().getError() == null) { // Job completed successfully } else { // Handle error case } } catch (InterruptedException e) { // Handle interrupted wait } // [END bigquery_extract_table] return job; }
/** Example usage of {@code waitFor()} with checking period and timeout. */ // [TARGET waitFor(RetryOption...)] public boolean waitForWithOptions() throws InterruptedException { try { // [START ] Job completedJob = job.waitFor( RetryOption.initialRetryDelay(Duration.ofSeconds(1)), RetryOption.totalTimeout(Duration.ofMinutes(1))); if (completedJob == null) { // job no longer exists } else if (completedJob.getStatus().getError() != null) { // job failed, handle error } else { // job completed successfully } // [END ] } catch (BigQueryException e) { if (e.getCause() instanceof PollException) { return false; } throw e; } return true; }
/** Example loading data from a single Google Cloud Storage file. */ // [TARGET load(FormatOptions, String, JobOption...)] // [VARIABLE "gs://my_bucket/filename.csv"] public Job loadSingle(String sourceUri) { // [START bigquery_load_table_gcs_csv] Job job = table.load(FormatOptions.csv(), sourceUri); // Wait for the job to complete try { Job completedJob = job.waitFor( RetryOption.initialRetryDelay(Duration.ofSeconds(1)), RetryOption.totalTimeout(Duration.ofMinutes(3))); if (completedJob != null && completedJob.getStatus().getError() == null) { // Job completed successfully } else { // Handle error case } } catch (InterruptedException e) { // Handle interrupted wait } // [END bigquery_load_table_gcs_csv] return job; } }
/** Example copying the table to a destination table. */ // [TARGET copy(TableId, JobOption...)] // [VARIABLE "my_dataset"] // [VARIABLE "my_destination_table"] public Job copyTableId(String dataset, String tableName) throws BigQueryException { // [START bigquery_copy_table] TableId destinationId = TableId.of(dataset, tableName); JobOption options = JobOption.fields(JobField.STATUS, JobField.USER_EMAIL); Job job = table.copy(destinationId, options); // Wait for the job to complete. try { Job completedJob = job.waitFor( RetryOption.initialRetryDelay(Duration.ofSeconds(1)), RetryOption.totalTimeout(Duration.ofMinutes(3))); if (completedJob != null && completedJob.getStatus().getError() == null) { // Job completed successfully. } else { // Handle error case. } } catch (InterruptedException e) { // Handle interrupted wait } // [END bigquery_copy_table] return job; }
@Test public void testListJobs() { Page<Job> jobs = bigquery.listJobs(); for (Job job : jobs.getValues()) { assertNotNull(job.getJobId()); assertNotNull(job.getStatistics()); assertNotNull(job.getStatus()); assertNotNull(job.getUserEmail()); assertNotNull(job.getGeneratedId()); } }
@Test public void testListJobsWithSelectedFields() { Page<Job> jobs = bigquery.listJobs(JobListOption.fields(JobField.USER_EMAIL)); for (Job job : jobs.getValues()) { assertNotNull(job.getJobId()); assertNotNull(job.getStatus()); assertNotNull(job.getUserEmail()); assertNull(job.getStatistics()); assertNull(job.getGeneratedId()); } }
@Test public void testCancelJob() throws InterruptedException, TimeoutException { String destinationTableName = "test_cancel_query_job_table"; String query = "SELECT TimestampField, StringField, BooleanField FROM " + TABLE_ID.getTable(); TableId destinationTable = TableId.of(DATASET, destinationTableName); QueryJobConfiguration configuration = QueryJobConfiguration.newBuilder(query) .setDefaultDataset(DatasetId.of(DATASET)) .setDestinationTable(destinationTable) .build(); Job remoteJob = bigquery.create(JobInfo.of(configuration)); assertTrue(remoteJob.cancel()); remoteJob = remoteJob.waitFor(); assertNull(remoteJob.getStatus().getError()); }
@Test public void testQueryJobWithDryRun() throws InterruptedException, TimeoutException { String tableName = "test_query_job_table"; String query = "SELECT TimestampField, StringField, BooleanField FROM " + TABLE_ID.getTable(); TableId destinationTable = TableId.of(DATASET, tableName); QueryJobConfiguration configuration = QueryJobConfiguration.newBuilder(query) .setDefaultDataset(DatasetId.of(DATASET)) .setDestinationTable(destinationTable) .setDryRun(true) .build(); Job remoteJob = bigquery.create(JobInfo.of(configuration)); assertNull(remoteJob.getJobId().getJob()); assertEquals(DONE, remoteJob.getStatus().getState()); assertNotNull(remoteJob.getConfiguration()); }
@BeforeClass public static void beforeClass() throws InterruptedException, TimeoutException { RemoteBigQueryHelper bigqueryHelper = RemoteBigQueryHelper.create(); RemoteStorageHelper storageHelper = RemoteStorageHelper.create(); bigquery = bigqueryHelper.getOptions().getService(); storage = storageHelper.getOptions().getService(); storage.create(BucketInfo.of(BUCKET)); storage.create( BlobInfo.newBuilder(BUCKET, LOAD_FILE).setContentType("text/plain").build(), CSV_CONTENT.getBytes(StandardCharsets.UTF_8)); storage.create( BlobInfo.newBuilder(BUCKET, JSON_LOAD_FILE).setContentType("application/json").build(), JSON_CONTENT.getBytes(StandardCharsets.UTF_8)); DatasetInfo info = DatasetInfo.newBuilder(DATASET).setDescription(DESCRIPTION).setLabels(LABELS).build(); bigquery.create(info); LoadJobConfiguration configuration = LoadJobConfiguration.newBuilder( TABLE_ID, "gs://" + BUCKET + "/" + JSON_LOAD_FILE, FormatOptions.json()) .setCreateDisposition(JobInfo.CreateDisposition.CREATE_IF_NEEDED) .setSchema(TABLE_SCHEMA) .build(); Job job = bigquery.create(JobInfo.of(configuration)); job = job.waitFor(); assertNull(job.getStatus().getError()); }
@Test public void testCopyJob() throws InterruptedException, TimeoutException { String sourceTableName = "test_copy_job_source_table"; String destinationTableName = "test_copy_job_destination_table"; TableId sourceTable = TableId.of(DATASET, sourceTableName); StandardTableDefinition tableDefinition = StandardTableDefinition.of(TABLE_SCHEMA); TableInfo tableInfo = TableInfo.of(sourceTable, tableDefinition); Table createdTable = bigquery.create(tableInfo); assertNotNull(createdTable); assertEquals(DATASET, createdTable.getTableId().getDataset()); assertEquals(sourceTableName, createdTable.getTableId().getTable()); TableId destinationTable = TableId.of(DATASET, destinationTableName); CopyJobConfiguration configuration = CopyJobConfiguration.of(destinationTable, sourceTable); Job remoteJob = bigquery.create(JobInfo.of(configuration)); remoteJob = remoteJob.waitFor(); assertNull(remoteJob.getStatus().getError()); Table remoteTable = bigquery.getTable(DATASET, destinationTableName); assertNotNull(remoteTable); assertEquals(destinationTable.getDataset(), remoteTable.getTableId().getDataset()); assertEquals(destinationTableName, remoteTable.getTableId().getTable()); assertEquals(TABLE_SCHEMA, remoteTable.getDefinition().getSchema()); assertTrue(createdTable.delete()); assertTrue(remoteTable.delete()); }