Refine search
/** Example usage of {@code waitFor()}. */ // [TARGET waitFor(RetryOption...)] public boolean waitFor() throws InterruptedException { try { // [START ] Job completedJob = job.waitFor(); if (completedJob == null) { // job no longer exists } else if (completedJob.getStatus().getError() != null) { // job failed, handle error } else { // job completed successfully } // [END ] } catch (BigQueryException e) { // Timeouts shouldn't happen without a timeout option. if (e.getCause() instanceof PollException) { return false; } throw e; } return true; }
static JobStatus fromPb(com.google.api.services.bigquery.model.JobStatus statusPb) { List<BigQueryError> allErrors = null; if (statusPb.getErrors() != null) { allErrors = Lists.transform(statusPb.getErrors(), BigQueryError.FROM_PB_FUNCTION); } BigQueryError error = statusPb.getErrorResult() != null ? BigQueryError.fromPb(statusPb.getErrorResult()) : null; return new JobStatus(State.valueOf(statusPb.getState()), error, allErrors); } }
private void compareStatus(JobStatus expected, JobStatus value) { assertEquals(expected, value); assertEquals(expected.getState(), value.getState()); assertEquals(expected.getError(), value.getError()); assertEquals(expected.getExecutionErrors(), value.getExecutionErrors()); } }
checkNotDryRun("getQueryResults"); if (getConfiguration().getType() != Type.QUERY) { throw new UnsupportedOperationException( "Getting query results is supported only for " + Type.QUERY + " jobs"); waitForQueryResults( DEFAULT_JOB_WAIT_SETTINGS, waitOptions.toArray(new QueryResultsOption[0])); if (job.getStatus() == null || !JobStatus.State.DONE.equals(job.getStatus().getState())) { job = reload(); if (job.getStatus() != null && job.getStatus().getError() != null) { throw new JobException( getJobId(), ImmutableList.copyOf(job.getStatus().getExecutionErrors()));
CopyJobConfiguration.of(destinationTable, sourceTable); Job createdJob = bigquery.create(JobInfo.of(copyJobConfiguration)); Job remoteJob = bigquery.getJob(createdJob.getJobId()); assertEquals(createdJob.getJobId(), remoteJob.getJobId()); CopyJobConfiguration createdConfiguration = createdJob.getConfiguration(); CopyJobConfiguration remoteConfiguration = remoteJob.getConfiguration(); assertNull(completedJob.getStatus().getError()); assertTrue(bigquery.delete(DATASET, destinationTableName));
@Override void run(BigQuery bigquery, JobInfo job) throws Exception { System.out.println("Creating job"); Job startedJob = bigquery.create(job); while (!startedJob.isDone()) { System.out.println("Waiting for job " + startedJob.getJobId().getJob() + " to complete"); Thread.sleep(1000L); } startedJob = startedJob.reload(); if (startedJob.getStatus().getError() == null) { System.out.println("Job " + startedJob.getJobId().getJob() + " succeeded"); } else { System.out.println("Job " + startedJob.getJobId().getJob() + " failed"); System.out.println("Error: " + startedJob.getStatus().getError()); } } }
.build(); Job remoteJob = bigquery.create(JobInfo.of(configuration)); remoteJob = remoteJob.waitFor(); assertNull(remoteJob.getStatus().getError()); TableResult result = remoteJob.getQueryResults(); assertEquals(QUERY_RESULT_SCHEMA, result.getSchema()); int rowCount = 0;
/** Example of loading a parquet file from GCS to a table. */ public void loadTableGcsParquet(String datasetName) throws InterruptedException { // [START bigquery_load_table_gcs_parquet] String sourceUri = "gs://cloud-samples-data/bigquery/us-states/us-states.parquet"; TableId tableId = TableId.of(datasetName, "us_states"); LoadJobConfiguration configuration = LoadJobConfiguration.builder(tableId, sourceUri) .setFormatOptions(FormatOptions.parquet()) .build(); // Load the table Job loadJob = bigquery.create(JobInfo.of(configuration)); loadJob = loadJob.waitFor(); // Check the table StandardTableDefinition destinationTable = bigquery.getTable(tableId).getDefinition(); System.out.println("State: " + loadJob.getStatus().getState()); System.out.printf("Loaded %d rows.\n", destinationTable.getNumRows()); // [END bigquery_load_table_gcs_parquet] }
Long timePeriod = context.getProperty(READ_TIMEOUT).evaluateAttributeExpressions(flowFile).asTimePeriod(TimeUnit.SECONDS); Duration waitFor = Duration.of(timePeriod, ChronoUnit.SECONDS); job = job.waitFor(RetryOption.totalTimeout(waitFor)); attributes.put(BigQueryAttributes.JOB_CREATE_TIME_ATTR, Long.toString(job.getStatistics().getCreationTime())); attributes.put(BigQueryAttributes.JOB_END_TIME_ATTR, Long.toString(job.getStatistics().getEndTime())); attributes.put(BigQueryAttributes.JOB_START_TIME_ATTR, Long.toString(job.getStatistics().getStartTime())); attributes.put(BigQueryAttributes.JOB_LINK_ATTR, job.getSelfLink()); boolean jobError = (job.getStatus().getError() != null); attributes.put(BigQueryAttributes.JOB_ERROR_MSG_ATTR, job.getStatus().getError().getMessage()); attributes.put(BigQueryAttributes.JOB_ERROR_REASON_ATTR, job.getStatus().getError().getReason()); attributes.put(BigQueryAttributes.JOB_ERROR_LOCATION_ATTR, job.getStatus().getError().getLocation()); } else { getLogger().log(LogLevel.WARN, job.getStatus().getError().getMessage()); flowFile = session.penalize(flowFile); session.transfer(flowFile, REL_FAILURE);
channel.close(); Job job = channel.getJob().waitFor(); LoadStatistics statistics = job.getStatistics(); assertEquals(1L, statistics.getInputFiles().longValue()); assertEquals(2L, statistics.getOutputRows().longValue()); LoadJobConfiguration jobConfiguration = job.getConfiguration(); assertEquals(TABLE_SCHEMA, jobConfiguration.getSchema()); assertNull(jobConfiguration.getSourceUris()); assertNull(job.getStatus().getError()); Page<FieldValueList> rows = bigquery.listTableData(tableId); int rowCount = 0;
/** * Checks if this job has completed its execution, either failing or succeeding. If the job does * not exist this method returns {@code true}. * * <p>Example of waiting for a job until it reports that it is done. * * <pre>{@code * while (!job.isDone()) { * Thread.sleep(1000L); * } * }</pre> * * @return {@code true} if this job is in {@link JobStatus.State#DONE} state or if it does not * exist, {@code false} if the state is not {@link JobStatus.State#DONE} * @throws BigQueryException upon failure */ public boolean isDone() { checkNotDryRun("isDone"); Job job = bigquery.getJob(getJobId(), JobOption.fields(BigQuery.JobField.STATUS)); return job == null || JobStatus.State.DONE.equals(job.getStatus().getState()); } /**
@Test public void testCancelJob() throws InterruptedException, TimeoutException { String destinationTableName = "test_cancel_query_job_table"; String query = "SELECT TimestampField, StringField, BooleanField FROM " + TABLE_ID.getTable(); TableId destinationTable = TableId.of(DATASET, destinationTableName); QueryJobConfiguration configuration = QueryJobConfiguration.newBuilder(query) .setDefaultDataset(DatasetId.of(DATASET)) .setDestinationTable(destinationTable) .build(); Job remoteJob = bigquery.create(JobInfo.of(configuration)); assertTrue(remoteJob.cancel()); remoteJob = remoteJob.waitFor(); assertNull(remoteJob.getStatus().getError()); }
/** Example of reloading all fields until job status is DONE. */ // [TARGET reload(JobOption...)] public JobStatus.State reload() throws InterruptedException { // [START ] while (!JobStatus.State.DONE.equals(job.getStatus().getState())) { Thread.sleep(1000L); job = job.reload(); } // [END ] return job.getStatus().getState(); }
@Test public void testQueryJobWithDryRun() throws InterruptedException, TimeoutException { String tableName = "test_query_job_table"; String query = "SELECT TimestampField, StringField, BooleanField FROM " + TABLE_ID.getTable(); TableId destinationTable = TableId.of(DATASET, tableName); QueryJobConfiguration configuration = QueryJobConfiguration.newBuilder(query) .setDefaultDataset(DatasetId.of(DATASET)) .setDestinationTable(destinationTable) .setDryRun(true) .build(); Job remoteJob = bigquery.create(JobInfo.of(configuration)); assertNull(remoteJob.getJobId().getJob()); assertEquals(DONE, remoteJob.getStatus().getState()); assertNotNull(remoteJob.getConfiguration()); }
/** Example of running a batch query. */ public void runBatchQuery() throws TimeoutException, InterruptedException { // [START bigquery_query_batch] // BigQuery bigquery = BigQueryOptions.getDefaultInstance().getService(); String query = "SELECT corpus FROM `bigquery-public-data.samples.shakespeare` GROUP BY corpus;"; QueryJobConfiguration queryConfig = QueryJobConfiguration.newBuilder(query) // Run at batch priority, which won't count toward concurrent rate // limit. .setPriority(QueryJobConfiguration.Priority.BATCH) .build(); // Location must match that of the dataset(s) referenced in the query. JobId jobId = JobId.newBuilder().setRandomJob().setLocation("US").build(); String jobIdString = jobId.getJob(); // API request - starts the query. bigquery.create(JobInfo.newBuilder(queryConfig).setJobId(jobId).build()); // Check on the progress by getting the job's updated state. Once the state // is `DONE`, the results are ready. Job queryJob = bigquery.getJob(JobId.newBuilder().setJob(jobIdString).setLocation("US").build()); System.out.printf( "Job %s in location %s currently in state: %s%n", queryJob.getJobId().getJob(), queryJob.getJobId().getLocation(), queryJob.getStatus().getState().toString()); // [END bigquery_query_batch] }
@Test public void testIsDone_True() throws Exception { initializeExpectedJob(2); BigQuery.JobOption[] expectedOptions = {BigQuery.JobOption.fields(BigQuery.JobField.STATUS)}; JobStatus status = createStrictMock(JobStatus.class); expect(status.getState()).andReturn(JobStatus.State.DONE); expect(bigquery.getOptions()).andReturn(mockOptions); expect(bigquery.getJob(JOB_INFO.getJobId(), expectedOptions)) .andReturn(expectedJob.toBuilder().setStatus(status).build()); replay(status, bigquery); initializeJob(); assertTrue(job.isDone()); verify(status); }