BuilderImpl(Job jobPb) { this.etag = jobPb.getEtag(); this.generatedId = jobPb.getId(); if (jobPb.getJobReference() != null) { this.jobId = JobId.fromPb(jobPb.getJobReference()); } this.selfLink = jobPb.getSelfLink(); if (jobPb.getStatus() != null) { this.status = JobStatus.fromPb(jobPb.getStatus()); } if (jobPb.getStatistics() != null) { this.statistics = JobStatistics.fromPb(jobPb); } this.userEmail = jobPb.getUserEmail(); if (jobPb.getConfiguration() != null) { this.configuration = JobConfiguration.fromPb(jobPb.getConfiguration()); } }
static Status parseStatus(@Nullable Job job) { if (job == null) { return Status.UNKNOWN; } JobStatus status = job.getStatus(); if (status.getErrorResult() != null) { return Status.FAILED; } else if (status.getErrors() != null && !status.getErrors().isEmpty()) { return Status.FAILED; } else { return Status.SUCCEEDED; } }
/** * Polls the status of a BigQuery job, returns Job reference if "Done" * * @param bigquery an authorized BigQuery client * @param projectId a string containing the current project ID * @param jobId a reference to an inserted query Job * @return a reference to the completed Job * @throws IOException * @throws InterruptedException */ private static Job checkQueryResults(Bigquery bigquery, String projectId, JobReference jobId) throws IOException, InterruptedException { // Variables to keep track of total query time long startTime = System.currentTimeMillis(); long elapsedTime; while (true) { Job pollJob = bigquery.jobs().get(projectId, jobId.getJobId()).execute(); elapsedTime = System.currentTimeMillis() - startTime; System.out.format("Job status (%dms) %s: %s\n", elapsedTime, jobId.getJobId(), pollJob.getStatus().getState()); if (pollJob.getStatus().getState().equals("DONE")) { return pollJob; } // Pause execution for one second before polling job status again, to // reduce unnecessary calls to the BigQUery API and lower overall // application bandwidth. Thread.sleep(1000); } } // [END start_query]
"Load job {} finished in unknown state: {}: {}", currentJobId, job.getStatus(), shouldRetry() ? "will retry" : "will not retry"); return false; oldJobId, shouldRetry() ? "will retry" : "will not retry", job.getStatus(), currentJobId); return false; String.format( "Unexpected status [%s] of load job: %s.", job.getStatus(), BigQueryHelpers.jobToPrettyString(job)));
continue; JobStatus status = job.getStatus(); if (status == null) { LOG.info("Still waiting for BigQuery job {} to enter pending state", jobRef);
JobStatus jobStatus = loadJob.getStatus(); if (jobStatus == null) { LOG.info("job status for {} not found, so retrying with that job id", jobId);
logger.atFine().log( "Job status (%s ms) %s: %s", elapsedTime, jobReference.getJobId(), pollJob.getStatus().getState()); if (pollJob.getStatus().getState().equals("DONE")) { notDone = false; if (pollJob.getStatus().getErrorResult() != null) { throw new IOException( "Error during BigQuery job execution: " + pollJob.getStatus().getErrorResult());
BuilderImpl(Job jobPb) { this.etag = jobPb.getEtag(); this.generatedId = jobPb.getId(); if (jobPb.getJobReference() != null) { this.jobId = JobId.fromPb(jobPb.getJobReference()); } this.selfLink = jobPb.getSelfLink(); if (jobPb.getStatus() != null) { this.status = JobStatus.fromPb(jobPb.getStatus()); } if (jobPb.getStatistics() != null) { this.statistics = JobStatistics.fromPb(jobPb.getStatistics()); } this.userEmail = jobPb.getUserEmail(); this.configuration = JobConfiguration.fromPb(jobPb.getConfiguration()); }
BuilderImpl(Job jobPb) { this.etag = jobPb.getEtag(); this.generatedId = jobPb.getId(); if (jobPb.getJobReference() != null) { this.jobId = JobId.fromPb(jobPb.getJobReference()); } this.selfLink = jobPb.getSelfLink(); if (jobPb.getStatus() != null) { this.status = JobStatus.fromPb(jobPb.getStatus()); } if (jobPb.getStatistics() != null) { this.statistics = JobStatistics.fromPb(jobPb.getStatistics()); } this.userEmail = jobPb.getUserEmail(); this.configuration = JobConfiguration.fromPb(jobPb.getConfiguration()); }
private List<ResourceId> executeExtract( String jobId, TableReference table, JobService jobService, String executingProject, String extractDestinationDir, String bqLocation) throws InterruptedException, IOException { JobReference jobRef = new JobReference().setProjectId(executingProject).setLocation(bqLocation).setJobId(jobId); String destinationUri = BigQueryIO.getExtractDestinationUri(extractDestinationDir); JobConfigurationExtract extract = new JobConfigurationExtract() .setSourceTable(table) .setDestinationFormat("AVRO") .setDestinationUris(ImmutableList.of(destinationUri)); LOG.info("Starting BigQuery extract job: {}", jobId); jobService.startExtractJob(jobRef, extract); Job extractJob = jobService.pollJob(jobRef, JOB_POLL_MAX_RETRIES); if (BigQueryHelpers.parseStatus(extractJob) != Status.SUCCEEDED) { throw new IOException( String.format( "Extract job %s failed, status: %s.", extractJob.getJobReference().getJobId(), BigQueryHelpers.statusToPrettyString(extractJob.getStatus()))); } LOG.info("BigQuery extract job completed: {}", jobId); return BigQueryIO.getExtractFilePaths(extractDestinationDir, extractJob); }
BuilderImpl(Job jobPb) { this.etag = jobPb.getEtag(); this.generatedId = jobPb.getId(); if (jobPb.getJobReference() != null) { this.jobId = JobId.fromPb(jobPb.getJobReference()); } this.selfLink = jobPb.getSelfLink(); if (jobPb.getStatus() != null) { this.status = JobStatus.fromPb(jobPb.getStatus()); } if (jobPb.getStatistics() != null) { this.statistics = JobStatistics.fromPb(jobPb); } this.userEmail = jobPb.getUserEmail(); if (jobPb.getConfiguration() != null) { this.configuration = JobConfiguration.fromPb(jobPb.getConfiguration()); } }
String.format( "Query job %s failed, status: %s.", queryJobId, BigQueryHelpers.statusToPrettyString(job.getStatus())));
String.format("Error when trying to get status of the job for query %s.", queryConfig.toPrettyString())); JobStatus status = pollJob.getStatus(); if (status.getState().equals("DONE")) {
if (!"FAILED".equals(job.job.getStatus().getState())) { if (numFailures < numFailuresExpected) { ++numFailures; job.job.getStatus().setState("RUNNING"); } else if (job.getJobCount == 2 * GET_JOBS_TRANSITION_INTERVAL + 1) { job.job.setStatus(runJob(job.job)); .getStatus() .setState("FAILED") .setErrorResult(
@Override public Job pollJob(JobReference jobRef, int maxAttempts) throws InterruptedException { BackOff backoff = BackOffAdapter.toGcpBackOff( FluentBackoff.DEFAULT .withMaxRetries(maxAttempts) .withInitialBackoff(Duration.millis(10)) .withMaxBackoff(Duration.standardSeconds(1)) .backoff()); Sleeper sleeper = Sleeper.DEFAULT; try { do { Job job = getJob(jobRef); if (job != null) { JobStatus status = job.getStatus(); if (status != null && ("DONE".equals(status.getState()) || "FAILED".equals(status.getState()))) { return job; } } } while (BackOffUtils.next(sleeper, backoff)); } catch (IOException e) { return null; } return null; }