@Test public void testCreateJobSuccess() { String id = "testCreateJobSuccess-id"; JobId jobId = JobId.of(id); String query = "SELECT * in FOO"; Capture<com.google.api.services.bigquery.model.Job> jobCapture = EasyMock.newCapture(); EasyMock.expect( bigqueryRpcMock.create(EasyMock.capture(jobCapture), EasyMock.eq(EMPTY_RPC_OPTIONS))) .andReturn(newJobPb()); EasyMock.replay(bigqueryRpcMock); bigquery = options.getService(); assertThat(bigquery.create(JobInfo.of(jobId, QueryJobConfiguration.of(query)))).isNotNull(); assertThat(jobCapture.getValue().getJobReference().getJobId()).isEqualTo(id); }
JobReference toPb() { return new JobReference() .setProjectId(getProject()) .setJobId(getJob()) .setLocation(getLocation()); }
static JobId fromPb(JobReference jobRef) { return newBuilder() .setProject(jobRef.getProjectId()) .setJob(jobRef.getJobId()) .setLocation(jobRef.getLocation()) .build(); } }
JobReference toPb() { return new JobReference().setProjectId(project).setJobId(job); }
static JobId fromPb(JobReference jobRef) { return new JobId(jobRef.getProjectId(), jobRef.getJobId()); } }
@Override public JobStatistics dryRunQuery( String projectId, JobConfigurationQuery queryConfig, String location) throws InterruptedException, IOException { JobReference jobRef = new JobReference().setLocation(location).setProjectId(projectId); Job job = new Job() .setJobReference(jobRef) .setConfiguration(new JobConfiguration().setQuery(queryConfig).setDryRun(true)); return executeWithRetries( client.jobs().insert(projectId, job), String.format( "Unable to dry run query: %s, aborting after %d retries.", queryConfig, MAX_RPC_RETRIES), Sleeper.DEFAULT, createDefaultBackoff(), ALWAYS_RETRY) .getStatistics(); }
bigquery .jobs() .get(projectId, jobReference.getJobId()) .setLocation(jobReference.getLocation()); logger.atFine().log( "Job status (%s ms) %s: %s", elapsedTime, jobReference.getJobId(), pollJob.getStatus().getState()); if (pollJob.getStatus().getState().equals("DONE")) { notDone = false; String.format( "Job %s failed to complete after %s millis.", jobReference.getJobId(), elapsedTime));
projectId, "global", new SubmitJobRequest() .setJob(new Job() .setReference(new JobReference() .setJobId(curJobId)) .setPlacement(new JobPlacement()
@Override public Job create(Job job, Map<Option, ?> options) { try { String projectId = job.getJobReference() != null ? job.getJobReference().getProjectId() : this.options.getProjectId(); return bigquery .jobs() .insert(projectId, job) .setFields(Option.FIELDS.getString(options)) .execute(); } catch (IOException ex) { throw translate(ex); } }
JobReference toPb() { return new JobReference().setProjectId(project).setJobId(job); }
static JobId fromPb(JobReference jobRef) { return new JobId(jobRef.getProjectId(), jobRef.getJobId()); } }
new JobReference() .setJobId(Joiner.on("-").join("INSERT", projectId, datasetId, tableId, DateTime.now().toString("dd-MM-yyyy_HH-mm-ss-SSS")))
@Override public Job create(Job job, Map<Option, ?> options) { try { String projectId = job.getJobReference() != null ? job.getJobReference().getProjectId() : this.options.getProjectId(); return bigquery .jobs() .insert(projectId, job) .setFields(Option.FIELDS.getString(options)) .execute(); } catch (IOException ex) { throw translate(ex); } }
JobReference toPb() { return new JobReference() .setProjectId(getProject()) .setJobId(getJob()) .setLocation(getLocation()); }
@Test public void testGetJobThrows() throws Exception { when(response.getContentType()).thenReturn(Json.MEDIA_TYPE); when(response.getStatusCode()).thenReturn(401); BigQueryServicesImpl.JobServiceImpl jobService = new BigQueryServicesImpl.JobServiceImpl(bigquery); JobReference jobRef = new JobReference().setProjectId("projectId").setJobId("jobId"); thrown.expect(IOException.class); thrown.expectMessage(String.format("Unable to find BigQuery job: %s", jobRef)); jobService.getJob(jobRef, Sleeper.DEFAULT, BackOff.STOP_BACKOFF); }
@VisibleForTesting public Job getJob(JobReference jobRef, Sleeper sleeper, BackOff backoff) throws IOException, InterruptedException { String jobId = jobRef.getJobId(); Exception lastException; do { try { return client.jobs().get(jobRef.getProjectId(), jobId).execute(); } catch (GoogleJsonResponseException e) { if (errorExtractor.itemNotFound(e)) { LOG.info("No BigQuery job with job id {} found.", jobId); return null; } LOG.info( "Ignoring the error encountered while trying to query the BigQuery job {}", jobId, e); lastException = e; } catch (IOException e) { LOG.info( "Ignoring the error encountered while trying to query the BigQuery job {}", jobId, e); lastException = e; } } while (nextBackOff(sleeper, backoff)); throw new IOException( String.format( "Unable to find BigQuery job: %s, aborting after %d retries.", jobRef, MAX_RPC_RETRIES), lastException); } }
@Test public void testCreateJobNoGet() { String id = "testCreateJobNoGet-id"; JobId jobId = JobId.of(id); String query = "SELECT * in FOO"; Capture<com.google.api.services.bigquery.model.Job> jobCapture = EasyMock.newCapture(); EasyMock.expect( bigqueryRpcMock.create(EasyMock.capture(jobCapture), EasyMock.eq(EMPTY_RPC_OPTIONS))) .andThrow(new BigQueryException(409, "already exists, for some reason")); EasyMock.replay(bigqueryRpcMock); bigquery = options.getService(); try { bigquery.create(JobInfo.of(jobId, QueryJobConfiguration.of(query))); fail("should throw"); } catch (BigQueryException e) { assertThat(jobCapture.getValue().getJobReference().getJobId()).isEqualTo(id); } }
static JobId fromPb(JobReference jobRef) { return newBuilder() .setProject(jobRef.getProjectId()) .setJob(jobRef.getJobId()) .setLocation(jobRef.getLocation()) .build(); } }
/** * Creates a new JobReference with a unique jobId generated from {@code jobIdPrefix} plus a * randomly generated UUID String. */ public JobReference createJobReference( String projectId, String jobIdPrefix, @Nullable String location) { Preconditions.checkArgument(projectId != null, "projectId must not be null."); Preconditions.checkArgument(jobIdPrefix != null, "jobIdPrefix must not be null."); Preconditions.checkArgument(jobIdPrefix.matches(BIGQUERY_JOB_ID_PATTERN), "jobIdPrefix '%s' must match pattern '%s'", jobIdPrefix, BIGQUERY_JOB_ID_PATTERN); String fullJobId = String.format("%s-%s", jobIdPrefix, UUID.randomUUID().toString()); Preconditions.checkArgument(fullJobId.length() <= BIGQUERY_JOB_ID_MAX_LENGTH, "fullJobId '%s' has length '%s'; must be less than or equal to %s", fullJobId, fullJobId.length(), BIGQUERY_JOB_ID_MAX_LENGTH); return new JobReference().setProjectId(projectId).setJobId(fullJobId).setLocation(location); }
@Test public void testGetJobNotFound() throws Exception { when(response.getContentType()).thenReturn(Json.MEDIA_TYPE); when(response.getStatusCode()).thenReturn(404); BigQueryServicesImpl.JobServiceImpl jobService = new BigQueryServicesImpl.JobServiceImpl(bigquery); JobReference jobRef = new JobReference().setProjectId("projectId").setJobId("jobId"); Job job = jobService.getJob(jobRef, Sleeper.DEFAULT, BackOff.ZERO_BACKOFF); assertEquals(null, job); verify(response, times(1)).getStatusCode(); verify(response, times(1)).getContent(); verify(response, times(1)).getContentType(); }