JobReference toPb() { return new JobReference() .setProjectId(getProject()) .setJobId(getJob()) .setLocation(getLocation()); }
/** * Creates a new JobReference with a unique jobId generated from {@code jobIdPrefix} plus a * randomly generated UUID String. */ public JobReference createJobReference( String projectId, String jobIdPrefix, @Nullable String location) { Preconditions.checkArgument(projectId != null, "projectId must not be null."); Preconditions.checkArgument(jobIdPrefix != null, "jobIdPrefix must not be null."); Preconditions.checkArgument(jobIdPrefix.matches(BIGQUERY_JOB_ID_PATTERN), "jobIdPrefix '%s' must match pattern '%s'", jobIdPrefix, BIGQUERY_JOB_ID_PATTERN); String fullJobId = String.format("%s-%s", jobIdPrefix, UUID.randomUUID().toString()); Preconditions.checkArgument(fullJobId.length() <= BIGQUERY_JOB_ID_MAX_LENGTH, "fullJobId '%s' has length '%s'; must be less than or equal to %s", fullJobId, fullJobId.length(), BIGQUERY_JOB_ID_MAX_LENGTH); return new JobReference().setProjectId(projectId).setJobId(fullJobId).setLocation(location); }
JobReference toPb() { return new JobReference().setProjectId(project).setJobId(job); }
JobReference toPb() { return new JobReference().setProjectId(project).setJobId(job); }
JobReference toPb() { return new JobReference() .setProjectId(getProject()) .setJobId(getJob()) .setLocation(getLocation()); }
@Test public void testGetJobNotFound() throws Exception { when(response.getContentType()).thenReturn(Json.MEDIA_TYPE); when(response.getStatusCode()).thenReturn(404); BigQueryServicesImpl.JobServiceImpl jobService = new BigQueryServicesImpl.JobServiceImpl(bigquery); JobReference jobRef = new JobReference().setProjectId("projectId").setJobId("jobId"); Job job = jobService.getJob(jobRef, Sleeper.DEFAULT, BackOff.ZERO_BACKOFF); assertEquals(null, job); verify(response, times(1)).getStatusCode(); verify(response, times(1)).getContent(); verify(response, times(1)).getContentType(); }
@Test public void testGetJobThrows() throws Exception { when(response.getContentType()).thenReturn(Json.MEDIA_TYPE); when(response.getStatusCode()).thenReturn(401); BigQueryServicesImpl.JobServiceImpl jobService = new BigQueryServicesImpl.JobServiceImpl(bigquery); JobReference jobRef = new JobReference().setProjectId("projectId").setJobId("jobId"); thrown.expect(IOException.class); thrown.expectMessage(String.format("Unable to find BigQuery job: %s", jobRef)); jobService.getJob(jobRef, Sleeper.DEFAULT, BackOff.STOP_BACKOFF); }
private List<ResourceId> executeExtract( String jobId, TableReference table, JobService jobService, String executingProject, String extractDestinationDir, String bqLocation) throws InterruptedException, IOException { JobReference jobRef = new JobReference().setProjectId(executingProject).setLocation(bqLocation).setJobId(jobId); String destinationUri = BigQueryIO.getExtractDestinationUri(extractDestinationDir); JobConfigurationExtract extract = new JobConfigurationExtract() .setSourceTable(table) .setDestinationFormat("AVRO") .setDestinationUris(ImmutableList.of(destinationUri)); LOG.info("Starting BigQuery extract job: {}", jobId); jobService.startExtractJob(jobRef, extract); Job extractJob = jobService.pollJob(jobRef, JOB_POLL_MAX_RETRIES); if (BigQueryHelpers.parseStatus(extractJob) != Status.SUCCEEDED) { throw new IOException( String.format( "Extract job %s failed, status: %s.", extractJob.getJobReference().getJobId(), BigQueryHelpers.statusToPrettyString(extractJob.getStatus()))); } LOG.info("BigQuery extract job completed: {}", jobId); return BigQueryIO.getExtractFilePaths(extractDestinationDir, extractJob); }
@Test public void testGetJobSucceeds() throws Exception { Job testJob = new Job(); testJob.setStatus(new JobStatus()); when(response.getContentType()).thenReturn(Json.MEDIA_TYPE); when(response.getStatusCode()).thenReturn(200); when(response.getContent()).thenReturn(toStream(testJob)); BigQueryServicesImpl.JobServiceImpl jobService = new BigQueryServicesImpl.JobServiceImpl(bigquery); JobReference jobRef = new JobReference().setProjectId("projectId").setJobId("jobId"); Job job = jobService.getJob(jobRef, Sleeper.DEFAULT, BackOff.ZERO_BACKOFF); assertEquals(testJob, job); verify(response, times(1)).getStatusCode(); verify(response, times(1)).getContent(); verify(response, times(1)).getContentType(); }
/** Tests that {@link BigQueryServicesImpl.JobServiceImpl#pollJob} returns UNKNOWN. */ @Test public void testPollJobUnknown() throws IOException, InterruptedException { Job testJob = new Job(); testJob.setStatus(new JobStatus()); when(response.getContentType()).thenReturn(Json.MEDIA_TYPE); when(response.getStatusCode()).thenReturn(200); when(response.getContent()).thenReturn(toStream(testJob)); BigQueryServicesImpl.JobServiceImpl jobService = new BigQueryServicesImpl.JobServiceImpl(bigquery); JobReference jobRef = new JobReference().setProjectId("projectId").setJobId("jobId"); Job job = jobService.pollJob(jobRef, Sleeper.DEFAULT, BackOff.STOP_BACKOFF); assertEquals(null, job); verify(response, times(1)).getStatusCode(); verify(response, times(1)).getContent(); verify(response, times(1)).getContentType(); }
/** Tests that {@link BigQueryServicesImpl.JobServiceImpl#pollJob} succeeds. */ @Test public void testPollJobSucceeds() throws IOException, InterruptedException { Job testJob = new Job(); testJob.setStatus(new JobStatus().setState("DONE")); when(response.getContentType()).thenReturn(Json.MEDIA_TYPE); when(response.getStatusCode()).thenReturn(200); when(response.getContent()).thenReturn(toStream(testJob)); BigQueryServicesImpl.JobServiceImpl jobService = new BigQueryServicesImpl.JobServiceImpl(bigquery); JobReference jobRef = new JobReference().setProjectId("projectId").setJobId("jobId"); Job job = jobService.pollJob(jobRef, Sleeper.DEFAULT, BackOff.ZERO_BACKOFF); assertEquals(testJob, job); verify(response, times(1)).getStatusCode(); verify(response, times(1)).getContent(); verify(response, times(1)).getContentType(); }
.setProjectId(executingProject) .setLocation(bqLocation) .setJobId(queryJobId);
/** Tests that {@link BigQueryServicesImpl.JobServiceImpl#pollJob} fails. */ @Test public void testPollJobFailed() throws IOException, InterruptedException { Job testJob = new Job(); testJob.setStatus(new JobStatus().setState("DONE").setErrorResult(new ErrorProto())); when(response.getContentType()).thenReturn(Json.MEDIA_TYPE); when(response.getStatusCode()).thenReturn(200); when(response.getContent()).thenReturn(toStream(testJob)); BigQueryServicesImpl.JobServiceImpl jobService = new BigQueryServicesImpl.JobServiceImpl(bigquery); JobReference jobRef = new JobReference().setProjectId("projectId").setJobId("jobId"); Job job = jobService.pollJob(jobRef, Sleeper.DEFAULT, BackOff.ZERO_BACKOFF); assertEquals(testJob, job); verify(response, times(1)).getStatusCode(); verify(response, times(1)).getContent(); verify(response, times(1)).getContentType(); }
@Override public JobStatistics dryRunQuery( String projectId, JobConfigurationQuery queryConfig, String location) throws InterruptedException, IOException { JobReference jobRef = new JobReference().setLocation(location).setProjectId(projectId); Job job = new Job() .setJobReference(jobRef) .setConfiguration(new JobConfiguration().setQuery(queryConfig).setDryRun(true)); return executeWithRetries( client.jobs().insert(projectId, job), String.format( "Unable to dry run query: %s, aborting after %d retries.", queryConfig, MAX_RPC_RETRIES), Sleeper.DEFAULT, createDefaultBackoff(), ALWAYS_RETRY) .getStatistics(); }
/** Tests that {@link BigQueryServicesImpl.JobServiceImpl#startLoadJob} succeeds. */ @Test public void testStartLoadJobSucceeds() throws IOException, InterruptedException { Job testJob = new Job(); JobReference jobRef = new JobReference(); jobRef.setJobId("jobId"); jobRef.setProjectId("projectId"); testJob.setJobReference(jobRef); when(response.getContentType()).thenReturn(Json.MEDIA_TYPE); when(response.getStatusCode()).thenReturn(200); when(response.getContent()).thenReturn(toStream(testJob)); Sleeper sleeper = new FastNanoClockAndSleeper(); JobServiceImpl.startJob( testJob, new ApiErrorExtractor(), bigquery, sleeper, BackOffAdapter.toGcpBackOff(FluentBackoff.DEFAULT.backoff())); verify(response, times(1)).getStatusCode(); verify(response, times(1)).getContent(); verify(response, times(1)).getContentType(); expectedLogs.verifyInfo(String.format("Started BigQuery job: %s", jobRef)); }
/** Tests that {@link BigQueryServicesImpl.JobServiceImpl#startLoadJob} succeeds with a retry. */ @Test public void testStartLoadJobRetry() throws IOException, InterruptedException { Job testJob = new Job(); JobReference jobRef = new JobReference(); jobRef.setJobId("jobId"); jobRef.setProjectId("projectId"); testJob.setJobReference(jobRef); // First response is 403 rate limited, second response has valid payload. when(response.getContentType()).thenReturn(Json.MEDIA_TYPE); when(response.getStatusCode()).thenReturn(403).thenReturn(200); when(response.getContent()) .thenReturn(toStream(errorWithReasonAndStatus("rateLimitExceeded", 403))) .thenReturn(toStream(testJob)); Sleeper sleeper = new FastNanoClockAndSleeper(); JobServiceImpl.startJob( testJob, new ApiErrorExtractor(), bigquery, sleeper, BackOffAdapter.toGcpBackOff(FluentBackoff.DEFAULT.backoff())); verify(response, times(2)).getStatusCode(); verify(response, times(2)).getContent(); verify(response, times(2)).getContentType(); }
.setProjectId(projectId) .setJobId(jobId.get());
/** * Tests that {@link BigQueryServicesImpl.JobServiceImpl#startLoadJob} succeeds with an already * exist job. */ @Test public void testStartLoadJobSucceedsAlreadyExists() throws IOException, InterruptedException { Job testJob = new Job(); JobReference jobRef = new JobReference(); jobRef.setJobId("jobId"); jobRef.setProjectId("projectId"); testJob.setJobReference(jobRef); when(response.getStatusCode()).thenReturn(409); // 409 means already exists Sleeper sleeper = new FastNanoClockAndSleeper(); JobServiceImpl.startJob( testJob, new ApiErrorExtractor(), bigquery, sleeper, BackOffAdapter.toGcpBackOff(FluentBackoff.DEFAULT.backoff())); verify(response, times(1)).getStatusCode(); verify(response, times(1)).getContent(); verify(response, times(1)).getContentType(); expectedLogs.verifyNotLogged("Started BigQuery job"); }
@Override void cleanup(PassThroughThenCleanup.ContextContainer c) throws Exception { PipelineOptions options = c.getPipelineOptions(); BigQueryOptions bqOptions = options.as(BigQueryOptions.class); String jobUuid = c.getJobId(); final String extractDestinationDir = resolveTempLocation(bqOptions.getTempLocation(), "BigQueryExtractTemp", jobUuid); final String executingProject = bqOptions.getProject(); JobReference jobRef = new JobReference() .setProjectId(executingProject) .setJobId(getExtractJobId(createJobIdToken(bqOptions.getJobName(), jobUuid))); Job extractJob = getBigQueryServices().getJobService(bqOptions).getJob(jobRef); if (extractJob != null) { List<ResourceId> extractFiles = getExtractFilePaths(extractDestinationDir, extractJob); if (extractFiles != null && !extractFiles.isEmpty()) { FileSystems.delete( extractFiles, MoveOptions.StandardMoveOptions.IGNORE_MISSING_FILES); } } } };
.setProjectId("") .setLocation("") .setJobId(retryId.getJobId()));