static JobId fromPb(JobReference jobRef) { return newBuilder() .setProject(jobRef.getProjectId()) .setJob(jobRef.getJobId()) .setLocation(jobRef.getLocation()) .build(); } }
@Test public void testCreateJobSuccess() { String id = "testCreateJobSuccess-id"; JobId jobId = JobId.of(id); String query = "SELECT * in FOO"; Capture<com.google.api.services.bigquery.model.Job> jobCapture = EasyMock.newCapture(); EasyMock.expect( bigqueryRpcMock.create(EasyMock.capture(jobCapture), EasyMock.eq(EMPTY_RPC_OPTIONS))) .andReturn(newJobPb()); EasyMock.replay(bigqueryRpcMock); bigquery = options.getService(); assertThat(bigquery.create(JobInfo.of(jobId, QueryJobConfiguration.of(query)))).isNotNull(); assertThat(jobCapture.getValue().getJobReference().getJobId()).isEqualTo(id); }
@Test public void testCreateJobNoGet() { String id = "testCreateJobNoGet-id"; JobId jobId = JobId.of(id); String query = "SELECT * in FOO"; Capture<com.google.api.services.bigquery.model.Job> jobCapture = EasyMock.newCapture(); EasyMock.expect( bigqueryRpcMock.create(EasyMock.capture(jobCapture), EasyMock.eq(EMPTY_RPC_OPTIONS))) .andThrow(new BigQueryException(409, "already exists, for some reason")); EasyMock.replay(bigqueryRpcMock); bigquery = options.getService(); try { bigquery.create(JobInfo.of(jobId, QueryJobConfiguration.of(query))); fail("should throw"); } catch (BigQueryException e) { assertThat(jobCapture.getValue().getJobReference().getJobId()).isEqualTo(id); } }
@Test public void testCreateJobTryGet() { final String id = "testCreateJobTryGet-id"; String query = "SELECT * in FOO"; Supplier<JobId> idProvider = new Supplier<JobId>() { @Override public JobId get() { return JobId.of(id); } }; Capture<com.google.api.services.bigquery.model.Job> jobCapture = EasyMock.newCapture(); EasyMock.expect( bigqueryRpcMock.create(EasyMock.capture(jobCapture), EasyMock.eq(EMPTY_RPC_OPTIONS))) .andThrow(new BigQueryException(409, "already exists, for some reason")); EasyMock.expect( bigqueryRpcMock.getJob( anyString(), EasyMock.eq(id), EasyMock.eq((String) null), EasyMock.eq(EMPTY_RPC_OPTIONS))) .andReturn(newJobPb()); EasyMock.replay(bigqueryRpcMock); bigquery = options.getService(); ((BigQueryImpl) bigquery).create(JobInfo.of(QueryJobConfiguration.of(query)), idProvider); assertThat(jobCapture.getValue().getJobReference().getJobId()).isEqualTo(id); }
/** * Helper to check for non-null Job.getJobReference().getJobId() and quality of the getJobId() * between {@code expected} and {@code actual}, using Preconditions.checkState. */ public void checkJobIdEquality(Job expected, Job actual) { Preconditions.checkState(actual.getJobReference() != null && actual.getJobReference().getJobId() != null && expected.getJobReference() != null && expected.getJobReference().getJobId() != null && actual.getJobReference().getJobId().equals(expected.getJobReference().getJobId()), "jobIds must match in '[expected|actual].getJobReference()' (got '%s' vs '%s')", expected.getJobReference(), actual.getJobReference()); }
static JobId fromPb(JobReference jobRef) { return new JobId(jobRef.getProjectId(), jobRef.getJobId()); } }
static JobId fromPb(JobReference jobRef) { return new JobId(jobRef.getProjectId(), jobRef.getJobId()); } }
@VisibleForTesting public Job getJob(JobReference jobRef, Sleeper sleeper, BackOff backoff) throws IOException, InterruptedException { String jobId = jobRef.getJobId(); Exception lastException; do { try { return client.jobs().get(jobRef.getProjectId(), jobId).execute(); } catch (GoogleJsonResponseException e) { if (errorExtractor.itemNotFound(e)) { LOG.info("No BigQuery job with job id {} found.", jobId); return null; } LOG.info( "Ignoring the error encountered while trying to query the BigQuery job {}", jobId, e); lastException = e; } catch (IOException e) { LOG.info( "Ignoring the error encountered while trying to query the BigQuery job {}", jobId, e); lastException = e; } } while (nextBackOff(sleeper, backoff)); throw new IOException( String.format( "Unable to find BigQuery job: %s, aborting after %d retries.", jobRef, MAX_RPC_RETRIES), lastException); } }
/** * Polls the status of a BigQuery job, returns Job reference if "Done" * * @param bigquery an authorized BigQuery client * @param projectId a string containing the current project ID * @param jobId a reference to an inserted query Job * @return a reference to the completed Job * @throws IOException * @throws InterruptedException */ private static Job checkQueryResults(Bigquery bigquery, String projectId, JobReference jobId) throws IOException, InterruptedException { // Variables to keep track of total query time long startTime = System.currentTimeMillis(); long elapsedTime; while (true) { Job pollJob = bigquery.jobs().get(projectId, jobId.getJobId()).execute(); elapsedTime = System.currentTimeMillis() - startTime; System.out.format("Job status (%dms) %s: %s\n", elapsedTime, jobId.getJobId(), pollJob.getStatus().getState()); if (pollJob.getStatus().getState().equals("DONE")) { return pollJob; } // Pause execution for one second before polling job status again, to // reduce unnecessary calls to the BigQUery API and lower overall // application bandwidth. Thread.sleep(1000); } } // [END start_query]
/** * Makes an API call to the BigQuery API * * @param bigquery an authorized BigQuery client * @param projectId a string containing the current project ID * @param completedJob to the completed Job * @throws IOException */ private static void displayQueryResults(Bigquery bigquery, String projectId, Job completedJob) throws IOException { GetQueryResultsResponse queryResult = bigquery.jobs() .getQueryResults( projectId, completedJob .getJobReference() .getJobId() ).execute(); List<TableRow> rows = queryResult.getRows(); System.out.print("\nQuery Results:\n------------\n"); for (TableRow row : rows) { for (TableCell field : row.getF()) { System.out.printf("%-50s", field.getV()); } System.out.println(); } } // [END display_result]
job.getJobReference() != null && job.getJobReference().getJobId() != null, "Require non-null JobReference and JobId inside; getJobReference() == '%s'", job.getJobReference()); logger.atInfo().withCause(ioe).log( "Fetching existing job after catching exception for duplicate jobId '%s'", job.getJobReference().getJobId()); response = service.jobs().get(projectId, job.getJobReference().getJobId()).execute(); } else { throw new IOException(
static JobId fromPb(JobReference jobRef) { return newBuilder() .setProject(jobRef.getProjectId()) .setJob(jobRef.getJobId()) .setLocation(jobRef.getLocation()) .build(); } }
private List<ResourceId> executeExtract( String jobId, TableReference table, JobService jobService, String executingProject, String extractDestinationDir, String bqLocation) throws InterruptedException, IOException { JobReference jobRef = new JobReference().setProjectId(executingProject).setLocation(bqLocation).setJobId(jobId); String destinationUri = BigQueryIO.getExtractDestinationUri(extractDestinationDir); JobConfigurationExtract extract = new JobConfigurationExtract() .setSourceTable(table) .setDestinationFormat("AVRO") .setDestinationUris(ImmutableList.of(destinationUri)); LOG.info("Starting BigQuery extract job: {}", jobId); jobService.startExtractJob(jobRef, extract); Job extractJob = jobService.pollJob(jobRef, JOB_POLL_MAX_RETRIES); if (BigQueryHelpers.parseStatus(extractJob) != Status.SUCCEEDED) { throw new IOException( String.format( "Extract job %s failed, status: %s.", extractJob.getJobReference().getJobId(), BigQueryHelpers.statusToPrettyString(extractJob.getStatus()))); } LOG.info("BigQuery extract job completed: {}", jobId); return BigQueryIO.getExtractFilePaths(extractDestinationDir, extractJob); }
/** * Creates a Query Job for a particular query on a dataset * * @param bigquery an authorized BigQuery client * @param projectId a String containing the project ID * @param querySql the actual query string * @return a reference to the inserted query job * @throws IOException */ public static JobReference startQuery(Bigquery bigquery, String projectId, String querySql) throws IOException { System.out.format("\nInserting Query Job: %s\n", querySql); Job job = new Job(); JobConfiguration config = new JobConfiguration(); JobConfigurationQuery queryConfig = new JobConfigurationQuery(); config.setQuery(queryConfig); job.setConfiguration(config); queryConfig.setQuery(querySql); Insert insert = bigquery.jobs().insert(projectId, job); insert.setProjectId(projectId); JobReference jobId = insert.execute().getJobReference(); System.out.format("\nJob ID of Query Job is: %s\n", jobId.getJobId()); return jobId; }
@Override public void startLoadJob(JobReference jobRef, JobConfigurationLoad loadConfig) throws IOException { synchronized (allJobs) { verifyUniqueJobId(jobRef.getJobId()); Job job = new Job(); job.setJobReference(jobRef); job.setConfiguration(new JobConfiguration().setLoad(loadConfig)); job.setKind(" bigquery#job"); job.setStatus(new JobStatus().setState("PENDING")); // Copy the files to a new location for import, as the temporary files will be deleted by // the caller. if (loadConfig.getSourceUris().size() > 0) { ImmutableList.Builder<ResourceId> sourceFiles = ImmutableList.builder(); ImmutableList.Builder<ResourceId> loadFiles = ImmutableList.builder(); for (String filename : loadConfig.getSourceUris()) { sourceFiles.add(FileSystems.matchNewResource(filename, false /* isDirectory */)); loadFiles.add( FileSystems.matchNewResource( filename + ThreadLocalRandom.current().nextInt(), false /* isDirectory */)); } FileSystems.copy(sourceFiles.build(), loadFiles.build()); filesForLoadJobs.put(jobRef.getProjectId(), jobRef.getJobId(), loadFiles.build()); } allJobs.put(jobRef.getProjectId(), jobRef.getJobId(), new JobInfo(job)); } }
TableSchema schema = new TableSchema(); schema.setFields(new ArrayList<TableFieldSchema>()); JacksonFactory JACKSON = new JacksonFactory(); JACKSON.createJsonParser(new FileInputStream("schema.json")) .parseArrayAndClose(schema.getFields(), TableFieldSchema.class, null); schema.setFactory(JACKSON); TableReference destTable = new TableReference(); destTable.setProjectId(projectId); destTable.setDatasetId(datasetId); destTable.setTableId(tableId); FileContent content = new FileContent("application/octet-stream", new File(csv)); Job job = new Job(); JobConfiguration config = new JobConfiguration(); JobConfigurationLoad configLoad = new JobConfigurationLoad(); configLoad.setSchema(schema); configLoad.setDestinationTable(destTable); configLoad.setEncoding("UTF-8"); configLoad.setCreateDisposition("CREATE_IF_NEEDED"); config.setLoad(configLoad); job.setConfiguration(config); Insert insert = bigquery.jobs().insert(projectId, job, content); insert.setProjectId(projectId); JobReference jobRef = insert.execute().getJobReference(); String jobId = jobRef.getJobId();
@Override public void startExtractJob(JobReference jobRef, JobConfigurationExtract extractConfig) throws IOException { checkArgument( "AVRO".equals(extractConfig.getDestinationFormat()), "Only extract to AVRO is supported"); synchronized (allJobs) { verifyUniqueJobId(jobRef.getJobId()); ++numExtractJobCalls; Job job = new Job(); job.setJobReference(jobRef); job.setConfiguration(new JobConfiguration().setExtract(extractConfig)); job.setKind(" bigquery#job"); job.setStatus(new JobStatus().setState("PENDING")); allJobs.put(jobRef.getProjectId(), jobRef.getJobId(), new JobInfo(job)); } }
@Override public void startCopyJob(JobReference jobRef, JobConfigurationTableCopy copyConfig) throws IOException { synchronized (allJobs) { verifyUniqueJobId(jobRef.getJobId()); Job job = new Job(); job.setJobReference(jobRef); job.setConfiguration(new JobConfiguration().setCopy(copyConfig)); job.setKind(" bigquery#job"); job.setStatus(new JobStatus().setState("PENDING")); allJobs.put(jobRef.getProjectId(), jobRef.getJobId(), new JobInfo(job)); } }
@Override public void startQueryJob(JobReference jobRef, JobConfigurationQuery query) { synchronized (allJobs) { Job job = new Job(); job.setJobReference(jobRef); job.setConfiguration(new JobConfiguration().setQuery(query)); job.setKind(" bigquery#job"); job.setStatus(new JobStatus().setState("PENDING")); allJobs.put(jobRef.getProjectId(), jobRef.getJobId(), new JobInfo(job)); } }