@Override com.google.api.services.bigquery.model.JobConfiguration toPb() { JobConfigurationExtract extractConfigurationPb = new JobConfigurationExtract(); extractConfigurationPb.setDestinationUris(destinationUris); extractConfigurationPb.setSourceTable(sourceTable.toPb()); extractConfigurationPb.setPrintHeader(printHeader); extractConfigurationPb.setFieldDelimiter(fieldDelimiter); extractConfigurationPb.setDestinationFormat(format); extractConfigurationPb.setCompression(compression); return new com.google.api.services.bigquery.model.JobConfiguration() .setExtract(extractConfigurationPb); }
@Test public void testIncomplete() { // https://github.com/googleapis/google-cloud-java/issues/2357 com.google.api.services.bigquery.model.Job job = new com.google.api.services.bigquery.model.Job() .setStatistics( new com.google.api.services.bigquery.model.JobStatistics() .setCreationTime(1234L) .setStartTime(5678L)); job.setConfiguration( new com.google.api.services.bigquery.model.JobConfiguration() .setCopy(new com.google.api.services.bigquery.model.JobConfigurationTableCopy())); assertThat(JobStatistics.fromPb(job)).isInstanceOf(CopyStatistics.class); job.setConfiguration( new com.google.api.services.bigquery.model.JobConfiguration() .setLoad(new com.google.api.services.bigquery.model.JobConfigurationLoad())); assertThat(JobStatistics.fromPb(job)).isInstanceOf(LoadStatistics.class); job.setConfiguration( new com.google.api.services.bigquery.model.JobConfiguration() .setExtract(new com.google.api.services.bigquery.model.JobConfigurationExtract())); assertThat(JobStatistics.fromPb(job)).isInstanceOf(ExtractStatistics.class); job.setConfiguration( new com.google.api.services.bigquery.model.JobConfiguration() .setQuery(new com.google.api.services.bigquery.model.JobConfigurationQuery())); assertThat(JobStatistics.fromPb(job)).isInstanceOf(QueryStatistics.class); }
private List<ResourceId> executeExtract( String jobId, TableReference table, JobService jobService, String executingProject, String extractDestinationDir, String bqLocation) throws InterruptedException, IOException { JobReference jobRef = new JobReference().setProjectId(executingProject).setLocation(bqLocation).setJobId(jobId); String destinationUri = BigQueryIO.getExtractDestinationUri(extractDestinationDir); JobConfigurationExtract extract = new JobConfigurationExtract() .setSourceTable(table) .setDestinationFormat("AVRO") .setDestinationUris(ImmutableList.of(destinationUri)); LOG.info("Starting BigQuery extract job: {}", jobId); jobService.startExtractJob(jobRef, extract); Job extractJob = jobService.pollJob(jobRef, JOB_POLL_MAX_RETRIES); if (BigQueryHelpers.parseStatus(extractJob) != Status.SUCCEEDED) { throw new IOException( String.format( "Extract job %s failed, status: %s.", extractJob.getJobReference().getJobId(), BigQueryHelpers.statusToPrettyString(extractJob.getStatus()))); } LOG.info("BigQuery extract job completed: {}", jobId); return BigQueryIO.getExtractFilePaths(extractDestinationDir, extractJob); }
@Override com.google.api.services.bigquery.model.JobConfiguration toPb() { JobConfigurationExtract extractConfigurationPb = new JobConfigurationExtract(); extractConfigurationPb.setDestinationUris(destinationUris); extractConfigurationPb.setSourceTable(sourceTable.toPb()); extractConfigurationPb.setPrintHeader(printHeader); extractConfigurationPb.setFieldDelimiter(fieldDelimiter); extractConfigurationPb.setDestinationFormat(format); extractConfigurationPb.setCompression(compression); return new com.google.api.services.bigquery.model.JobConfiguration() .setExtract(extractConfigurationPb); }
@Override com.google.api.services.bigquery.model.JobConfiguration toPb() { JobConfigurationExtract extractConfigurationPb = new JobConfigurationExtract(); extractConfigurationPb.setDestinationUris(destinationUris); extractConfigurationPb.setSourceTable(sourceTable.toPb()); extractConfigurationPb.setPrintHeader(printHeader); extractConfigurationPb.setFieldDelimiter(fieldDelimiter); extractConfigurationPb.setDestinationFormat(format); extractConfigurationPb.setCompression(compression); return new com.google.api.services.bigquery.model.JobConfiguration() .setExtract(extractConfigurationPb); }
@Override com.google.api.services.bigquery.model.JobConfiguration toPb() { JobConfigurationExtract extractConfigurationPb = new JobConfigurationExtract(); extractConfigurationPb.setDestinationUris(destinationUris); extractConfigurationPb.setSourceTable(sourceTable.toPb()); extractConfigurationPb.setPrintHeader(printHeader); extractConfigurationPb.setFieldDelimiter(fieldDelimiter); extractConfigurationPb.setDestinationFormat(format); extractConfigurationPb.setCompression(compression); return new com.google.api.services.bigquery.model.JobConfiguration() .setExtract(extractConfigurationPb); }
JobConfigurationExtract extractConfig = new JobConfigurationExtract();
@Override public void beginExport() throws IOException { // Create job and configuration. JobConfigurationExtract extractConfig = new JobConfigurationExtract(); // Set source. extractConfig.setSourceTable(tableToExport.getTableReference()); // Set destination. extractConfig.setDestinationUris(getExportPaths()); extractConfig.set(DESTINATION_FORMAT_KEY, fileFormat.getFormatIdentifier()); JobConfiguration config = new JobConfiguration(); config.setExtract(extractConfig); JobReference jobReference = bigQueryHelper.createJobReference( projectId, "exporttocloudstorage", tableToExport.getLocation()); Job job = new Job(); job.setConfiguration(config); job.setJobReference(jobReference); // Insert and run job. try { Job response = bigQueryHelper.insertJobOrFetchDuplicate(projectId, job); logger.atFine().log("Got response '%s'", response); exportJobReference = response.getJobReference(); } catch (IOException e) { String error = String.format( "Error while exporting table %s", BigQueryStrings.toString(tableToExport.getTableReference())); throw new IOException(error, e); } }
@Override protected JobConfiguration jobConfiguration(String projectId) { JobConfigurationExtract cfg = new JobConfigurationExtract(); try { cfg.setDestinationUris(params.getList("destination", String.class)); } catch (ConfigException ignore) { cfg.setDestinationUri(params.get("destination", String.class)); } Optional<DatasetReference> defaultDataset = params.getOptional("dataset", String.class) .transform(Bq::datasetReference); String sourceTable = params.get("_command", String.class); cfg.setSourceTable(tableReference(projectId, defaultDataset, sourceTable)); params.getOptional("print_header", boolean.class).transform(cfg::setPrintHeader); params.getOptional("field_delimiter", String.class).transform(cfg::setFieldDelimiter); params.getOptional("destination_format", String.class).transform(cfg::setDestinationFormat); params.getOptional("compression", String.class).transform(cfg::setCompression); return new JobConfiguration() .setExtract(cfg); } }