@Override com.google.api.services.bigquery.model.JobConfiguration toPb() { JobConfigurationExtract extractConfigurationPb = new JobConfigurationExtract(); extractConfigurationPb.setDestinationUris(destinationUris); extractConfigurationPb.setSourceTable(sourceTable.toPb()); extractConfigurationPb.setPrintHeader(printHeader); extractConfigurationPb.setFieldDelimiter(fieldDelimiter); extractConfigurationPb.setDestinationFormat(format); extractConfigurationPb.setCompression(compression); return new com.google.api.services.bigquery.model.JobConfiguration() .setExtract(extractConfigurationPb); }
private Builder(com.google.api.services.bigquery.model.JobConfiguration configurationPb) { this(); JobConfigurationExtract extractConfigurationPb = configurationPb.getExtract(); this.sourceTable = TableId.fromPb(extractConfigurationPb.getSourceTable()); this.destinationUris = extractConfigurationPb.getDestinationUris(); this.printHeader = extractConfigurationPb.getPrintHeader(); this.fieldDelimiter = extractConfigurationPb.getFieldDelimiter(); this.format = extractConfigurationPb.getDestinationFormat(); this.compression = extractConfigurationPb.getCompression(); }
@Override public void beginExport() throws IOException { // Create job and configuration. JobConfigurationExtract extractConfig = new JobConfigurationExtract(); // Set source. extractConfig.setSourceTable(tableToExport.getTableReference()); // Set destination. extractConfig.setDestinationUris(getExportPaths()); extractConfig.set(DESTINATION_FORMAT_KEY, fileFormat.getFormatIdentifier()); JobConfiguration config = new JobConfiguration(); config.setExtract(extractConfig); JobReference jobReference = bigQueryHelper.createJobReference( projectId, "exporttocloudstorage", tableToExport.getLocation()); Job job = new Job(); job.setConfiguration(config); job.setJobReference(jobReference); // Insert and run job. try { Job response = bigQueryHelper.insertJobOrFetchDuplicate(projectId, job); logger.atFine().log("Got response '%s'", response); exportJobReference = response.getJobReference(); } catch (IOException e) { String error = String.format( "Error while exporting table %s", BigQueryStrings.toString(tableToExport.getTableReference())); throw new IOException(error, e); } }
private List<ResourceId> executeExtract( String jobId, TableReference table, JobService jobService, String executingProject, String extractDestinationDir, String bqLocation) throws InterruptedException, IOException { JobReference jobRef = new JobReference().setProjectId(executingProject).setLocation(bqLocation).setJobId(jobId); String destinationUri = BigQueryIO.getExtractDestinationUri(extractDestinationDir); JobConfigurationExtract extract = new JobConfigurationExtract() .setSourceTable(table) .setDestinationFormat("AVRO") .setDestinationUris(ImmutableList.of(destinationUri)); LOG.info("Starting BigQuery extract job: {}", jobId); jobService.startExtractJob(jobRef, extract); Job extractJob = jobService.pollJob(jobRef, JOB_POLL_MAX_RETRIES); if (BigQueryHelpers.parseStatus(extractJob) != Status.SUCCEEDED) { throw new IOException( String.format( "Extract job %s failed, status: %s.", extractJob.getJobReference().getJobId(), BigQueryHelpers.statusToPrettyString(extractJob.getStatus()))); } LOG.info("BigQuery extract job completed: {}", jobId); return BigQueryIO.getExtractFilePaths(extractDestinationDir, extractJob); }
@Override protected JobConfiguration jobConfiguration(String projectId) { JobConfigurationExtract cfg = new JobConfigurationExtract(); try { cfg.setDestinationUris(params.getList("destination", String.class)); } catch (ConfigException ignore) { cfg.setDestinationUri(params.get("destination", String.class)); } Optional<DatasetReference> defaultDataset = params.getOptional("dataset", String.class) .transform(Bq::datasetReference); String sourceTable = params.get("_command", String.class); cfg.setSourceTable(tableReference(projectId, defaultDataset, sourceTable)); params.getOptional("print_header", boolean.class).transform(cfg::setPrintHeader); params.getOptional("field_delimiter", String.class).transform(cfg::setFieldDelimiter); params.getOptional("destination_format", String.class).transform(cfg::setDestinationFormat); params.getOptional("compression", String.class).transform(cfg::setCompression); return new JobConfiguration() .setExtract(cfg); } }
@Test public void testIncomplete() { // https://github.com/googleapis/google-cloud-java/issues/2357 com.google.api.services.bigquery.model.Job job = new com.google.api.services.bigquery.model.Job() .setStatistics( new com.google.api.services.bigquery.model.JobStatistics() .setCreationTime(1234L) .setStartTime(5678L)); job.setConfiguration( new com.google.api.services.bigquery.model.JobConfiguration() .setCopy(new com.google.api.services.bigquery.model.JobConfigurationTableCopy())); assertThat(JobStatistics.fromPb(job)).isInstanceOf(CopyStatistics.class); job.setConfiguration( new com.google.api.services.bigquery.model.JobConfiguration() .setLoad(new com.google.api.services.bigquery.model.JobConfigurationLoad())); assertThat(JobStatistics.fromPb(job)).isInstanceOf(LoadStatistics.class); job.setConfiguration( new com.google.api.services.bigquery.model.JobConfiguration() .setExtract(new com.google.api.services.bigquery.model.JobConfigurationExtract())); assertThat(JobStatistics.fromPb(job)).isInstanceOf(ExtractStatistics.class); job.setConfiguration( new com.google.api.services.bigquery.model.JobConfiguration() .setQuery(new com.google.api.services.bigquery.model.JobConfigurationQuery())); assertThat(JobStatistics.fromPb(job)).isInstanceOf(QueryStatistics.class); }
private JobStatus runExtractJob(Job job, JobConfigurationExtract extract) throws InterruptedException, IOException { TableReference sourceTable = extract.getSourceTable(); List<TableRow> rows = datasetService.getAllRows( sourceTable.getProjectId(), sourceTable.getDatasetId(), sourceTable.getTableId()); TableSchema schema = datasetService.getTable(sourceTable).getSchema(); List<Long> destinationFileCounts = Lists.newArrayList(); for (String destination : extract.getDestinationUris()) { destinationFileCounts.add(writeRows(sourceTable.getTableId(), rows, schema, destination)); } job.setStatistics( new JobStatistics() .setExtract(new JobStatistics4().setDestinationUriFileCounts(destinationFileCounts))); return new JobStatus().setState("DONE"); }
@Override public void startExtractJob(JobReference jobRef, JobConfigurationExtract extractConfig) throws IOException { checkArgument( "AVRO".equals(extractConfig.getDestinationFormat()), "Only extract to AVRO is supported"); synchronized (allJobs) { verifyUniqueJobId(jobRef.getJobId()); ++numExtractJobCalls; Job job = new Job(); job.setJobReference(jobRef); job.setConfiguration(new JobConfiguration().setExtract(extractConfig)); job.setKind(" bigquery#job"); job.setStatus(new JobStatus().setState("PENDING")); allJobs.put(jobRef.getProjectId(), jobRef.getJobId(), new JobInfo(job)); } }
JobConfigurationExtract extractConfig = new JobConfigurationExtract(); extractConfig.setSourceTable(tableRef); extractConfig.setDestinationUris(gcsPaths); extractConfig.set("destinationFormat", "NEWLINE_DELIMITED_JSON");
@Override com.google.api.services.bigquery.model.JobConfiguration toPb() { JobConfigurationExtract extractConfigurationPb = new JobConfigurationExtract(); extractConfigurationPb.setDestinationUris(destinationUris); extractConfigurationPb.setSourceTable(sourceTable.toPb()); extractConfigurationPb.setPrintHeader(printHeader); extractConfigurationPb.setFieldDelimiter(fieldDelimiter); extractConfigurationPb.setDestinationFormat(format); extractConfigurationPb.setCompression(compression); return new com.google.api.services.bigquery.model.JobConfiguration() .setExtract(extractConfigurationPb); }
private Builder(com.google.api.services.bigquery.model.JobConfiguration configurationPb) { this(); JobConfigurationExtract extractConfigurationPb = configurationPb.getExtract(); this.sourceTable = TableId.fromPb(extractConfigurationPb.getSourceTable()); this.destinationUris = extractConfigurationPb.getDestinationUris(); this.printHeader = extractConfigurationPb.getPrintHeader(); this.fieldDelimiter = extractConfigurationPb.getFieldDelimiter(); this.format = extractConfigurationPb.getDestinationFormat(); this.compression = extractConfigurationPb.getCompression(); }
@Override com.google.api.services.bigquery.model.JobConfiguration toPb() { JobConfigurationExtract extractConfigurationPb = new JobConfigurationExtract(); extractConfigurationPb.setDestinationUris(destinationUris); extractConfigurationPb.setSourceTable(sourceTable.toPb()); extractConfigurationPb.setPrintHeader(printHeader); extractConfigurationPb.setFieldDelimiter(fieldDelimiter); extractConfigurationPb.setDestinationFormat(format); extractConfigurationPb.setCompression(compression); return new com.google.api.services.bigquery.model.JobConfiguration() .setExtract(extractConfigurationPb); }
private Builder(com.google.api.services.bigquery.model.JobConfiguration configurationPb) { this(); JobConfigurationExtract extractConfigurationPb = configurationPb.getExtract(); this.sourceTable = TableId.fromPb(extractConfigurationPb.getSourceTable()); this.destinationUris = extractConfigurationPb.getDestinationUris(); this.printHeader = extractConfigurationPb.getPrintHeader(); this.fieldDelimiter = extractConfigurationPb.getFieldDelimiter(); this.format = extractConfigurationPb.getDestinationFormat(); this.compression = extractConfigurationPb.getCompression(); }
@Override com.google.api.services.bigquery.model.JobConfiguration toPb() { JobConfigurationExtract extractConfigurationPb = new JobConfigurationExtract(); extractConfigurationPb.setDestinationUris(destinationUris); extractConfigurationPb.setSourceTable(sourceTable.toPb()); extractConfigurationPb.setPrintHeader(printHeader); extractConfigurationPb.setFieldDelimiter(fieldDelimiter); extractConfigurationPb.setDestinationFormat(format); extractConfigurationPb.setCompression(compression); return new com.google.api.services.bigquery.model.JobConfiguration() .setExtract(extractConfigurationPb); }
private Builder(com.google.api.services.bigquery.model.JobConfiguration configurationPb) { this(); JobConfigurationExtract extractConfigurationPb = configurationPb.getExtract(); this.sourceTable = TableId.fromPb(extractConfigurationPb.getSourceTable()); this.destinationUris = extractConfigurationPb.getDestinationUris(); this.printHeader = extractConfigurationPb.getPrintHeader(); this.fieldDelimiter = extractConfigurationPb.getFieldDelimiter(); this.format = extractConfigurationPb.getDestinationFormat(); this.compression = extractConfigurationPb.getCompression(); }