private PendingJobData startWriteRename( TableDestination finalTableDestination, Iterable<String> tempTableNames, ProcessContext c) throws Exception { WriteDisposition writeDisposition = (c.pane().getIndex() == 0) ? firstPaneWriteDisposition : WriteDisposition.WRITE_APPEND; CreateDisposition createDisposition = (c.pane().getIndex() == 0) ? firstPaneCreateDisposition : CreateDisposition.CREATE_NEVER; List<TableReference> tempTables = StreamSupport.stream(tempTableNames.spliterator(), false) .map(table -> BigQueryHelpers.fromJsonString(table, TableReference.class)) .collect(Collectors.toList()); ; // Make sure each destination table gets a unique job id. String jobIdPrefix = BigQueryHelpers.createJobId( c.sideInput(jobIdToken), finalTableDestination, -1, c.pane().getIndex()); BigQueryHelpers.PendingJob retryJob = startCopy( bqServices.getJobService(c.getPipelineOptions().as(BigQueryOptions.class)), bqServices.getDatasetService(c.getPipelineOptions().as(BigQueryOptions.class)), jobIdPrefix, finalTableDestination.getTableReference(), tempTables, writeDisposition, createDisposition); return new PendingJobData(retryJob, finalTableDestination, tempTables); }
TableReference tableReference = tableDestination.getTableReference(); Table table = checkNotNull(fakeDatasetService.getTable(tableReference)); assertEquals(tableReference.getTableId() + "_desc", tableDestination.getTableDescription());
@FinishBundle public void finishBundle(FinishBundleContext c) throws Exception { DatasetService datasetService = bqServices.getDatasetService(c.getPipelineOptions().as(BigQueryOptions.class)); PendingJobManager jobManager = new PendingJobManager(); for (PendingJobData pendingJob : pendingJobs) { jobManager.addPendingJob( pendingJob.retryJob, j -> { try { if (pendingJob.tableDestination.getTableDescription() != null) { TableReference ref = pendingJob.tableDestination.getTableReference(); datasetService.patchTableDescription( ref.clone() .setTableId(BigQueryHelpers.stripPartitionDecorator(ref.getTableId())), pendingJob.tableDestination.getTableDescription()); } removeTemporaryTables(datasetService, pendingJob.tempTables); return null; } catch (IOException | InterruptedException e) { return e; } }); } jobManager.waitForDone(); }