@Override public TableDestination getDestination(ValueInSingleWindow<T> element) { TableDestination destination = super.getDestination(element); String partitioning = this.jsonTimePartitioning.get(); checkArgument(partitioning != null, "jsonTimePartitioning can not be null"); return new TableDestination( destination.getTableSpec(), destination.getTableDescription(), partitioning); }
@Override public void encode(TableDestination value, OutputStream outStream) throws IOException { if (value == null) { throw new CoderException("cannot encode a null value"); } tableSpecCoder.encode(value.getTableSpec(), outStream); tableDescriptionCoder.encode(value.getTableDescription(), outStream); }
@Override public TableDestination decode(InputStream inStream) throws IOException { TableDestination destination = TableDestinationCoder.of().decode(inStream); String jsonTimePartitioning = timePartitioningCoder.decode(inStream); return new TableDestination( destination.getTableSpec(), destination.getTableDescription(), jsonTimePartitioning); }
@FinishBundle public void finishBundle(FinishBundleContext c) throws Exception { DatasetService datasetService = bqServices.getDatasetService(c.getPipelineOptions().as(BigQueryOptions.class)); PendingJobManager jobManager = new PendingJobManager(); for (PendingJobData pendingJob : pendingJobs) { jobManager.addPendingJob( pendingJob.retryJob, j -> { try { if (pendingJob.tableDestination.getTableDescription() != null) { TableReference ref = pendingJob.tableDestination.getTableReference(); datasetService.patchTableDescription( ref.clone() .setTableId(BigQueryHelpers.stripPartitionDecorator(ref.getTableId())), pendingJob.tableDestination.getTableDescription()); } removeTemporaryTables(datasetService, pendingJob.tempTables); return null; } catch (IOException | InterruptedException e) { return e; } }); } jobManager.waitForDone(); }
TableReference tableReference = tableDestination.getTableReference(); Table table = checkNotNull(fakeDatasetService.getTable(tableReference)); assertEquals(tableReference.getTableId() + "_desc", tableDestination.getTableDescription());