@Override public TableDestination getDestination(ValueInSingleWindow<T> element) { TableDestination destination = super.getDestination(element); String partitioning = this.jsonTimePartitioning.get(); checkArgument(partitioning != null, "jsonTimePartitioning can not be null"); return new TableDestination( destination.getTableSpec(), destination.getTableDescription(), partitioning); }
static String createJobId( String prefix, TableDestination tableDestination, int partition, long index) { // Job ID must be different for each partition of each table. String destinationHash = Hashing.murmur3_128().hashUnencodedChars(tableDestination.toString()).toString(); String jobId = String.format("%s_%s", prefix, destinationHash); if (partition >= 0) { jobId += String.format("_%05d", partition); } if (index >= 0) { jobId += String.format("_%05d", index); } return jobId; }
@Override public void encode(TableDestination value, OutputStream outStream) throws IOException { if (value == null) { throw new CoderException("cannot encode a null value"); } tableSpecCoder.encode(value.getTableSpec(), outStream); tableDescriptionCoder.encode(value.getTableDescription(), outStream); }
public TableDestination getTable(String tableSpec) { return new TableDestination(tableSpec, "Table " + tableSpec); }
List<ShardedKey<TableDestination>> expectedPartitions = Lists.newArrayList(); if (isSingleton) { expectedPartitions.add(ShardedKey.of(new TableDestination("SINGLETON", ""), 1)); } else { for (int i = 0; i < numTables; ++i) { for (int j = 1; j <= expectedNumPartitionsPerTable; ++j) { String tableName = String.format("project-id:dataset-id.tables%05d", i); expectedPartitions.add(ShardedKey.of(new TableDestination(tableName, ""), j)); files.add( new WriteBundlesToFiles.Result<>( fileName, fileSize, new TableDestination(tableName, ""))); Map<String, List<String>> filesPerTableResult = Maps.newHashMap(); for (KV<ShardedKey<TableDestination>, List<String>> partition : partitions) { String table = partition.getKey().getKey().getTableSpec(); partitionsResult.add(partition.getKey()); List<String> tableFilesResult =
for (int i = 0; i < numFinalTables; ++i) { String tableName = "project-id:dataset-id.table_" + i; TableDestination tableDestination = new TableDestination(tableName, "table_" + i + "_desc"); for (int j = 0; i < numTempTablesPerFinalTable; ++i) { TableReference tempTable = TableReference tableReference = tableDestination.getTableReference(); Table table = checkNotNull(fakeDatasetService.getTable(tableReference)); assertEquals(tableReference.getTableId() + "_desc", tableDestination.getTableDescription());
@FinishBundle public void finishBundle(FinishBundleContext c) throws Exception { DatasetService datasetService = bqServices.getDatasetService(c.getPipelineOptions().as(BigQueryOptions.class)); PendingJobManager jobManager = new PendingJobManager(); for (PendingJobData pendingJob : pendingJobs) { jobManager.addPendingJob( pendingJob.retryJob, j -> { try { if (pendingJob.tableDestination.getTableDescription() != null) { TableReference ref = pendingJob.tableDestination.getTableReference(); datasetService.patchTableDescription( ref.clone() .setTableId(BigQueryHelpers.stripPartitionDecorator(ref.getTableId())), pendingJob.tableDestination.getTableDescription()); } removeTemporaryTables(datasetService, pendingJob.tempTables); return null; } catch (IOException | InterruptedException e) { return e; } }); } jobManager.waitForDone(); }
@Override public TableSchema getSchema(TableDestination destination) { Map<String, String> mapValue = sideInput(schemaView); String schema = mapValue.get(destination.getTableSpec()); checkArgument( schema != null, "Schema view must contain data for every destination used, " + "but view %s does not contain data for table destination %s " + "produced by %s", schemaView, destination.getTableSpec(), inner); return BigQueryHelpers.fromJsonString(schema, TableSchema.class); }
private PendingJobData startWriteRename( TableDestination finalTableDestination, Iterable<String> tempTableNames, ProcessContext c) throws Exception { WriteDisposition writeDisposition = (c.pane().getIndex() == 0) ? firstPaneWriteDisposition : WriteDisposition.WRITE_APPEND; CreateDisposition createDisposition = (c.pane().getIndex() == 0) ? firstPaneCreateDisposition : CreateDisposition.CREATE_NEVER; List<TableReference> tempTables = StreamSupport.stream(tempTableNames.spliterator(), false) .map(table -> BigQueryHelpers.fromJsonString(table, TableReference.class)) .collect(Collectors.toList()); ; // Make sure each destination table gets a unique job id. String jobIdPrefix = BigQueryHelpers.createJobId( c.sideInput(jobIdToken), finalTableDestination, -1, c.pane().getIndex()); BigQueryHelpers.PendingJob retryJob = startCopy( bqServices.getJobService(c.getPipelineOptions().as(BigQueryOptions.class)), bqServices.getDatasetService(c.getPipelineOptions().as(BigQueryOptions.class)), jobIdPrefix, finalTableDestination.getTableReference(), tempTables, writeDisposition, createDisposition); return new PendingJobData(retryJob, finalTableDestination, tempTables); }
@Override public void encode(TableDestination value, OutputStream outStream) throws IOException { TableDestinationCoder.of().encode(value, outStream); timePartitioningCoder.encode(value.getJsonTimePartitioning(), outStream); }
public TableDestination withTableReference(TableReference tableReference) { return new TableDestination(tableReference, tableDescription, jsonTimePartitioning); }
for (int i = 0; i < numTables; ++i) { String tableName = String.format("project-id:dataset-id.table%05d", i); TableDestination tableDestination = new TableDestination(tableName, tableName); for (int j = 0; j < numPartitions; ++j) { String tempTableId = BigQueryHelpers.createJobId(jobIdToken, tableDestination, j, 0); partitions.add(KV.of(ShardedKey.of(tableDestination.getTableSpec(), j), filesPerPartition));
@ProcessElement public void processElement(ProcessContext context, BoundedWindow window) throws IOException { ThreadLocalRandom randomGenerator = ThreadLocalRandom.current(); // We output on keys 0-numShards. String tableSpec = context.element().getKey().getTableSpec(); context.output( KV.of( ShardedKey.of(tableSpec, randomGenerator.nextInt(0, numShards)), context.element().getValue())); } }
@Override public TableDestination decode(InputStream inStream) throws IOException { TableDestination destination = TableDestinationCoder.of().decode(inStream); String jsonTimePartitioning = timePartitioningCoder.decode(inStream); return new TableDestination( destination.getTableSpec(), destination.getTableDescription(), jsonTimePartitioning); }
@Override public TableDestination getTable(String destination) { return new TableDestination(destination, destination); }
for (int i = 0; i < 5; i++) { TableDestination destination = new TableDestination("project-id:dataset-id" + ".table-id-" + i, ""); targetTables.put(i, destination); destination.getTableSpec(), toJsonString( new TableSchema()
@Override public TableDestination getDestination(ValueInSingleWindow<T> element) { String tableSpec = this.tableSpec.get(); checkArgument(tableSpec != null, "tableSpec can not be null"); return new TableDestination(tableSpec, tableDescription); }
@Override public TableDestination decode(InputStream inStream) throws IOException { String tableSpec = tableSpecCoder.decode(inStream); String tableDescription = tableDescriptionCoder.decode(inStream); return new TableDestination(tableSpec, tableDescription); }
@Override public TableDestination getTable(Integer userId) { verifySideInputs(); // Each user in it's own table. return new TableDestination( "dataset-id.userid-" + userId + "$" + partitionDecorator, "table for userid " + userId); }
@Override public TableDestination getTable(Long destination) { return new TableDestination( new TableReference() .setProjectId(writeProject) .setDatasetId(writeDataset) .setTableId(writeTable + "_" + destination), "Table for year " + destination); }