@Override public TableSchema getSchema(TableDestination destination) { Map<String, String> mapValue = sideInput(schemaView); String schema = mapValue.get(destination.getTableSpec()); checkArgument( schema != null, "Schema view must contain data for every destination used, " + "but view %s does not contain data for table destination %s " + "produced by %s", schemaView, destination.getTableSpec(), inner); return BigQueryHelpers.fromJsonString(schema, TableSchema.class); }
@Override public TableDestination getDestination(ValueInSingleWindow<T> element) { TableDestination destination = super.getDestination(element); String partitioning = this.jsonTimePartitioning.get(); checkArgument(partitioning != null, "jsonTimePartitioning can not be null"); return new TableDestination( destination.getTableSpec(), destination.getTableDescription(), partitioning); }
@ProcessElement public void processElement(ProcessContext context, BoundedWindow window) throws IOException { ThreadLocalRandom randomGenerator = ThreadLocalRandom.current(); // We output on keys 0-numShards. String tableSpec = context.element().getKey().getTableSpec(); context.output( KV.of( ShardedKey.of(tableSpec, randomGenerator.nextInt(0, numShards)), context.element().getValue())); } }
@Override public void encode(TableDestination value, OutputStream outStream) throws IOException { if (value == null) { throw new CoderException("cannot encode a null value"); } tableSpecCoder.encode(value.getTableSpec(), outStream); tableDescriptionCoder.encode(value.getTableDescription(), outStream); }
@Override public TableDestination decode(InputStream inStream) throws IOException { TableDestination destination = TableDestinationCoder.of().decode(inStream); String jsonTimePartitioning = timePartitioningCoder.decode(inStream); return new TableDestination( destination.getTableSpec(), destination.getTableDescription(), jsonTimePartitioning); }
Map<String, List<String>> filesPerTableResult = Maps.newHashMap(); for (KV<ShardedKey<TableDestination>, List<String>> partition : partitions) { String table = partition.getKey().getKey().getTableSpec(); partitionsResult.add(partition.getKey()); List<String> tableFilesResult =
partitions.add(KV.of(ShardedKey.of(tableDestination.getTableSpec(), j), filesPerPartition));
destination.getTableSpec(), toJsonString( new TableSchema()