SpannerConfig spannerConfig = SpannerConfig.create() .withInstanceId(instanceId) .withDatabaseId(databaseId); PCollectionView<Transaction> tx = p.apply( SpannerIO.createTransaction() .withSpannerConfig(spannerConfig) .withTimestampBound(TimestampBound.strong())); PCollection<Struct> singers = p.apply(SpannerIO.read() .withSpannerConfig(spannerConfig) .withQuery("SELECT SingerID, FirstName, LastName FROM Singers") .withTransaction(tx)); PCollection<Struct> albums = p.apply(SpannerIO.read().withSpannerConfig(spannerConfig) .withQuery("SELECT SingerId, AlbumId, AlbumTitle FROM Albums") .withTransaction(tx));
public static void main(String[] args) { Options options = PipelineOptionsFactory.fromArgs(args).withValidation().as(Options.class); Pipeline p = Pipeline.create(options); SpannerConfig spannerConfig = SpannerConfig.create() .withInstanceId(options.getInstanceId()) .withDatabaseId(options.getDatabaseId()); // [START spanner_dataflow_readall] PCollection<Struct> allRecords = p.apply(SpannerIO.read() .withSpannerConfig(spannerConfig) .withQuery("SELECT t.table_name FROM information_schema.tables AS t WHERE t" + ".table_catalog = '' AND t.table_schema = ''")).apply( MapElements.into(TypeDescriptor.of(ReadOperation.class)) .via((SerializableFunction<Struct, ReadOperation>) input -> { String tableName = input.getString(0); return ReadOperation.create().withQuery("SELECT * FROM " + tableName); })).apply(SpannerIO.readAll().withSpannerConfig(spannerConfig)); // [END spanner_dataflow_readall] PCollection<Long> dbEstimatedSize = allRecords.apply(EstimateSize.create()) .apply(Sum.longsGlobally()); dbEstimatedSize.apply(ToString.elements()).apply(TextIO.write().to(options.getOutput()) .withoutSharding()); p.run().waitUntilFinish(); }
/** Specifies the Cloud Spanner database. */ public ReadAll withDatabaseId(ValueProvider<String> databaseId) { SpannerConfig config = getSpannerConfig(); return withSpannerConfig(config.withDatabaseId(databaseId)); }
SpannerIO.read() .withInstanceId(instanceId) .withDatabaseId(databaseId) .withTable("Singers") .withColumns("singerId", "firstName", "lastName"));
mutations.apply(SpannerIO.write() .withInstanceId(instanceId) .withDatabaseId(databaseId) .grouped());
public static void main(String[] args) { Options options = PipelineOptionsFactory.fromArgs(args).withValidation().as(Options.class); Pipeline p = Pipeline.create(options); String instanceId = options.getInstanceId(); String databaseId = options.getDatabaseId(); // [START spanner_dataflow_read] // Query for all the columns and rows in the specified Spanner table PCollection<Struct> records = p.apply( SpannerIO.read() .withInstanceId(instanceId) .withDatabaseId(databaseId) .withQuery("SELECT * FROM " + options.getTable())); // [END spanner_dataflow_read] PCollection<Long> tableEstimatedSize = records // Estimate the size of every row .apply(EstimateSize.create()) // Sum all the row sizes to get the total estimated size of the table .apply(Sum.longsGlobally()); // Write the total size to a file tableEstimatedSize .apply(ToString.elements()) .apply(TextIO.write().to(options.getOutput()).withoutSharding()); p.run().waitUntilFinish(); } }
/** Specifies the Cloud Spanner instance. */ public Read withInstanceId(ValueProvider<String> instanceId) { SpannerConfig config = getSpannerConfig(); return withSpannerConfig(config.withInstanceId(instanceId)); }
/** Specifies the Cloud Spanner instance. */ public CreateTransaction withInstanceId(ValueProvider<String> instanceId) { SpannerConfig config = getSpannerConfig(); return withSpannerConfig(config.withInstanceId(instanceId)); }
.apply("WriteSingers", SpannerIO.write() .withInstanceId(instanceId) .withDatabaseId(databaseId)); .apply("WriteAlbums", SpannerIO.write() .withInstanceId(instanceId) .withDatabaseId(databaseId));
/** Specifies the Cloud Spanner project. */ public Read withProjectId(ValueProvider<String> projectId) { SpannerConfig config = getSpannerConfig(); return withSpannerConfig(config.withProjectId(projectId)); }
private SpannerSchema getSchema() { return SpannerSchema.builder() .addColumn("tEsT", "key", "INT64", CELLS_PER_KEY) .addKeyPart("tEsT", "key", false) .build(); }
/** Specifies the Cloud Spanner instance. */ public Write withInstanceId(ValueProvider<String> instanceId) { SpannerConfig config = getSpannerConfig(); return withSpannerConfig(config.withInstanceId(instanceId)); }
/** Specifies the Cloud Spanner database. */ public Write withDatabaseId(ValueProvider<String> databaseId) { SpannerConfig config = getSpannerConfig(); return withSpannerConfig(config.withDatabaseId(databaseId)); }
@Override byte[] encodeIncreasing() { OrderedCode orderedCode = new OrderedCode(); orderedCode.writeNumIncreasing(value()); return orderedCode.getEncodedBytes(); }
@Override byte[] encodeDecreasing() { OrderedCode orderedCode = new OrderedCode(); orderedCode.writeBytesDecreasing(bytesFromHexString(value())); return orderedCode.getEncodedBytes(); }
/** Specifies the Cloud Spanner project. */ public Write withProjectId(ValueProvider<String> projectId) { SpannerConfig config = getSpannerConfig(); return withSpannerConfig(config.withProjectId(projectId)); }
/** * Specifies the batch size limit (max number of bytes mutated per batch). Default value is 1MB */ public Write withBatchSizeBytes(long batchSizeBytes) { return toBuilder().setBatchSizeBytes(batchSizeBytes).build(); }
/** * By default Batch API is used to read data from Cloud Spanner. It is useful to disable * batching when the underlying query is not root-partitionable. */ public ReadAll withBatching(boolean batching) { return toBuilder().setBatching(batching).build(); }
/** * Specifies the cell mutation limit (maximum number of mutated cells per batch). Default value * is 5000 */ public Write withMaxNumMutations(long maxNumMutations) { return toBuilder().setMaxNumMutations(maxNumMutations).build(); }
@Override public MutationGroup apply(String userId) { // Immediately block the user. Mutation userMutation = Mutation.newUpdateBuilder("Users") .set("id").to(userId) .set("state").to("BLOCKED") .build(); long generatedId = Hashing.sha1().newHasher() .putString(userId, Charsets.UTF_8) .putLong(timestamp.getSeconds()) .putLong(timestamp.getNanos()) .hash() .asLong(); // Add an entry to pending review requests. Mutation pendingReview = Mutation.newInsertOrUpdateBuilder("PendingReviews") .set("id").to(generatedId) // Must be deterministically generated. .set("userId").to(userId) .set("action").to("REVIEW ACCOUNT") .set("note").to("Suspicious activity detected.") .build(); return MutationGroup.create(userMutation, pendingReview); } }));