"tx", SpannerIO.createTransaction() .withSpannerConfig(spannerConfig) .withTimestampBound(timestampBound));
@VisibleForTesting CreateTransaction withServiceFactory(ServiceFactory<Spanner, SpannerOptions> serviceFactory) { SpannerConfig config = getSpannerConfig(); return withSpannerConfig(config.withServiceFactory(serviceFactory)); }
Pipeline p = begin.getPipeline(); PCollectionView<Transaction> tx = p.apply(SpannerIO.createTransaction().withSpannerConfig(spannerConfig)); PCollection<Ddl> ddl = p.apply("Read Information Schema", new ReadInformationSchema(spannerConfig, tx));
begin.apply(SpannerIO.createTransaction().withSpannerConfig(spannerConfig));
@Test public void testReadAllRecordsInDb() throws Exception { SpannerConfig spannerConfig = createSpannerConfig(); PCollectionView<Transaction> tx = p.apply( SpannerIO.createTransaction() .withSpannerConfig(spannerConfig) .withTimestampBound(TimestampBound.strong())); PCollection<Struct> allRecords = p.apply( SpannerIO.read() .withSpannerConfig(spannerConfig) .withBatching(false) .withQuery( "SELECT t.table_name FROM information_schema.tables AS t WHERE t" + ".table_catalog = '' AND t.table_schema = ''")) .apply( MapElements.into(TypeDescriptor.of(ReadOperation.class)) .via( (SerializableFunction<Struct, ReadOperation>) input -> { String tableName = input.getString(0); return ReadOperation.create().withQuery("SELECT * FROM " + tableName); })) .apply(SpannerIO.readAll().withTransaction(tx).withSpannerConfig(spannerConfig)); PAssert.thatSingleton(allRecords.apply("Count rows", Count.globally())).isEqualTo(5L); p.run(); }
@Override public PCollection<Struct> expand(PCollection<ReadOperation> input) { PCollectionView<Transaction> txView = getTxView(); if (txView == null) { Pipeline begin = input.getPipeline(); SpannerIO.CreateTransaction createTx = SpannerIO.createTransaction() .withSpannerConfig(getSpannerConfig()) .withTimestampBound(getTimestampBound()); txView = begin.apply(createTx); } return input .apply( "Generate Partitions", ParDo.of(new GeneratePartitionsFn(getSpannerConfig(), txView)).withSideInputs(txView)) .apply("Shuffle partitions", Reshuffle.<Partition>viaRandomKey()) .apply( "Read from Partitions", ParDo.of(new ReadFromPartitionFn(getSpannerConfig(), txView)).withSideInputs(txView)); }
@Test public void testQuery() throws Exception { SpannerConfig spannerConfig = createSpannerConfig(); PCollectionView<Transaction> tx = p.apply( SpannerIO.createTransaction() .withSpannerConfig(spannerConfig) .withTimestampBound(TimestampBound.strong())); PCollection<Struct> output = p.apply( SpannerIO.read() .withSpannerConfig(spannerConfig) .withQuery("SELECT * FROM " + options.getTable()) .withTransaction(tx)); PAssert.thatSingleton(output.apply("Count rows", Count.globally())).isEqualTo(5L); p.run(); }
@Test public void testRead() throws Exception { SpannerConfig spannerConfig = createSpannerConfig(); PCollectionView<Transaction> tx = p.apply( SpannerIO.createTransaction() .withSpannerConfig(spannerConfig) .withTimestampBound(TimestampBound.strong())); PCollection<Struct> output = p.apply( SpannerIO.read() .withSpannerConfig(spannerConfig) .withTable(options.getTable()) .withColumns("Key", "Value") .withTransaction(tx)); PAssert.thatSingleton(output.apply("Count rows", Count.<Struct>globally())).isEqualTo(5L); p.run(); }
public CreateTransaction withTimestampBound(TimestampBound timestampBound) { return toBuilder().setTimestampBound(timestampBound).build(); }
@Override public PCollection<Struct> expand(PCollection<ReadOperation> input) { PCollectionView<Transaction> txView = getTxView(); if (txView == null) { Pipeline begin = input.getPipeline(); SpannerIO.CreateTransaction createTx = SpannerIO.createTransaction() .withSpannerConfig(getSpannerConfig()) .withTimestampBound(getTimestampBound()); txView = begin.apply(createTx); } return input.apply( "Naive read from Cloud Spanner", ParDo.of(new NaiveSpannerReadFn(getSpannerConfig(), txView)).withSideInputs(txView)); }
@DoFn.Setup public void setup() throws Exception { spannerAccessor = config.getSpannerConfig().connectToSpanner(); }
public CreateTransaction withHost(String host) { return withHost(ValueProvider.StaticValueProvider.of(host)); }
/** Specifies the Cloud Spanner host. */ public CreateTransaction withHost(ValueProvider<String> host) { SpannerConfig config = getSpannerConfig(); return withSpannerConfig(config.withHost(host)); }
/** Specifies the Cloud Spanner configuration. */ public CreateTransaction withSpannerConfig(SpannerConfig spannerConfig) { return toBuilder().setSpannerConfig(spannerConfig).build(); }
/** Specifies the Cloud Spanner database. */ public CreateTransaction withDatabaseId(String databaseId) { return withDatabaseId(ValueProvider.StaticValueProvider.of(databaseId)); }
/** Specifies the Cloud Spanner database. */ public CreateTransaction withDatabaseId(ValueProvider<String> databaseId) { SpannerConfig config = getSpannerConfig(); return withSpannerConfig(config.withDatabaseId(databaseId)); }
/** Specifies the Cloud Spanner project. */ public CreateTransaction withProjectId(ValueProvider<String> projectId) { SpannerConfig config = getSpannerConfig(); return withSpannerConfig(config.withProjectId(projectId)); }
@ProcessElement public void processElement(ProcessContext c) throws Exception { BatchReadOnlyTransaction tx = spannerAccessor.getBatchClient().batchReadOnlyTransaction(config.getTimestampBound()); c.output(Transaction.create(tx.getBatchTransactionId())); } }
@Override public PCollectionView<Transaction> expand(PBegin input) { getSpannerConfig().validate(); return input .apply(Create.of(1)) .apply("Create transaction", ParDo.of(new CreateTransactionFn(this))) .apply("As PCollectionView", View.asSingleton()); }
/** Specifies the Cloud Spanner instance. */ public CreateTransaction withInstanceId(ValueProvider<String> instanceId) { SpannerConfig config = getSpannerConfig(); return withSpannerConfig(config.withInstanceId(instanceId)); }