@Before public void setup() { MockitoAnnotations.initMocks(this); BigtableOptions options = new BigtableOptions.Builder().setProjectId("project").setInstanceId("instance").build(); when(mockSession.getOptions()).thenReturn(options); when(mockSession.createBulkMutation(eq(TABLE_NAME))).thenReturn(mockBulkMutation); when(mockSession.getDataClient()).thenReturn(mockBigtableDataClient); }
return bigtableOptionsBuilder.build();
static SerializableFunction<BigtableOptions.Builder, BigtableOptions.Builder> enableBulkApiConfigurator( final @Nullable SerializableFunction<BigtableOptions.Builder, BigtableOptions.Builder> userConfigurator) { return optionsBuilder -> { if (userConfigurator != null) { optionsBuilder = userConfigurator.apply(optionsBuilder); } return optionsBuilder.setBulkOptions( optionsBuilder.build().getBulkOptions().toBuilder().setUseBulkApi(true).build()); }; }
/** Tests that credentials are used from PipelineOptions if not supplied by BigtableOptions. */ @Test public void testUsePipelineOptionsCredentialsIfNotSpecifiedInBigtableOptions() throws Exception { BigtableOptions options = BIGTABLE_OPTIONS .toBuilder() .setCredentialOptions(CredentialOptions.defaultCredentials()) .build(); GcpOptions pipelineOptions = PipelineOptionsFactory.as(GcpOptions.class); pipelineOptions.setGcpCredential(new TestCredential()); BigtableService readService = BigtableIO.read() .withBigtableOptions(options) .withTableId("TEST-TABLE") .getBigtableConfig() .getBigtableService(pipelineOptions); BigtableService writeService = BigtableIO.write() .withBigtableOptions(options) .withTableId("TEST-TABLE") .getBigtableConfig() .getBigtableService(pipelineOptions); assertEquals( CredentialType.SuppliedCredentials, readService.getBigtableOptions().getCredentialOptions().getCredentialType()); assertEquals( CredentialType.SuppliedCredentials, writeService.getBigtableOptions().getCredentialOptions().getCredentialType()); }
/** Tests that credentials are not used from PipelineOptions if supplied by BigtableOptions. */ @Test public void testDontUsePipelineOptionsCredentialsIfSpecifiedInBigtableOptions() throws Exception { BigtableOptions options = BIGTABLE_OPTIONS .toBuilder() .setCredentialOptions(CredentialOptions.nullCredential()) .build(); GcpOptions pipelineOptions = PipelineOptionsFactory.as(GcpOptions.class); pipelineOptions.setGcpCredential(new TestCredential()); BigtableService readService = BigtableIO.read() .withBigtableOptions(options) .withTableId("TEST-TABLE") .getBigtableConfig() .getBigtableService(pipelineOptions); BigtableService writeService = BigtableIO.write() .withBigtableOptions(options) .withTableId("TEST-TABLE") .getBigtableConfig() .getBigtableService(pipelineOptions); assertEquals( CredentialType.None, readService.getBigtableOptions().getCredentialOptions().getCredentialType()); assertEquals( CredentialType.None, writeService.getBigtableOptions().getCredentialOptions().getCredentialType()); }
/** * Helper function that either returns the mock Bigtable service supplied by {@link * #withBigtableService} or creates and returns an implementation that talks to {@code Cloud * Bigtable}. * * <p>Also populate the credentials option from {@link GcpOptions#getGcpCredential()} if the * default credentials are being used on {@link BigtableOptions}. */ @VisibleForTesting BigtableService getBigtableService(PipelineOptions pipelineOptions) { if (getBigtableService() != null) { return getBigtableService(); } BigtableOptions.Builder bigtableOptions = effectiveUserProvidedBigtableOptions(); bigtableOptions.setUserAgent(pipelineOptions.getUserAgent()); if (bigtableOptions.build().getCredentialOptions().getCredentialType() == CredentialOptions.CredentialType.DefaultCredentials) { bigtableOptions.setCredentialOptions( CredentialOptions.credential(pipelineOptions.as(GcpOptions.class).getGcpCredential())); } // Default option that should be forced bigtableOptions.setUseCachedDataPool(true); return new BigtableServiceImpl(bigtableOptions.build()); }
/** * WARNING: Should be used only to specify additional parameters for connection to the Cloud * Bigtable, instanceId and projectId should be provided over {@link #withInstanceId} and {@link * #withProjectId} respectively. * * <p>Returns a new {@link BigtableIO.Write} that will write to the Cloud Bigtable instance * indicated by the given options, and using any other specified customizations. * * <p>Clones the given {@link BigtableOptions} builder so that any further changes will have no * effect on the returned {@link BigtableIO.Write}. * * <p>Does not modify this object. * * @deprecated will be replaced by bigtable options configurator. */ @Deprecated public Write withBigtableOptions(BigtableOptions.Builder optionsBuilder) { BigtableConfig config = getBigtableConfig(); // TODO: is there a better way to clone a Builder? Want it to be immune from user changes. return toBuilder() .setBigtableConfig(config.withBigtableOptions(optionsBuilder.build().toBuilder().build())) .build(); }
/** * WARNING: Should be used only to specify additional parameters for connection to the Cloud * Bigtable, instanceId and projectId should be provided over {@link #withInstanceId} and {@link * #withProjectId} respectively. * * <p>Returns a new {@link BigtableIO.Read} that will read from the Cloud Bigtable instance * indicated by the given options, and using any other specified customizations. * * <p>Clones the given {@link BigtableOptions} builder so that any further changes will have no * effect on the returned {@link BigtableIO.Read}. * * <p>Does not modify this object. * * @deprecated will be replaced by bigtable options configurator. */ @Deprecated public Read withBigtableOptions(BigtableOptions.Builder optionsBuilder) { BigtableConfig config = getBigtableConfig(); // TODO: is there a better way to clone a Builder? Want it to be immune from user changes. return toBuilder() .setBigtableConfig(config.withBigtableOptions(optionsBuilder.build().toBuilder().build())) .build(); }
@Test public void testWriteWithBigTableOptionsSetsBulkOptionsAndRetryOptions() { final int maxInflightRpcs = 1; final int initialBackoffMillis = -1; BigtableOptions.Builder optionsBuilder = BIGTABLE_OPTIONS.toBuilder(); BulkOptions.Builder bulkOptionsBuilder = new BulkOptions.Builder(); bulkOptionsBuilder.setMaxInflightRpcs(maxInflightRpcs); RetryOptions.Builder retryOptionsBuilder = new RetryOptions.Builder(); retryOptionsBuilder.setInitialBackoffMillis(initialBackoffMillis); optionsBuilder .setBulkOptions(bulkOptionsBuilder.build()) .setRetryOptions(retryOptionsBuilder.build()); BigtableIO.Write write = BigtableIO.write().withBigtableOptions(optionsBuilder.build()); BigtableOptions options = write.getBigtableOptions(); assertEquals(true, options.getBulkOptions().useBulkApi()); assertEquals(maxInflightRpcs, options.getBulkOptions().getMaxInflightRpcs()); assertEquals(initialBackoffMillis, options.getRetryOptions().getInitialBackoffMillis()); assertThat( options.getBulkOptions(), Matchers.equalTo(bulkOptionsBuilder.setUseBulkApi(true).build())); assertThat(options.getRetryOptions(), Matchers.equalTo(retryOptionsBuilder.build())); }
@Before public void setup() throws Exception { PipelineOptionsFactory.register(BigtableTestOptions.class); options = TestPipeline.testingPipelineOptions().as(BigtableTestOptions.class); project = options.as(GcpOptions.class).getProject(); bigtableOptions = new Builder() .setProjectId(project) .setInstanceId(options.getInstanceId()) .setUserAgent("apache-beam-test") .build(); session = new BigtableSession( bigtableOptions .toBuilder() .setCredentialOptions( CredentialOptions.credential(options.as(GcpOptions.class).getGcpCredential())) .build()); tableAdminClient = session.getTableAdminClient(); }
/** * Creates a {@link BigtableClusterUtilities} for all instances in a projectId. * * @param projectId * * @return a {@link BigtableClusterUtilities} for a all instances in a projectId. * @throws GeneralSecurityException if ssl configuration fails * @throws IOException if some aspect of the connection fails. */ public static BigtableClusterUtilities forAllInstances(String projectId) throws IOException, GeneralSecurityException { // '-' means all instanceids. return new BigtableClusterUtilities( BigtableOptions.builder().setProjectId(projectId).setInstanceId("-").build()); }
/** * Creates a {@link BigtableClusterUtilities} for a projectId and an instanceId. * * @param projectId * @param instanceId * * @return a {@link BigtableClusterUtilities} for a specific projectId/instanceId. * @throws GeneralSecurityException if ssl configuration fails * @throws IOException if some aspect of the connection fails. */ public static BigtableClusterUtilities forInstance(String projectId, String instanceId) throws IOException, GeneralSecurityException { return new BigtableClusterUtilities( BigtableOptions.builder().setProjectId(projectId).setInstanceId(instanceId).build()); }
public static BigtableOptions getDefaultOptions() { return builder().build(); }
@Test public void testReadValidationFailsMissingInstanceIdAndProjectId() { BigtableIO.Read read = BigtableIO.read() .withTableId("table") .withBigtableOptions(new BigtableOptions.Builder().build()); thrown.expect(IllegalArgumentException.class); read.expand(null); }
@Test public void testWriteValidationFailsMissingInstanceId() { BigtableIO.Write write = BigtableIO.write() .withTableId("table") .withProjectId("project") .withBigtableOptions(new BigtableOptions.Builder().build()); thrown.expect(IllegalArgumentException.class); write.expand(null); }
@Test public void testWriteValidationFailsMissingInstanceIdAndProjectId() { BigtableIO.Write write = BigtableIO.write() .withTableId("table") .withBigtableOptions(new BigtableOptions.Builder().build()); thrown.expect(IllegalArgumentException.class); write.expand(null); }
@Test public void testReadValidationFailsMissingInstanceId() { BigtableIO.Read read = BigtableIO.read() .withTableId("table") .withProjectId("project") .withBigtableOptions(new BigtableOptions.Builder().build()); thrown.expect(IllegalArgumentException.class); read.expand(null); }
@Test public void testWriteValidationFailsMissingProjectId() { BigtableIO.Write write = BigtableIO.write() .withTableId("table") .withInstanceId("instance") .withBigtableOptions(new BigtableOptions.Builder().build()); thrown.expect(IllegalArgumentException.class); write.expand(null); }
@Test public void testReadWithBigTableOptionsSetsRetryOptions() { final int initialBackoffMillis = -1; BigtableOptions.Builder optionsBuilder = BIGTABLE_OPTIONS.toBuilder(); RetryOptions.Builder retryOptionsBuilder = new RetryOptions.Builder(); retryOptionsBuilder.setInitialBackoffMillis(initialBackoffMillis); optionsBuilder.setRetryOptions(retryOptionsBuilder.build()); BigtableIO.Read read = BigtableIO.read().withBigtableOptions(optionsBuilder.build()); BigtableOptions options = read.getBigtableOptions(); assertEquals(initialBackoffMillis, options.getRetryOptions().getInitialBackoffMillis()); assertThat(options.getRetryOptions(), Matchers.equalTo(retryOptionsBuilder.build())); }
@Test public void testReadValidationFailsMissingProjectId() { BigtableIO.Read read = BigtableIO.read() .withTableId("table") .withInstanceId("instance") .withBigtableOptions(new BigtableOptions.Builder().build()); thrown.expect(IllegalArgumentException.class); read.expand(null); }