bigtableOptionsBuilder.setProjectId(getValue(configuration, PROJECT_ID_KEY, "Project ID")); bigtableOptionsBuilder.setInstanceId(getValue(configuration, INSTANCE_ID_KEY, "Instance ID")); String appProfileId = configuration.get(APP_PROFILE_ID_KEY, null); if (appProfileId != null) { bigtableOptionsBuilder.setAppProfileId(appProfileId); bigtableOptionsBuilder.setDataHost( getHost(configuration, BIGTABLE_HOST_KEY, BIGTABLE_DATA_HOST_DEFAULT, "API Data")); bigtableOptionsBuilder.setAdminHost(getHost( configuration, BIGTABLE_ADMIN_HOST_KEY, BIGTABLE_ADMIN_HOST_DEFAULT, "Admin")); bigtableOptionsBuilder.setPort(port); bigtableOptionsBuilder.setUsePlaintextNegotiation( configuration.getBoolean(BIGTABLE_USE_PLAINTEXT_NEGOTIATION, false)); bigtableOptionsBuilder.enableEmulator(emulatorHost); bigtableOptionsBuilder.setUseBatch(configuration.getBoolean(BIGTABLE_USE_BATCH, false)); return bigtableOptionsBuilder.build();
builder.setCredentialOptions(CredentialOptions.credential(credentials)); } else if (configuration.get(BIGTABLE_SERVICE_ACCOUNT_JSON_VALUE_KEY) != null) { String jsonValue = configuration.get(BIGTABLE_SERVICE_ACCOUNT_JSON_VALUE_KEY); LOG.debug("Using json value"); builder.setCredentialOptions( CredentialOptions.jsonCredentials(jsonValue)); } else if (configuration.get(BIGTABLE_SERVICE_ACCOUNT_JSON_KEYFILE_LOCATION_KEY) != null) { configuration.get(BIGTABLE_SERVICE_ACCOUNT_JSON_KEYFILE_LOCATION_KEY); LOG.debug("Using json keyfile: %s", keyFileLocation); builder.setCredentialOptions( CredentialOptions.jsonCredentials( new FileInputStream(keyFileLocation))); "Key file location must be specified when setting service account email"); LOG.debug("Using p12 keyfile: %s", keyFileLocation); builder.setCredentialOptions( CredentialOptions.p12Credential(serviceAccount, keyFileLocation)); } else { LOG.debug("Using default credentials."); builder.setCredentialOptions( CredentialOptions.defaultCredentials()); builder.setCredentialOptions(CredentialOptions.nullCredential()); LOG.info("Enabling the use of null credentials. This should not be used in production."); } else {
@Before public void setup() { MockitoAnnotations.initMocks(this); BigtableOptions options = new BigtableOptions.Builder().setProjectId("project").setInstanceId("instance").build(); when(mockSession.getOptions()).thenReturn(options); when(mockSession.createBulkMutation(eq(TABLE_NAME))).thenReturn(mockBulkMutation); when(mockSession.getDataClient()).thenReturn(mockBigtableDataClient); }
* bigtableOptionsBuilder.getDataChannelCount(); int maxInflightRpcs = configuration.getInt(MAX_INFLIGHT_RPCS_KEY, defaultRpcCount); bulkOptionsBuilder.setMaxInflightRpcs(maxInflightRpcs); bigtableOptionsBuilder.setBulkOptions(bulkOptionsBuilder.build());
* bigtableOptionsBuilder.getDataChannelCount(); int maxInflightRpcs = configuration.getInt(MAX_INFLIGHT_RPCS_KEY, defaultRpcCount); bulkOptionsBuilder.setMaxInflightRpcs(maxInflightRpcs); bigtableOptionsBuilder.setBulkOptions(bulkOptionsBuilder.build());
builder.setCredentialOptions(CredentialOptions.credential(credentials)); } else if (configuration.get(BIGTABLE_SERVICE_ACCOUNT_JSON_VALUE_KEY) != null) { String jsonValue = configuration.get(BIGTABLE_SERVICE_ACCOUNT_JSON_VALUE_KEY); LOG.debug("Using json value"); builder.setCredentialOptions( CredentialOptions.jsonCredentials(jsonValue)); } else if (configuration.get(BIGTABLE_SERVICE_ACCOUNT_JSON_KEYFILE_LOCATION_KEY) != null) { configuration.get(BIGTABLE_SERVICE_ACCOUNT_JSON_KEYFILE_LOCATION_KEY); LOG.debug("Using json keyfile: %s", keyFileLocation); builder.setCredentialOptions( CredentialOptions.jsonCredentials( new FileInputStream(keyFileLocation))); "Key file location must be specified when setting service account email"); LOG.debug("Using p12 keyfile: %s", keyFileLocation); builder.setCredentialOptions( CredentialOptions.p12Credential(serviceAccount, keyFileLocation)); } else { LOG.debug("Using default credentials."); builder.setCredentialOptions( CredentialOptions.defaultCredentials()); builder.setCredentialOptions(CredentialOptions.nullCredential()); LOG.info("Enabling the use of null credentials. This should not be used in production."); } else {
.withTableId(TABLE_ID) .withProjectId(PROJECT_ID) .withBigtableOptions(new BigtableOptions.Builder().setInstanceId("instance_id").build()) .isDataAccessible()); assertTrue( .withTableId(TABLE_ID) .withInstanceId(INSTANCE_ID) .withBigtableOptions(new BigtableOptions.Builder().setProjectId("project_id").build()) .isDataAccessible()); assertTrue( .withTableId(TABLE_ID) .withBigtableOptions( new BigtableOptions.Builder() .setProjectId("project_id") .setInstanceId("instance_id") .build()) .isDataAccessible());
/** Tests that credentials are used from PipelineOptions if not supplied by BigtableOptions. */ @Test public void testUsePipelineOptionsCredentialsIfNotSpecifiedInBigtableOptions() throws Exception { BigtableOptions options = BIGTABLE_OPTIONS .toBuilder() .setCredentialOptions(CredentialOptions.defaultCredentials()) .build(); GcpOptions pipelineOptions = PipelineOptionsFactory.as(GcpOptions.class); pipelineOptions.setGcpCredential(new TestCredential()); BigtableService readService = BigtableIO.read() .withBigtableOptions(options) .withTableId("TEST-TABLE") .getBigtableConfig() .getBigtableService(pipelineOptions); BigtableService writeService = BigtableIO.write() .withBigtableOptions(options) .withTableId("TEST-TABLE") .getBigtableConfig() .getBigtableService(pipelineOptions); assertEquals( CredentialType.SuppliedCredentials, readService.getBigtableOptions().getCredentialOptions().getCredentialType()); assertEquals( CredentialType.SuppliedCredentials, writeService.getBigtableOptions().getCredentialOptions().getCredentialType()); }
bigtableOptionsBuilder.setProjectId(getValue(configuration, PROJECT_ID_KEY, "Project ID")); bigtableOptionsBuilder.setInstanceId(getValue(configuration, INSTANCE_ID_KEY, "Instance ID")); String appProfileId = configuration.get(APP_PROFILE_ID_KEY, null); if (appProfileId != null) { bigtableOptionsBuilder.setAppProfileId(appProfileId); bigtableOptionsBuilder.setDataHost( getHost(configuration, BIGTABLE_HOST_KEY, BIGTABLE_DATA_HOST_DEFAULT, "API Data")); bigtableOptionsBuilder.setAdminHost(getHost( configuration, BIGTABLE_ADMIN_HOST_KEY, BIGTABLE_ADMIN_HOST_DEFAULT, "Admin")); bigtableOptionsBuilder.setPort(port); bigtableOptionsBuilder.setUsePlaintextNegotiation( configuration.getBoolean(BIGTABLE_USE_PLAINTEXT_NEGOTIATION, false)); bigtableOptionsBuilder.enableEmulator(emulatorHost); bigtableOptionsBuilder.setUseBatch(configuration.getBoolean(BIGTABLE_USE_BATCH, false)); return bigtableOptionsBuilder.build();
/** Tests that credentials are not used from PipelineOptions if supplied by BigtableOptions. */ @Test public void testDontUsePipelineOptionsCredentialsIfSpecifiedInBigtableOptions() throws Exception { BigtableOptions options = BIGTABLE_OPTIONS .toBuilder() .setCredentialOptions(CredentialOptions.nullCredential()) .build(); GcpOptions pipelineOptions = PipelineOptionsFactory.as(GcpOptions.class); pipelineOptions.setGcpCredential(new TestCredential()); BigtableService readService = BigtableIO.read() .withBigtableOptions(options) .withTableId("TEST-TABLE") .getBigtableConfig() .getBigtableService(pipelineOptions); BigtableService writeService = BigtableIO.write() .withBigtableOptions(options) .withTableId("TEST-TABLE") .getBigtableConfig() .getBigtableService(pipelineOptions); assertEquals( CredentialType.None, readService.getBigtableOptions().getCredentialOptions().getCredentialType()); assertEquals( CredentialType.None, writeService.getBigtableOptions().getCredentialOptions().getCredentialType()); }
private static void setChannelOptions(Configuration configuration, BigtableOptions.Builder builder) throws IOException { setCredentialOptions(builder, configuration); builder.setRetryOptions(createRetryOptions(configuration)); int channelCount = configuration.getInt( BIGTABLE_DATA_CHANNEL_COUNT_KEY, BigtableOptions.BIGTABLE_DATA_CHANNEL_COUNT_DEFAULT); builder.setDataChannelCount(channelCount); // This is primarily used by Dataflow where connections open and close often. This is a // performance optimization that will reduce the cost to open connections. boolean useCachedDataPool = configuration.getBoolean(BIGTABLE_USE_CACHED_DATA_CHANNEL_POOL, false); builder.setUseCachedDataPool(useCachedDataPool); // This information is in addition to bigtable-client-core version, and jdk version. StringBuilder agentBuilder = new StringBuilder(); agentBuilder.append("hbase-").append(VersionInfo.getVersion()); String customUserAgent = configuration.get(CUSTOM_USER_AGENT_KEY); if (customUserAgent != null) { agentBuilder.append(',').append(customUserAgent); } builder.setUserAgent(agentBuilder.toString()); }
private static void setChannelOptions(Configuration configuration, BigtableOptions.Builder builder) throws IOException { setCredentialOptions(builder, configuration); builder.setRetryOptions(createRetryOptions(configuration)); int channelCount = configuration.getInt( BIGTABLE_DATA_CHANNEL_COUNT_KEY, BigtableOptions.BIGTABLE_DATA_CHANNEL_COUNT_DEFAULT); builder.setDataChannelCount(channelCount); // This is primarily used by Dataflow where connections open and close often. This is a // performance optimization that will reduce the cost to open connections. boolean useCachedDataPool = configuration.getBoolean(BIGTABLE_USE_CACHED_DATA_CHANNEL_POOL, false); builder.setUseCachedDataPool(useCachedDataPool); // This information is in addition to bigtable-client-core version, and jdk version. StringBuilder agentBuilder = new StringBuilder(); agentBuilder.append("hbase-").append(VersionInfo.getVersion()); String customUserAgent = configuration.get(CUSTOM_USER_AGENT_KEY); if (customUserAgent != null) { agentBuilder.append(',').append(customUserAgent); } builder.setUserAgent(agentBuilder.toString()); }
/** * WARNING: Should be used only to specify additional parameters for connection to the Cloud * Bigtable, instanceId and projectId should be provided over {@link #withInstanceId} and {@link * #withProjectId} respectively. * * <p>Returns a new {@link BigtableIO.Write} that will write to the Cloud Bigtable instance * indicated by the given options, and using any other specified customizations. * * <p>Clones the given {@link BigtableOptions} builder so that any further changes will have no * effect on the returned {@link BigtableIO.Write}. * * <p>Does not modify this object. * * @deprecated will be replaced by bigtable options configurator. */ @Deprecated public Write withBigtableOptions(BigtableOptions.Builder optionsBuilder) { BigtableConfig config = getBigtableConfig(); // TODO: is there a better way to clone a Builder? Want it to be immune from user changes. return toBuilder() .setBigtableConfig(config.withBigtableOptions(optionsBuilder.build().toBuilder().build())) .build(); }
/** * WARNING: Should be used only to specify additional parameters for connection to the Cloud * Bigtable, instanceId and projectId should be provided over {@link #withInstanceId} and {@link * #withProjectId} respectively. * * <p>Returns a new {@link BigtableIO.Read} that will read from the Cloud Bigtable instance * indicated by the given options, and using any other specified customizations. * * <p>Clones the given {@link BigtableOptions} builder so that any further changes will have no * effect on the returned {@link BigtableIO.Read}. * * <p>Does not modify this object. * * @deprecated will be replaced by bigtable options configurator. */ @Deprecated public Read withBigtableOptions(BigtableOptions.Builder optionsBuilder) { BigtableConfig config = getBigtableConfig(); // TODO: is there a better way to clone a Builder? Want it to be immune from user changes. return toBuilder() .setBigtableConfig(config.withBigtableOptions(optionsBuilder.build().toBuilder().build())) .build(); }
/** * Helper function that either returns the mock Bigtable service supplied by {@link * #withBigtableService} or creates and returns an implementation that talks to {@code Cloud * Bigtable}. * * <p>Also populate the credentials option from {@link GcpOptions#getGcpCredential()} if the * default credentials are being used on {@link BigtableOptions}. */ @VisibleForTesting BigtableService getBigtableService(PipelineOptions pipelineOptions) { if (getBigtableService() != null) { return getBigtableService(); } BigtableOptions.Builder bigtableOptions = effectiveUserProvidedBigtableOptions(); bigtableOptions.setUserAgent(pipelineOptions.getUserAgent()); if (bigtableOptions.build().getCredentialOptions().getCredentialType() == CredentialOptions.CredentialType.DefaultCredentials) { bigtableOptions.setCredentialOptions( CredentialOptions.credential(pipelineOptions.as(GcpOptions.class).getGcpCredential())); } // Default option that should be forced bigtableOptions.setUseCachedDataPool(true); return new BigtableServiceImpl(bigtableOptions.build()); }
@Test public void testWriteWithBigTableOptionsSetsBulkOptionsAndRetryOptions() { final int maxInflightRpcs = 1; final int initialBackoffMillis = -1; BigtableOptions.Builder optionsBuilder = BIGTABLE_OPTIONS.toBuilder(); BulkOptions.Builder bulkOptionsBuilder = new BulkOptions.Builder(); bulkOptionsBuilder.setMaxInflightRpcs(maxInflightRpcs); RetryOptions.Builder retryOptionsBuilder = new RetryOptions.Builder(); retryOptionsBuilder.setInitialBackoffMillis(initialBackoffMillis); optionsBuilder .setBulkOptions(bulkOptionsBuilder.build()) .setRetryOptions(retryOptionsBuilder.build()); BigtableIO.Write write = BigtableIO.write().withBigtableOptions(optionsBuilder.build()); BigtableOptions options = write.getBigtableOptions(); assertEquals(true, options.getBulkOptions().useBulkApi()); assertEquals(maxInflightRpcs, options.getBulkOptions().getMaxInflightRpcs()); assertEquals(initialBackoffMillis, options.getRetryOptions().getInitialBackoffMillis()); assertThat( options.getBulkOptions(), Matchers.equalTo(bulkOptionsBuilder.setUseBulkApi(true).build())); assertThat(options.getRetryOptions(), Matchers.equalTo(retryOptionsBuilder.build())); }
@Test public void testE2EBigtableRead() throws Exception { PipelineOptionsFactory.register(BigtableTestOptions.class); BigtableTestOptions options = TestPipeline.testingPipelineOptions().as(BigtableTestOptions.class); String project = options.getBigtableProject(); if (project.equals("")) { project = options.as(GcpOptions.class).getProject(); } BigtableOptions.Builder bigtableOptionsBuilder = new BigtableOptions.Builder().setProjectId(project).setInstanceId(options.getInstanceId()); final String tableId = "BigtableReadTest"; final long numRows = 1000L; Pipeline p = Pipeline.create(options); PCollection<Long> count = p.apply(BigtableIO.read().withBigtableOptions(bigtableOptionsBuilder).withTableId(tableId)) .apply(Count.globally()); PAssert.thatSingleton(count).isEqualTo(numRows); p.run(); } }
@Test public void testGetBigtableServiceWithConfigurator() { SerializableFunction<BigtableOptions.Builder, BigtableOptions.Builder> configurator = (SerializableFunction<BigtableOptions.Builder, BigtableOptions.Builder>) input -> input .setInstanceId(INSTANCE_ID.get() + INSTANCE_ID.get()) .setProjectId(PROJECT_ID.get() + PROJECT_ID.get()) .setBulkOptions(new BulkOptions.Builder().setUseBulkApi(true).build()); BigtableService service = config .withProjectId(PROJECT_ID) .withInstanceId(INSTANCE_ID) .withBigtableOptionsConfigurator(configurator) .getBigtableService(PipelineOptionsFactory.as(GcpOptions.class)); assertEquals(PROJECT_ID.get(), service.getBigtableOptions().getProjectId()); assertEquals(INSTANCE_ID.get(), service.getBigtableOptions().getInstanceId()); assertEquals(true, service.getBigtableOptions().getBulkOptions().useBulkApi()); }
@Before public void setup() throws Exception { PipelineOptionsFactory.register(BigtableTestOptions.class); options = TestPipeline.testingPipelineOptions().as(BigtableTestOptions.class); project = options.as(GcpOptions.class).getProject(); bigtableOptions = new Builder() .setProjectId(project) .setInstanceId(options.getInstanceId()) .setUserAgent("apache-beam-test") .build(); session = new BigtableSession( bigtableOptions .toBuilder() .setCredentialOptions( CredentialOptions.credential(options.as(GcpOptions.class).getGcpCredential())) .build()); tableAdminClient = session.getTableAdminClient(); }
private BigtableOptions.Builder effectiveUserProvidedBigtableOptions() { BigtableOptions.Builder effectiveOptions = getBigtableOptions() != null ? getBigtableOptions().toBuilder() : new BigtableOptions.Builder(); if (getBigtableOptionsConfigurator() != null) { effectiveOptions = getBigtableOptionsConfigurator().apply(effectiveOptions); } if (getInstanceId() != null) { effectiveOptions.setInstanceId(getInstanceId().get()); } if (getProjectId() != null) { effectiveOptions.setProjectId(getProjectId().get()); } return effectiveOptions; }