public Builder setBatchingSettings(BatchingSettings batchingSettings) { Preconditions.checkNotNull(batchingSettings); Preconditions.checkNotNull(batchingSettings.getElementCountThreshold()); Preconditions.checkArgument(batchingSettings.getElementCountThreshold() > 0); Preconditions.checkNotNull(batchingSettings.getRequestByteThreshold()); Preconditions.checkArgument(batchingSettings.getRequestByteThreshold() > 0); Preconditions.checkNotNull(batchingSettings.getDelayThreshold()); Preconditions.checkArgument(batchingSettings.getDelayThreshold().toMillis() > 0); this.batchingSettings = batchingSettings; return this; }
/** Return the target table name. This will be used to combine batcheable requests */ @Override public PartitionKey getBatchPartitionKey(MutateRowsRequest request) { return new PartitionKey(request.getTableName()); }
private long getMaxBatchBytes() { return getBatchingSettings().getRequestByteThreshold(); }
.writeLogEntriesSettings() .setBatchingSettings( BatchingSettings.newBuilder() .setElementCountThreshold(1000L) .setRequestByteThreshold(1048576L) .setDelayThreshold(Duration.ofMillis(50)) .setFlowControlSettings( FlowControlSettings.newBuilder() .setMaxOutstandingElementCount(100000L) .setMaxOutstandingRequestBytes(10485760L) .setLimitExceededBehavior(LimitExceededBehavior.ThrowException) .build()) .build()); builder .writeLogEntriesSettings()
public Publisher getPublisherWithCustomBatchSettings(ProjectTopicName topicName) throws Exception { // [START pubsub_publisher_batch_settings] // Batch settings control how the publisher batches messages long requestBytesThreshold = 5000L; // default : 1 byte long messageCountBatchSize = 10L; // default : 1 message Duration publishDelayThreshold = Duration.ofMillis(100); // default : 1 ms // Publish request get triggered based on request size, messages count & time since last publish BatchingSettings batchingSettings = BatchingSettings.newBuilder() .setElementCountThreshold(messageCountBatchSize) .setRequestByteThreshold(requestBytesThreshold) .setDelayThreshold(publishDelayThreshold) .build(); Publisher publisher = Publisher.newBuilder(topicName).setBatchingSettings(batchingSettings).build(); // [END pubsub_publisher_batch_settings] return publisher; }
@Test public void testPublisherGetters() throws Exception { Publisher.Builder builder = Publisher.newBuilder(TEST_TOPIC); builder.setChannelProvider(TEST_CHANNEL_PROVIDER); builder.setExecutorProvider(SINGLE_THREAD_EXECUTOR); builder.setBatchingSettings( BatchingSettings.newBuilder() .setRequestByteThreshold(10L) .setDelayThreshold(Duration.ofMillis(11)) .setElementCountThreshold(12L) .build()); builder.setCredentialsProvider(NoCredentialsProvider.create()); Publisher publisher = builder.build(); assertEquals(TEST_TOPIC, publisher.getTopicName()); assertEquals(10, (long) publisher.getBatchingSettings().getRequestByteThreshold()); assertEquals(Duration.ofMillis(11), publisher.getBatchingSettings().getDelayThreshold()); assertEquals(12, (long) publisher.getBatchingSettings().getElementCountThreshold()); publisher.shutdown(); publisher.awaitTermination(1, TimeUnit.MINUTES); }
@Test public void testErrorPropagation() throws Exception { Publisher publisher = getTestPublisherBuilder() .setExecutorProvider(SINGLE_THREAD_EXECUTOR) .setBatchingSettings( Publisher.Builder.DEFAULT_BATCHING_SETTINGS .toBuilder() .setElementCountThreshold(1L) .setDelayThreshold(Duration.ofSeconds(5)) .build()) .build(); testPublisherServiceImpl.addPublishError(Status.DATA_LOSS.asException()); try { sendTestMessage(publisher, "A").get(); fail("should throw exception"); } catch (ExecutionException e) { assertThat(e.getCause()).isInstanceOf(DataLossException.class); } }
private Publisher.Builder getPublisherBuilder(ProcessContext context) { final Long batchSize = context.getProperty(BATCH_SIZE).asLong(); return Publisher.newBuilder(getTopicName(context)) .setCredentialsProvider(FixedCredentialsProvider.create(getGoogleCredentials(context))) .setBatchingSettings(BatchingSettings.newBuilder() .setElementCountThreshold(batchSize) .setIsEnabled(true) .build()); } }
protected Builder(ClientContext clientContext) { super(clientContext); deleteLogSettings = UnaryCallSettings.newUnaryCallSettingsBuilder(); writeLogEntriesSettings = BatchingCallSettings.newBuilder(WRITE_LOG_ENTRIES_BATCHING_DESC) .setBatchingSettings(BatchingSettings.newBuilder().build()); listLogEntriesSettings = PagedCallSettings.newBuilder(LIST_LOG_ENTRIES_PAGE_STR_FACT); listMonitoredResourceDescriptorsSettings = PagedCallSettings.newBuilder(LIST_MONITORED_RESOURCE_DESCRIPTORS_PAGE_STR_FACT); listLogsSettings = PagedCallSettings.newBuilder(LIST_LOGS_PAGE_STR_FACT); unaryMethodSettingsBuilders = ImmutableList.<UnaryCallSettings.Builder<?, ?>>of( deleteLogSettings, writeLogEntriesSettings, listLogEntriesSettings, listMonitoredResourceDescriptorsSettings, listLogsSettings); initDefaults(this); }
@Before public void setUp() throws Exception { server = new InProcessServer<>(new BigtableGrpc.BigtableImplBase() {}, NAME); server.start(); channelProvider = LocalChannelProvider.create(NAME); BigtableDataSettings.Builder settings = BigtableDataSettings.newBuilder() .setProjectId(PROJECT_ID) .setInstanceId(INSTANCE_ID) .setTransportChannelProvider(channelProvider) .setCredentialsProvider(NoCredentialsProvider.create()); // Force immediate flush settings .bulkMutationsSettings() .setBatchingSettings( settings .bulkMutationsSettings() .getBatchingSettings() .toBuilder() .setElementCountThreshold(1L) .build()); client = BigtableDataClient.create(settings.build()); }
new FlowController( FlowControlSettings.newBuilder() .setMaxOutstandingElementCount(1L) .setLimitExceededBehavior(FlowController.LimitExceededBehavior.ThrowException) .build());
/** Stop extending deadlines for this message and free flow control. */ private void forget() { if (pendingMessages.remove(ackId) == null) { /* * We're forgetting the message for the second time. Probably because we ran out of total * expiration, forget the message, then the user finishes working on the message, and forget * again. Turn the second forget into a no-op so we don't free twice. */ return; } flowController.release(1, outstandingBytes); messagesWaiter.incrementPendingMessages(-1); processOutstandingBatches(); }
private void setupDurationBasedPublishAlarm() { if (!activeAlarm.getAndSet(true)) { long delayThresholdMs = getBatchingSettings().getDelayThreshold().toMillis(); logger.log(Level.FINER, "Setting up alarm for the next {0} ms.", delayThresholdMs); currentAlarmFuture = executor.schedule( new Runnable() { @Override public void run() { logger.log(Level.FINER, "Sending messages based on schedule."); activeAlarm.getAndSet(false); publishAllOutstanding(); } }, delayThresholdMs, TimeUnit.MILLISECONDS); } }
.publishSettings() .setBatchingSettings( BatchingSettings.newBuilder() .setElementCountThreshold(10L) .setRequestByteThreshold(1024L) .setDelayThreshold(Duration.ofMillis(10)) .setFlowControlSettings( FlowControlSettings.newBuilder() .setLimitExceededBehavior(LimitExceededBehavior.Ignore) .build()) .build()); builder .publishSettings()
@Test public void testPublishFailureRetries() throws Exception { Publisher publisher = getTestPublisherBuilder() .setExecutorProvider(SINGLE_THREAD_EXECUTOR) .setBatchingSettings( Publisher.Builder.DEFAULT_BATCHING_SETTINGS .toBuilder() .setElementCountThreshold(1L) .setDelayThreshold(Duration.ofSeconds(5)) .build()) .build(); // To demonstrate that reaching duration will trigger publish testPublisherServiceImpl.addPublishError(new Throwable("Transiently failing")); testPublisherServiceImpl.addPublishResponse(PublishResponse.newBuilder().addMessageIds("1")); ApiFuture<String> publishFuture1 = sendTestMessage(publisher, "A"); assertEquals("1", publishFuture1.get()); assertEquals(2, testPublisherServiceImpl.getCapturedRequests().size()); publisher.shutdown(); publisher.awaitTermination(1, TimeUnit.MINUTES); }
protected Builder(ClientContext clientContext) { super(clientContext); createTopicSettings = UnaryCallSettings.newUnaryCallSettingsBuilder(); updateTopicSettings = UnaryCallSettings.newUnaryCallSettingsBuilder(); publishSettings = BatchingCallSettings.newBuilder(PUBLISH_BATCHING_DESC) .setBatchingSettings(BatchingSettings.newBuilder().build()); getTopicSettings = UnaryCallSettings.newUnaryCallSettingsBuilder(); listTopicsSettings = PagedCallSettings.newBuilder(LIST_TOPICS_PAGE_STR_FACT); listTopicSubscriptionsSettings = PagedCallSettings.newBuilder(LIST_TOPIC_SUBSCRIPTIONS_PAGE_STR_FACT); deleteTopicSettings = UnaryCallSettings.newUnaryCallSettingsBuilder(); setIamPolicySettings = UnaryCallSettings.newUnaryCallSettingsBuilder(); getIamPolicySettings = UnaryCallSettings.newUnaryCallSettingsBuilder(); testIamPermissionsSettings = UnaryCallSettings.newUnaryCallSettingsBuilder(); unaryMethodSettingsBuilders = ImmutableList.<UnaryCallSettings.Builder<?, ?>>of( createTopicSettings, updateTopicSettings, publishSettings, getTopicSettings, listTopicsSettings, listTopicSubscriptionsSettings, deleteTopicSettings, setIamPolicySettings, getIamPolicySettings, testIamPermissionsSettings); initDefaults(this); }
@Test public void testBuilderParametersAndDefaults() { Publisher.Builder builder = Publisher.newBuilder(TEST_TOPIC); assertEquals(TEST_TOPIC.toString(), builder.topicName); assertEquals(Publisher.Builder.DEFAULT_EXECUTOR_PROVIDER, builder.executorProvider); assertEquals( Publisher.Builder.DEFAULT_REQUEST_BYTES_THRESHOLD, builder.batchingSettings.getRequestByteThreshold().longValue()); assertEquals( Publisher.Builder.DEFAULT_DELAY_THRESHOLD, builder.batchingSettings.getDelayThreshold()); assertEquals( Publisher.Builder.DEFAULT_ELEMENT_COUNT_THRESHOLD, builder.batchingSettings.getElementCountThreshold().longValue()); assertEquals(Publisher.Builder.DEFAULT_RETRY_SETTINGS, builder.retrySettings); }
@Override public PartitionKey getBatchPartitionKey(PublishRequest request) { return new PartitionKey(request.getTopic()); }
@Override public PartitionKey getBatchPartitionKey(WriteLogEntriesRequest request) { return new PartitionKey( request.getLogName(), request.getResource(), request.getLabelsMap()); }
@Test public void partitionKeyTest() { String myTableName = NameUtil.formatTableName("my-project", "my-instance", "my-table"); MutateRowsRequest request = createRequest(2).toBuilder().setTableName(myTableName).build(); PartitionKey actual = descriptor.getBatchPartitionKey(request); assertThat(actual).isEqualTo(new PartitionKey(myTableName)); }