protected Builder(ClientContext clientContext) { super(clientContext); deleteLogSettings = UnaryCallSettings.newUnaryCallSettingsBuilder(); writeLogEntriesSettings = BatchingCallSettings.newBuilder(WRITE_LOG_ENTRIES_BATCHING_DESC) .setBatchingSettings(BatchingSettings.newBuilder().build()); listLogEntriesSettings = PagedCallSettings.newBuilder(LIST_LOG_ENTRIES_PAGE_STR_FACT); listMonitoredResourceDescriptorsSettings = PagedCallSettings.newBuilder(LIST_MONITORED_RESOURCE_DESCRIPTORS_PAGE_STR_FACT); listLogsSettings = PagedCallSettings.newBuilder(LIST_LOGS_PAGE_STR_FACT); unaryMethodSettingsBuilders = ImmutableList.<UnaryCallSettings.Builder<?, ?>>of( deleteLogSettings, writeLogEntriesSettings, listLogEntriesSettings, listMonitoredResourceDescriptorsSettings, listLogsSettings); initDefaults(this); }
.publishSettings() .setBatchingSettings( BatchingSettings.newBuilder() .setElementCountThreshold(10L) .setRequestByteThreshold(1024L)
.writeLogEntriesSettings() .setBatchingSettings( BatchingSettings.newBuilder() .setElementCountThreshold(1000L) .setRequestByteThreshold(1048576L)
StatusCode.Code.UNAVAILABLE) .setRetrySettings(retrySettings) .setBatchingSettings(BatchingSettings.newBuilder().setIsEnabled(false).build()); this.publisherStub = GrpcPublisherStub.create(stubSettings.build());
public Publisher getPublisherWithCustomBatchSettings(ProjectTopicName topicName) throws Exception { // [START pubsub_publisher_batch_settings] // Batch settings control how the publisher batches messages long requestBytesThreshold = 5000L; // default : 1 byte long messageCountBatchSize = 10L; // default : 1 message Duration publishDelayThreshold = Duration.ofMillis(100); // default : 1 ms // Publish request get triggered based on request size, messages count & time since last publish BatchingSettings batchingSettings = BatchingSettings.newBuilder() .setElementCountThreshold(messageCountBatchSize) .setRequestByteThreshold(requestBytesThreshold) .setDelayThreshold(publishDelayThreshold) .build(); Publisher publisher = Publisher.newBuilder(topicName).setBatchingSettings(batchingSettings).build(); // [END pubsub_publisher_batch_settings] return publisher; }
private Publisher.Builder getPublisherBuilder(ProcessContext context) { final Long batchSize = context.getProperty(BATCH_SIZE).asLong(); return Publisher.newBuilder(getTopicName(context)) .setCredentialsProvider(FixedCredentialsProvider.create(getGoogleCredentials(context))) .setBatchingSettings(BatchingSettings.newBuilder() .setElementCountThreshold(batchSize) .setIsEnabled(true) .build()); } }
protected Builder(ClientContext clientContext) { super(clientContext); createTopicSettings = UnaryCallSettings.newUnaryCallSettingsBuilder(); updateTopicSettings = UnaryCallSettings.newUnaryCallSettingsBuilder(); publishSettings = BatchingCallSettings.newBuilder(PUBLISH_BATCHING_DESC) .setBatchingSettings(BatchingSettings.newBuilder().build()); getTopicSettings = UnaryCallSettings.newUnaryCallSettingsBuilder(); listTopicsSettings = PagedCallSettings.newBuilder(LIST_TOPICS_PAGE_STR_FACT); listTopicSubscriptionsSettings = PagedCallSettings.newBuilder(LIST_TOPIC_SUBSCRIPTIONS_PAGE_STR_FACT); deleteTopicSettings = UnaryCallSettings.newUnaryCallSettingsBuilder(); setIamPolicySettings = UnaryCallSettings.newUnaryCallSettingsBuilder(); getIamPolicySettings = UnaryCallSettings.newUnaryCallSettingsBuilder(); testIamPermissionsSettings = UnaryCallSettings.newUnaryCallSettingsBuilder(); unaryMethodSettingsBuilders = ImmutableList.<UnaryCallSettings.Builder<?, ?>>of( createTopicSettings, updateTopicSettings, publishSettings, getTopicSettings, listTopicsSettings, listTopicSubscriptionsSettings, deleteTopicSettings, setIamPolicySettings, getIamPolicySettings, testIamPermissionsSettings); initDefaults(this); }
.build(); BatchingSettings batchingSettings = BatchingSettings.newBuilder().build();
@Test public void testPublisherGetters() throws Exception { Publisher.Builder builder = Publisher.newBuilder(TEST_TOPIC); builder.setChannelProvider(TEST_CHANNEL_PROVIDER); builder.setExecutorProvider(SINGLE_THREAD_EXECUTOR); builder.setBatchingSettings( BatchingSettings.newBuilder() .setRequestByteThreshold(10L) .setDelayThreshold(Duration.ofMillis(11)) .setElementCountThreshold(12L) .build()); builder.setCredentialsProvider(NoCredentialsProvider.create()); Publisher publisher = builder.build(); assertEquals(TEST_TOPIC, publisher.getTopicName()); assertEquals(10, (long) publisher.getBatchingSettings().getRequestByteThreshold()); assertEquals(Duration.ofMillis(11), publisher.getBatchingSettings().getDelayThreshold()); assertEquals(12, (long) publisher.getBatchingSettings().getElementCountThreshold()); publisher.shutdown(); publisher.awaitTermination(1, TimeUnit.MINUTES); }
@Bean @ConditionalOnMissingBean(name = "publisherBatchSettings") public BatchingSettings publisherBatchSettings() { BatchingSettings.Builder builder = BatchingSettings.newBuilder(); GcpPubSubProperties.Batching batching = this.gcpPubSubProperties.getPublisher() .getBatching(); FlowControlSettings flowControlSettings = buildFlowControlSettings(batching.getFlowControl()); if (flowControlSettings != null) { builder.setFlowControlSettings(flowControlSettings); } return ifNotNull(batching.getDelayThresholdSeconds(), (x) -> builder.setDelayThreshold(Duration.ofSeconds(x))) .apply(ifNotNull(batching.getElementCountThreshold(), builder::setElementCountThreshold) .apply(ifNotNull(batching.getEnabled(), builder::setIsEnabled) .apply(ifNotNull(batching.getRequestByteThreshold(), builder::setRequestByteThreshold) .apply(false)))) ? builder.build() : null; }
protected Builder(ClientContext clientContext) { super(clientContext); deleteLogSettings = UnaryCallSettings.newUnaryCallSettingsBuilder(); writeLogEntriesSettings = BatchingCallSettings.newBuilder(WRITE_LOG_ENTRIES_BATCHING_DESC) .setBatchingSettings(BatchingSettings.newBuilder().build()); listLogEntriesSettings = PagedCallSettings.newBuilder(LIST_LOG_ENTRIES_PAGE_STR_FACT); listMonitoredResourceDescriptorsSettings = PagedCallSettings.newBuilder(LIST_MONITORED_RESOURCE_DESCRIPTORS_PAGE_STR_FACT); listLogsSettings = PagedCallSettings.newBuilder(LIST_LOGS_PAGE_STR_FACT); unaryMethodSettingsBuilders = ImmutableList.<UnaryCallSettings.Builder<?, ?>>of( deleteLogSettings, writeLogEntriesSettings, listLogEntriesSettings, listMonitoredResourceDescriptorsSettings, listLogsSettings); initDefaults(this); }
private CPSPublisherTask(StartRequest request) { super(request, "gcloud", MetricsHandler.MetricName.PUBLISH_ACK_LATENCY); try { this.publisher = Publisher.defaultBuilder(TopicName.create(request.getProject(), request.getTopic())) .setBatchingSettings( BatchingSettings.newBuilder() .setElementCountThreshold(950L) .setRequestByteThreshold(9500000L) .setDelayThreshold( Duration.ofMillis(Durations.toMillis(request.getPublishBatchDuration()))) .build()) .build(); } catch (Exception e) { throw new RuntimeException(e); } this.payload = ByteString.copyFromUtf8(LoadTestRunner.createMessage(request.getMessageSize())); this.batchSize = request.getPublishBatchSize(); this.messageSize = request.getMessageSize(); this.id = (new Random()).nextInt(); }
@Test public void testNoFlowControlSettings() throws Exception { @SuppressWarnings("unchecked") BatchingDescriptor<Integer, Integer> batchingDescriptor = Mockito.mock(BatchingDescriptor.class); BatchingCallSettings.Builder<Integer, Integer> builder = BatchingCallSettings.newBuilder(batchingDescriptor); BatchingSettings batchingSettings = BatchingSettings.newBuilder().setElementCountThreshold(1L).build(); builder.setBatchingSettings(batchingSettings); BatchingCallSettings settings = builder.build(); Truth.assertThat(settings.getFlowController()).isNotNull(); } }
private Builder() { super((ClientContext) null); fakeMethodSimple = UnaryCallSettings.newUnaryCallSettingsBuilder(); fakePagedMethod = PagedCallSettings.newBuilder(fakePagedListResponseFactory); fakeMethodBatching = BatchingCallSettings.newBuilder(FAKE_BATCHING_DESCRIPTOR) .setBatchingSettings(BatchingSettings.newBuilder().build()); }
@Test public void testEmptyBuilder() { @SuppressWarnings("unchecked") BatchingDescriptor<Integer, Integer> batchingDescriptor = Mockito.mock(BatchingDescriptor.class); BatchingCallSettings.Builder<Integer, Integer> builder = BatchingCallSettings.newBuilder(batchingDescriptor); Truth.assertThat(builder.getBatchingDescriptor()).isSameAs(batchingDescriptor); Truth.assertThat(builder.getBatchingSettings()).isNull(); Truth.assertThat(builder.getFlowController()).isNull(); Truth.assertThat(builder.getRetryableCodes().size()).isEqualTo(0); Truth.assertThat(builder.getRetrySettings()).isNotNull(); BatchingSettings batchingSettings = BatchingSettings.newBuilder().setElementCountThreshold(1L).build(); builder.setBatchingSettings(batchingSettings); BatchingCallSettings settings = builder.build(); Truth.assertThat(settings.getBatchingDescriptor()).isSameAs(batchingDescriptor); Truth.assertThat(settings.getBatchingSettings()).isSameAs(batchingSettings); Truth.assertThat(settings.getFlowController()).isNotNull(); Truth.assertThat(settings.getRetryableCodes().size()).isEqualTo(0); Truth.assertThat(settings.getRetrySettings()).isNotNull(); }
@Test public void batchingDisabled() throws Exception { BatchingSettings batchingSettings = BatchingSettings.newBuilder().setIsEnabled(false).build(); BatchingCallSettings<LabeledIntList, List<Integer>> batchingCallSettings = BatchingCallSettings.newBuilder(SQUARER_BATCHING_DESC) .setBatchingSettings(batchingSettings) .build(); UnaryCallable<LabeledIntList, List<Integer>> callable = FakeCallableFactory.createBatchingCallable( callLabeledIntSquarer, batchingCallSettings, clientContext); ApiFuture<List<Integer>> f1 = callable.futureCall(new LabeledIntList("one", 1, 2)); ApiFuture<List<Integer>> f2 = callable.futureCall(new LabeledIntList("one", 3, 4)); Truth.assertThat(f1.get()).isEqualTo(Arrays.asList(1, 4)); Truth.assertThat(f2.get()).isEqualTo(Arrays.asList(9, 16)); }
public void batchingWithBlockingCallThreshold() throws Exception { BatchingSettings batchingSettings = BatchingSettings.newBuilder() .setDelayThreshold(Duration.ofSeconds(1)) .setElementCountThreshold(2L) .build(); BatchingCallSettings<LabeledIntList, List<Integer>> batchingCallSettings = BatchingCallSettings.newBuilder(SQUARER_BATCHING_DESC) .setBatchingSettings(batchingSettings) .build(); UnaryCallable<LabeledIntList, List<Integer>> callable = FakeCallableFactory.createBatchingCallable( callLabeledIntSquarer, batchingCallSettings, clientContext); ApiFuture<List<Integer>> f1 = callable.futureCall(new LabeledIntList("one", 1)); ApiFuture<List<Integer>> f2 = callable.futureCall(new LabeledIntList("one", 3)); Truth.assertThat(f1.get()).isEqualTo(Arrays.asList(1)); Truth.assertThat(f2.get()).isEqualTo(Arrays.asList(9)); }
@Test public void batching() throws Exception { BatchingSettings batchingSettings = BatchingSettings.newBuilder() .setDelayThreshold(Duration.ofSeconds(1)) .setElementCountThreshold(2L) .build(); BatchingCallSettings<LabeledIntList, List<Integer>> batchingCallSettings = BatchingCallSettings.newBuilder(SQUARER_BATCHING_DESC) .setBatchingSettings(batchingSettings) .build(); UnaryCallable<LabeledIntList, List<Integer>> callable = FakeCallableFactory.createBatchingCallable( callLabeledIntSquarer, batchingCallSettings, clientContext); ApiFuture<List<Integer>> f1 = callable.futureCall(new LabeledIntList("one", 1, 2)); ApiFuture<List<Integer>> f2 = callable.futureCall(new LabeledIntList("one", 3, 4)); Truth.assertThat(f1.get()).isEqualTo(Arrays.asList(1, 4)); Truth.assertThat(f2.get()).isEqualTo(Arrays.asList(9, 16)); }
@Test public void testUnbatchedCall() throws Exception { BatchingSettings batchingSettings = BatchingSettings.newBuilder().setIsEnabled(false).build(); FlowControlSettings flowControlSettings = FlowControlSettings.newBuilder() .setLimitExceededBehavior(LimitExceededBehavior.Ignore) .build(); FlowController flowController = new FlowController(flowControlSettings); BatcherFactory<LabeledIntList, List<Integer>> batcherFactory = new BatcherFactory<>( new SquarerBatchingDescriptor(), batchingSettings, batchingExecutor, flowController); BatchingCallable<LabeledIntList, List<Integer>> batchingCallable = new BatchingCallable<>( FakeBatchableApi.callLabeledIntSquarer, FakeBatchableApi.SQUARER_BATCHING_DESC, batcherFactory); LabeledIntList request1 = new LabeledIntList("label", 2); ApiFuture<List<Integer>> future1 = batchingCallable.futureCall(request1, FakeCallContext.createDefault()); List<Integer> response1 = future1.get(); Truth.assertThat(response1.size()).isEqualTo(1); Truth.assertThat(response1.get(0)).isEqualTo(2 * 2); } }
public static GooglePubsubPublisher buildPublisher(GooglePubsubPublisherConfig config, ObjectMapper mapper) { GooglePubsubPublisher publisher = new GooglePubsubPublisher(); publisher.setName(config.getName()); ProjectTopicName fullName = ProjectTopicName.of(config.getProject(), config.getTopicName()); publisher.setTopicName(config.getTopicName()); publisher.setFullTopicName(fullName.toString()); publisher.setContent(config.getContent()); publisher.setMapper(mapper); BatchingSettings batchingSettings = BatchingSettings.newBuilder() .setElementCountThreshold(config.getBatchCountThreshold()) .setDelayThreshold(Duration.ofMillis(config.getDelayMillisecondsThreshold())) .build(); try { Publisher p = Publisher.newBuilder(fullName) .setCredentialsProvider(new GooglePubsubCredentialsProvider(config.getJsonPath())) .setBatchingSettings(batchingSettings) .build(); publisher.setPublisher(p); } catch (IOException ioe) { log.error("Could not create Google Pubsub Publishers: {}", ioe); } return publisher; }