.setBatchingSettings( BatchingSettings.newBuilder() .setElementCountThreshold(10L) .setRequestByteThreshold(1024L) .setDelayThreshold(Duration.ofMillis(10)) .setFlowControlSettings( FlowControlSettings.newBuilder() .setLimitExceededBehavior(LimitExceededBehavior.Ignore) .build()) .build()); builder .publishSettings()
StatusCode.Code.UNAVAILABLE) .setRetrySettings(retrySettings) .setBatchingSettings(BatchingSettings.newBuilder().setIsEnabled(false).build()); this.publisherStub = GrpcPublisherStub.create(stubSettings.build());
oldBatchSettings .toBuilder() .setFlowControlSettings( oldBatchSettings .getFlowControlSettings() .setLimitExceededBehavior(LimitExceededBehavior.Block) .build()) .build());
BatchingSettings batchingSettings = BatchingSettings.newBuilder() .setDelayThreshold(Duration.ofSeconds(1)) .setElementCountThreshold(4L) .setRequestByteThreshold(null) .setFlowControlSettings( FlowControlSettings.newBuilder() .setLimitExceededBehavior(LimitExceededBehavior.Block) .setMaxOutstandingRequestBytes(10L) .build()) .build(); TrackedFlowController trackedFlowController = new TrackedFlowController(batchingSettings.getFlowControlSettings());
.setDelayThreshold(Duration.ofSeconds(10)) .setElementCountThreshold(2L) .setRequestByteThreshold(1000L) .build(); FlowControlSettings flowControlSettings = FlowControlSettings.newBuilder()
.setBatchingSettings( BatchingSettings.newBuilder() .setElementCountThreshold(1000L) .setRequestByteThreshold(1048576L) .setDelayThreshold(Duration.ofMillis(50)) .setFlowControlSettings( FlowControlSettings.newBuilder() .setMaxOutstandingElementCount(100000L) .setLimitExceededBehavior(LimitExceededBehavior.ThrowException) .build()) .build()); builder .writeLogEntriesSettings()
@Test public void testBuilder() { @SuppressWarnings("unchecked") BatchingDescriptor<Integer, Integer> batchingDescriptor = Mockito.mock(BatchingDescriptor.class); BatchingCallSettings.Builder<Integer, Integer> builder = BatchingCallSettings.newBuilder(batchingDescriptor); BatchingSettings batchingSettings = BatchingSettings.newBuilder().setElementCountThreshold(1L).build(); FlowController flowController = Mockito.mock(FlowController.class); Set<StatusCode.Code> retryCodes = Sets.newHashSet(Code.UNAVAILABLE); RetrySettings retrySettings = RetrySettings.newBuilder().build(); builder.setBatchingSettings(batchingSettings); builder.setFlowController(flowController); builder.setRetryableCodes(retryCodes); builder.setRetrySettings(retrySettings); Truth.assertThat(builder.getBatchingDescriptor()).isSameAs(batchingDescriptor); Truth.assertThat(builder.getBatchingSettings()).isSameAs(batchingSettings); Truth.assertThat(builder.getFlowController()).isSameAs(flowController); Truth.assertThat(builder.getRetryableCodes().size()).isEqualTo(1); Truth.assertThat(builder.getRetrySettings()).isSameAs(retrySettings); BatchingCallSettings settings = builder.build(); Truth.assertThat(settings.getBatchingDescriptor()).isSameAs(batchingDescriptor); Truth.assertThat(settings.getBatchingSettings()).isSameAs(batchingSettings); Truth.assertThat(settings.getFlowController()).isSameAs(flowController); Truth.assertThat(settings.getRetryableCodes().size()).isEqualTo(1); Truth.assertThat(settings.getRetrySettings()).isSameAs(retrySettings); }
@Test public void batchingException() throws Exception { BatchingSettings batchingSettings = BatchingSettings.newBuilder() .setDelayThreshold(Duration.ofSeconds(1)) .setElementCountThreshold(2L) .build(); BatchingCallSettings<LabeledIntList, List<Integer>> batchingCallSettings = BatchingCallSettings.newBuilder(SQUARER_BATCHING_DESC) .setBatchingSettings(batchingSettings) .build(); UnaryCallable<LabeledIntList, List<Integer>> callable = FakeCallableFactory.createBatchingCallable( callLabeledIntExceptionThrower, batchingCallSettings, clientContext); ApiFuture<List<Integer>> f1 = callable.futureCall(new LabeledIntList("one", 1, 2)); ApiFuture<List<Integer>> f2 = callable.futureCall(new LabeledIntList("one", 3, 4)); try { f1.get(); Assert.fail("Expected exception from batching call"); } catch (ExecutionException e) { // expected } try { f2.get(); Assert.fail("Expected exception from batching call"); } catch (ExecutionException e) { // expected } } }
private void createPublisher() { ProjectTopicName fullTopic = ProjectTopicName.of(cpsProject, cpsTopic); com.google.cloud.pubsub.v1.Publisher.Builder builder = com.google.cloud.pubsub.v1.Publisher.newBuilder(fullTopic) .setCredentialsProvider(gcpCredentialsProvider) .setBatchingSettings( BatchingSettings.newBuilder() .setDelayThreshold(Duration.ofMillis(maxDelayThresholdMs)) .setElementCountThreshold(maxBufferSize) .setRequestByteThreshold(maxBufferBytes) .build()) .setRetrySettings( RetrySettings.newBuilder() // All values that are not configurable come from the defaults for the publisher // client library. .setTotalTimeout(Duration.ofMillis(maxTotalTimeoutMs)) .setMaxRpcTimeout(Duration.ofMillis(maxRequestTimeoutMs)) .setInitialRetryDelay(Duration.ofMillis(5)) .setRetryDelayMultiplier(2) .setMaxRetryDelay(Duration.ofMillis(Long.MAX_VALUE)) .setInitialRpcTimeout(Duration.ofSeconds(10)) .setRpcTimeoutMultiplier(2) .build()); try { publisher = builder.build(); } catch (Exception e) { throw new RuntimeException(e); } }
private static Builder createDefault() { Builder builder = new Builder(); builder.setTransportChannelProvider(defaultTransportChannelProvider()); builder.setExecutorProvider(defaultExecutorProviderBuilder().build()); builder.setCredentialsProvider(defaultCredentialsProviderBuilder().build()); builder.setHeaderProvider(new NoHeaderProvider()); builder.setInternalHeaderProvider(defaultGoogleServiceHeaderProviderBuilder().build()); builder.setStreamWatchdogProvider(FixedWatchdogProvider.create(null)); builder .fakeMethodSimple() .setRetryableCodes(RETRYABLE_CODE_DEFINITIONS.get("idempotent")) .setRetrySettings(RETRY_PARAM_DEFINITIONS.get("default")); builder .fakePagedMethod() .setRetryableCodes(RETRYABLE_CODE_DEFINITIONS.get("idempotent")) .setRetrySettings(RETRY_PARAM_DEFINITIONS.get("default")); builder .fakeMethodBatching() .setBatchingSettings( BatchingSettings.newBuilder() .setElementCountThreshold(800L) .setRequestByteThreshold(8388608L) .setDelayThreshold(Duration.ofMillis(100)) .build()); builder .fakeMethodBatching() .setRetryableCodes(RETRYABLE_CODE_DEFINITIONS.get("idempotent")) .setRetrySettings(RETRY_PARAM_DEFINITIONS.get("default")); return builder; }
BatchingSettings batchingSettings = BatchingSettings.newBuilder() .setDelayThreshold(Duration.ofSeconds(1)) .setElementCountThreshold(2L) .setRequestByteThreshold(1000L) .build(); FlowControlSettings flowControlSettings = FlowControlSettings.newBuilder()
@Provides @Singleton public Publisher publisher() throws IOException { // Publish request based on request size, messages count & time since last publish BatchingSettings batchingSettings = BatchingSettings.newBuilder() .setElementCountThreshold(messageCountBatchSize) .setRequestByteThreshold(requestBytesThreshold) .setDelayThreshold(publishDelayThreshold) .build(); ExecutorProvider executorProvider = InstantiatingExecutorProvider.newBuilder() .setExecutorThreadCount(1).build(); final Publisher.Builder publisher = Publisher.newBuilder(topicName()) .setBatchingSettings(batchingSettings) .setExecutorProvider(executorProvider); final String emulatorHost = System.getenv("PUBSUB_EMULATOR_HOST"); if (emulatorHost != null) { ManagedChannel channel = ManagedChannelBuilder.forTarget(emulatorHost).usePlaintext().build(); TransportChannelProvider channelProvider = FixedTransportChannelProvider.create(GrpcTransportChannel.create(channel)); publisher.setChannelProvider(channelProvider); publisher.setCredentialsProvider(NoCredentialsProvider.create()); } return publisher.build(); }
@Test public void testBuilderFromSettings() throws Exception { @SuppressWarnings("unchecked") BatchingDescriptor<Integer, Integer> batchingDescriptor = Mockito.mock(BatchingDescriptor.class); BatchingCallSettings.Builder<Integer, Integer> builder = BatchingCallSettings.newBuilder(batchingDescriptor); BatchingSettings batchingSettings = BatchingSettings.newBuilder().setElementCountThreshold(1L).build(); FlowController flowController = Mockito.mock(FlowController.class); Set<StatusCode.Code> retryCodes = Sets.newHashSet(Code.UNAVAILABLE); RetrySettings retrySettings = RetrySettings.newBuilder().build(); builder.setBatchingSettings(batchingSettings); builder.setFlowController(flowController); builder.setRetryableCodes(retryCodes); builder.setRetrySettings(retrySettings); BatchingCallSettings settings = builder.build(); BatchingCallSettings.Builder newBuilder = settings.toBuilder(); Truth.assertThat(newBuilder.getBatchingDescriptor()).isSameAs(batchingDescriptor); Truth.assertThat(newBuilder.getBatchingSettings()).isSameAs(batchingSettings); Truth.assertThat(newBuilder.getFlowController()).isSameAs(flowController); Truth.assertThat(newBuilder.getRetryableCodes().size()).isEqualTo(1); Truth.assertThat(newBuilder.getRetrySettings()).isSameAs(retrySettings); }
@Test public void testUnbatchedCall() throws Exception { BatchingSettings batchingSettings = BatchingSettings.newBuilder().setIsEnabled(false).build(); FlowControlSettings flowControlSettings = FlowControlSettings.newBuilder() .setLimitExceededBehavior(LimitExceededBehavior.Ignore) .build(); FlowController flowController = new FlowController(flowControlSettings); BatcherFactory<LabeledIntList, List<Integer>> batcherFactory = new BatcherFactory<>( new SquarerBatchingDescriptor(), batchingSettings, batchingExecutor, flowController); BatchingCallable<LabeledIntList, List<Integer>> batchingCallable = new BatchingCallable<>( FakeBatchableApi.callLabeledIntSquarer, FakeBatchableApi.SQUARER_BATCHING_DESC, batcherFactory); LabeledIntList request1 = new LabeledIntList("label", 2); ApiFuture<List<Integer>> future1 = batchingCallable.futureCall(request1, FakeCallContext.createDefault()); List<Integer> response1 = future1.get(); Truth.assertThat(response1.size()).isEqualTo(1); Truth.assertThat(response1.get(0)).isEqualTo(2 * 2); } }
public static GooglePubsubPublisher buildPublisher(GooglePubsubPublisherConfig config, ObjectMapper mapper) { GooglePubsubPublisher publisher = new GooglePubsubPublisher(); publisher.setName(config.getName()); ProjectTopicName fullName = ProjectTopicName.of(config.getProject(), config.getTopicName()); publisher.setTopicName(config.getTopicName()); publisher.setFullTopicName(fullName.toString()); publisher.setContent(config.getContent()); publisher.setMapper(mapper); BatchingSettings batchingSettings = BatchingSettings.newBuilder() .setElementCountThreshold(config.getBatchCountThreshold()) .setDelayThreshold(Duration.ofMillis(config.getDelayMillisecondsThreshold())) .build(); try { Publisher p = Publisher.newBuilder(fullName) .setCredentialsProvider(new GooglePubsubCredentialsProvider(config.getJsonPath())) .setBatchingSettings(batchingSettings) .build(); publisher.setPublisher(p); } catch (IOException ioe) { log.error("Could not create Google Pubsub Publishers: {}", ioe); } return publisher; }
@Test public void testEmptyBuilder() { @SuppressWarnings("unchecked") BatchingDescriptor<Integer, Integer> batchingDescriptor = Mockito.mock(BatchingDescriptor.class); BatchingCallSettings.Builder<Integer, Integer> builder = BatchingCallSettings.newBuilder(batchingDescriptor); Truth.assertThat(builder.getBatchingDescriptor()).isSameAs(batchingDescriptor); Truth.assertThat(builder.getBatchingSettings()).isNull(); Truth.assertThat(builder.getFlowController()).isNull(); Truth.assertThat(builder.getRetryableCodes().size()).isEqualTo(0); Truth.assertThat(builder.getRetrySettings()).isNotNull(); BatchingSettings batchingSettings = BatchingSettings.newBuilder().setElementCountThreshold(1L).build(); builder.setBatchingSettings(batchingSettings); BatchingCallSettings settings = builder.build(); Truth.assertThat(settings.getBatchingDescriptor()).isSameAs(batchingDescriptor); Truth.assertThat(settings.getBatchingSettings()).isSameAs(batchingSettings); Truth.assertThat(settings.getFlowController()).isNotNull(); Truth.assertThat(settings.getRetryableCodes().size()).isEqualTo(0); Truth.assertThat(settings.getRetrySettings()).isNotNull(); }
private CPSPublisherTask(StartRequest request) { super(request, "gcloud", MetricsHandler.MetricName.PUBLISH_ACK_LATENCY); try { this.publisher = Publisher.defaultBuilder(TopicName.create(request.getProject(), request.getTopic())) .setBatchingSettings( BatchingSettings.newBuilder() .setElementCountThreshold(950L) .setRequestByteThreshold(9500000L) .setDelayThreshold( Duration.ofMillis(Durations.toMillis(request.getPublishBatchDuration()))) .build()) .build(); } catch (Exception e) { throw new RuntimeException(e); } this.payload = ByteString.copyFromUtf8(LoadTestRunner.createMessage(request.getMessageSize())); this.batchSize = request.getPublishBatchSize(); this.messageSize = request.getMessageSize(); this.id = (new Random()).nextInt(); }
@Test public void batching() throws Exception { BatchingSettings batchingSettings = BatchingSettings.newBuilder() .setDelayThreshold(Duration.ofSeconds(1)) .setElementCountThreshold(2L) .build(); BatchingCallSettings<LabeledIntList, List<Integer>> batchingCallSettings = BatchingCallSettings.newBuilder(SQUARER_BATCHING_DESC) .setBatchingSettings(batchingSettings) .build(); UnaryCallable<LabeledIntList, List<Integer>> callable = FakeCallableFactory.createBatchingCallable( callLabeledIntSquarer, batchingCallSettings, clientContext); ApiFuture<List<Integer>> f1 = callable.futureCall(new LabeledIntList("one", 1, 2)); ApiFuture<List<Integer>> f2 = callable.futureCall(new LabeledIntList("one", 3, 4)); Truth.assertThat(f1.get()).isEqualTo(Arrays.asList(1, 4)); Truth.assertThat(f2.get()).isEqualTo(Arrays.asList(9, 16)); }
protected Builder(ClientContext clientContext) { super(clientContext); deleteLogSettings = UnaryCallSettings.newUnaryCallSettingsBuilder(); writeLogEntriesSettings = BatchingCallSettings.newBuilder(WRITE_LOG_ENTRIES_BATCHING_DESC) .setBatchingSettings(BatchingSettings.newBuilder().build()); listLogEntriesSettings = PagedCallSettings.newBuilder(LIST_LOG_ENTRIES_PAGE_STR_FACT); listMonitoredResourceDescriptorsSettings = PagedCallSettings.newBuilder(LIST_MONITORED_RESOURCE_DESCRIPTORS_PAGE_STR_FACT); listLogsSettings = PagedCallSettings.newBuilder(LIST_LOGS_PAGE_STR_FACT); unaryMethodSettingsBuilders = ImmutableList.<UnaryCallSettings.Builder<?, ?>>of( deleteLogSettings, writeLogEntriesSettings, listLogEntriesSettings, listMonitoredResourceDescriptorsSettings, listLogsSettings); initDefaults(this); }
public void batchingWithBlockingCallThreshold() throws Exception { BatchingSettings batchingSettings = BatchingSettings.newBuilder() .setDelayThreshold(Duration.ofSeconds(1)) .setElementCountThreshold(2L) .build(); BatchingCallSettings<LabeledIntList, List<Integer>> batchingCallSettings = BatchingCallSettings.newBuilder(SQUARER_BATCHING_DESC) .setBatchingSettings(batchingSettings) .build(); UnaryCallable<LabeledIntList, List<Integer>> callable = FakeCallableFactory.createBatchingCallable( callLabeledIntSquarer, batchingCallSettings, clientContext); ApiFuture<List<Integer>> f1 = callable.futureCall(new LabeledIntList("one", 1)); ApiFuture<List<Integer>> f2 = callable.futureCall(new LabeledIntList("one", 3)); Truth.assertThat(f1.get()).isEqualTo(Arrays.asList(1)); Truth.assertThat(f2.get()).isEqualTo(Arrays.asList(9)); }