publisher = Publisher.newBuilder(topicName).build();
@OnScheduled public void onScheduled(ProcessContext context) { try { publisher = getPublisherBuilder(context).build(); } catch (IOException e) { getLogger().error("Failed to create Google Cloud PubSub Publisher due to {}", new Object[]{e}); storedException.set(e); } }
private Publisher createPublisherWithCustomCredentials(ProjectTopicName topicName) throws Exception { // [START pubsub_publisher_custom_credentials] // read service account credentials from file CredentialsProvider credentialsProvider = FixedCredentialsProvider.create( ServiceAccountCredentials.fromStream(new FileInputStream("credentials.json"))); Publisher publisher = Publisher.newBuilder(topicName).setCredentialsProvider(credentialsProvider).build(); // [END pubsub_publisher_custom_credentials] return publisher; } }
.setDelayThreshold(Duration.ofSeconds(5)) .build()) .build();
.setChannelProvider(channelProvider) .setCredentialsProvider(credentialsProvider) .build(); } finally { channel.shutdown();
@Test public void testSinglePublishByNumBytes() throws Exception { Publisher publisher = getTestPublisherBuilder() .setBatchingSettings( Publisher.Builder.DEFAULT_BATCHING_SETTINGS .toBuilder() .setElementCountThreshold(2L) .setDelayThreshold(Duration.ofSeconds(100)) .build()) .build(); testPublisherServiceImpl .addPublishResponse(PublishResponse.newBuilder().addMessageIds("1").addMessageIds("2")) .addPublishResponse(PublishResponse.newBuilder().addMessageIds("3").addMessageIds("4")); ApiFuture<String> publishFuture1 = sendTestMessage(publisher, "A"); ApiFuture<String> publishFuture2 = sendTestMessage(publisher, "B"); ApiFuture<String> publishFuture3 = sendTestMessage(publisher, "C"); // Note we are not advancing time but message should still get published assertEquals("1", publishFuture1.get()); assertEquals("2", publishFuture2.get()); assertFalse(publishFuture3.isDone()); ApiFuture<String> publishFuture4 = sendTestMessage(publisher, "D"); assertEquals("3", publishFuture3.get()); assertEquals("4", publishFuture4.get()); assertEquals(2, testPublisherServiceImpl.getCapturedRequests().size()); publisher.shutdown(); publisher.awaitTermination(1, TimeUnit.MINUTES); }
@Test(expected = ExecutionException.class) public void testPublishFailureRetries_nonRetryableFailsImmediately() throws Exception { Publisher publisher = getTestPublisherBuilder() .setExecutorProvider(SINGLE_THREAD_EXECUTOR) .setRetrySettings( Publisher.Builder.DEFAULT_RETRY_SETTINGS .toBuilder() .setTotalTimeout(Duration.ofSeconds(10)) .build()) .setBatchingSettings( Publisher.Builder.DEFAULT_BATCHING_SETTINGS .toBuilder() .setElementCountThreshold(1L) .setDelayThreshold(Duration.ofSeconds(5)) .build()) .build(); // To demonstrate that reaching duration will trigger publish testPublisherServiceImpl.addPublishError(new StatusException(Status.INVALID_ARGUMENT)); ApiFuture<String> publishFuture1 = sendTestMessage(publisher, "A"); try { publishFuture1.get(); } finally { assertTrue(testPublisherServiceImpl.getCapturedRequests().size() >= 1); publisher.shutdown(); publisher.awaitTermination(1, TimeUnit.MINUTES); } }
@Test public void testPublishByDuration() throws Exception { Publisher publisher = getTestPublisherBuilder() // To demonstrate that reaching duration will trigger publish .setBatchingSettings( Publisher.Builder.DEFAULT_BATCHING_SETTINGS .toBuilder() .setDelayThreshold(Duration.ofSeconds(5)) .setElementCountThreshold(10L) .build()) .build(); testPublisherServiceImpl.addPublishResponse( PublishResponse.newBuilder().addMessageIds("1").addMessageIds("2")); ApiFuture<String> publishFuture1 = sendTestMessage(publisher, "A"); ApiFuture<String> publishFuture2 = sendTestMessage(publisher, "B"); assertFalse(publishFuture1.isDone()); assertFalse(publishFuture2.isDone()); fakeExecutor.advanceTime(Duration.ofSeconds(10)); assertEquals("1", publishFuture1.get()); assertEquals("2", publishFuture2.get()); assertEquals(2, testPublisherServiceImpl.getCapturedRequests().get(0).getMessagesCount()); publisher.shutdown(); publisher.awaitTermination(1, TimeUnit.MINUTES); }
@Test public void testPublishByNumBatchedMessages() throws Exception { Publisher publisher = getTestPublisherBuilder() .setBatchingSettings( Publisher.Builder.DEFAULT_BATCHING_SETTINGS .toBuilder() .setElementCountThreshold(2L) .setDelayThreshold(Duration.ofSeconds(100)) .build()) .build(); testPublisherServiceImpl .addPublishResponse(PublishResponse.newBuilder().addMessageIds("1").addMessageIds("2")) .addPublishResponse(PublishResponse.newBuilder().addMessageIds("3").addMessageIds("4")); ApiFuture<String> publishFuture1 = sendTestMessage(publisher, "A"); ApiFuture<String> publishFuture2 = sendTestMessage(publisher, "B"); ApiFuture<String> publishFuture3 = sendTestMessage(publisher, "C"); // Note we are not advancing time but message should still get published assertEquals("1", publishFuture1.get()); assertEquals("2", publishFuture2.get()); assertFalse(publishFuture3.isDone()); ApiFuture<String> publishFuture4 = publisher.publish(PubsubMessage.newBuilder().setData(ByteString.copyFromUtf8("D")).build()); assertEquals("3", publishFuture3.get()); assertEquals("4", publishFuture4.get()); assertEquals(2, testPublisherServiceImpl.getCapturedRequests().get(0).getMessagesCount()); assertEquals(2, testPublisherServiceImpl.getCapturedRequests().get(1).getMessagesCount()); publisher.shutdown(); publisher.awaitTermination(1, TimeUnit.MINUTES); }
@Test(expected = ExecutionException.class) public void testPublishFailureRetries_retriesDisabled() throws Exception { Publisher publisher = getTestPublisherBuilder() .setExecutorProvider(SINGLE_THREAD_EXECUTOR) .setRetrySettings( Publisher.Builder.DEFAULT_RETRY_SETTINGS .toBuilder() .setTotalTimeout(Duration.ofSeconds(10)) .setMaxAttempts(1) .build()) .build(); testPublisherServiceImpl.addPublishError(new Throwable("Transiently failing")); ApiFuture<String> publishFuture1 = sendTestMessage(publisher, "A"); try { publishFuture1.get(); } finally { assertSame(testPublisherServiceImpl.getCapturedRequests().size(), 1); publisher.shutdown(); publisher.awaitTermination(1, TimeUnit.MINUTES); } }
public Publisher getPublisherWithCustomRetrySettings(ProjectTopicName topicName) throws Exception { // [START pubsub_publisher_retry_settings] // Retry settings control how the publisher handles retryable failures Duration retryDelay = Duration.ofMillis(100); // default : 1 ms double retryDelayMultiplier = 2.0; // back off for repeated failures Duration maxRetryDelay = Duration.ofSeconds(5); // default : 10 seconds Duration totalTimeout = Duration.ofSeconds(1); // default: 0 Duration initialRpcTimeout = Duration.ofSeconds(1); // default: 0 Duration maxRpcTimeout = Duration.ofSeconds(10); // default: 0 RetrySettings retrySettings = RetrySettings.newBuilder() .setInitialRetryDelay(retryDelay) .setRetryDelayMultiplier(retryDelayMultiplier) .setMaxRetryDelay(maxRetryDelay) .setTotalTimeout(totalTimeout) .setInitialRpcTimeout(initialRpcTimeout) .setMaxRpcTimeout(maxRpcTimeout) .build(); Publisher publisher = Publisher.newBuilder(topicName).setRetrySettings(retrySettings).build(); // [END pubsub_publisher_retry_settings] return publisher; }
@Test public void testPublishFailureRetries_maxRetriesSetup() throws Exception { Publisher publisher = getTestPublisherBuilder() .setExecutorProvider(SINGLE_THREAD_EXECUTOR) .setRetrySettings( Publisher.Builder.DEFAULT_RETRY_SETTINGS .toBuilder() .setTotalTimeout(Duration.ofSeconds(10)) .setMaxAttempts(3) .build()) .build(); testPublisherServiceImpl.addPublishError(new Throwable("Transiently failing")); testPublisherServiceImpl.addPublishError(new Throwable("Transiently failing")); testPublisherServiceImpl.addPublishResponse(PublishResponse.newBuilder().addMessageIds("1")); ApiFuture<String> publishFuture1 = sendTestMessage(publisher, "A"); assertEquals("1", publishFuture1.get()); assertEquals(3, testPublisherServiceImpl.getCapturedRequests().size()); publisher.shutdown(); publisher.awaitTermination(1, TimeUnit.MINUTES); }
public MaxwellPubsubProducerWorker(MaxwellContext context, String pubsubProjectId, String pubsubTopic, String ddlPubsubTopic, ArrayBlockingQueue<RowMap> queue) throws IOException { super(context); this.projectId = pubsubProjectId; this.topic = TopicName.of(pubsubProjectId, pubsubTopic); this.pubsub = Publisher.newBuilder(this.topic).build(); if ( context.getConfig().outputConfig.outputDDL == true && ddlPubsubTopic != pubsubTopic ) { this.ddlTopic = TopicName.of(pubsubProjectId, ddlPubsubTopic); this.ddlPubsub = Publisher.newBuilder(this.ddlTopic).build(); } else { this.ddlTopic = this.topic; this.ddlPubsub = this.pubsub; } Metrics metrics = context.getMetrics(); this.queue = queue; this.taskState = new StoppableTaskState("MaxwellPubsubProducerWorker"); }
@Test public void testPublishFailureRetries_maxRetriesSetUnlimited() throws Exception { Publisher publisher = getTestPublisherBuilder() .setExecutorProvider(SINGLE_THREAD_EXECUTOR) .setRetrySettings( Publisher.Builder.DEFAULT_RETRY_SETTINGS .toBuilder() .setTotalTimeout(Duration.ofSeconds(10)) .setMaxAttempts(0) .build()) .build(); testPublisherServiceImpl.addPublishError(new Throwable("Transiently failing")); testPublisherServiceImpl.addPublishError(new Throwable("Transiently failing")); testPublisherServiceImpl.addPublishResponse(PublishResponse.newBuilder().addMessageIds("1")); ApiFuture<String> publishFuture1 = sendTestMessage(publisher, "A"); assertEquals("1", publishFuture1.get()); assertEquals(3, testPublisherServiceImpl.getCapturedRequests().size()); publisher.shutdown(); publisher.awaitTermination(1, TimeUnit.MINUTES); }
public Publisher getPublisherWithCustomBatchSettings(ProjectTopicName topicName) throws Exception { // [START pubsub_publisher_batch_settings] // Batch settings control how the publisher batches messages long requestBytesThreshold = 5000L; // default : 1 byte long messageCountBatchSize = 10L; // default : 1 message Duration publishDelayThreshold = Duration.ofMillis(100); // default : 1 ms // Publish request get triggered based on request size, messages count & time since last publish BatchingSettings batchingSettings = BatchingSettings.newBuilder() .setElementCountThreshold(messageCountBatchSize) .setRequestByteThreshold(requestBytesThreshold) .setDelayThreshold(publishDelayThreshold) .build(); Publisher publisher = Publisher.newBuilder(topicName).setBatchingSettings(batchingSettings).build(); // [END pubsub_publisher_batch_settings] return publisher; }
@Test public void testPublishFailureRetries() throws Exception { Publisher publisher = getTestPublisherBuilder() .setExecutorProvider(SINGLE_THREAD_EXECUTOR) .setBatchingSettings( Publisher.Builder.DEFAULT_BATCHING_SETTINGS .toBuilder() .setElementCountThreshold(1L) .setDelayThreshold(Duration.ofSeconds(5)) .build()) .build(); // To demonstrate that reaching duration will trigger publish testPublisherServiceImpl.addPublishError(new Throwable("Transiently failing")); testPublisherServiceImpl.addPublishResponse(PublishResponse.newBuilder().addMessageIds("1")); ApiFuture<String> publishFuture1 = sendTestMessage(publisher, "A"); assertEquals("1", publishFuture1.get()); assertEquals(2, testPublisherServiceImpl.getCapturedRequests().size()); publisher.shutdown(); publisher.awaitTermination(1, TimeUnit.MINUTES); }
@Test public void testErrorPropagation() throws Exception { Publisher publisher = getTestPublisherBuilder() .setExecutorProvider(SINGLE_THREAD_EXECUTOR) .setBatchingSettings( Publisher.Builder.DEFAULT_BATCHING_SETTINGS .toBuilder() .setElementCountThreshold(1L) .setDelayThreshold(Duration.ofSeconds(5)) .build()) .build(); testPublisherServiceImpl.addPublishError(Status.DATA_LOSS.asException()); try { sendTestMessage(publisher, "A").get(); fail("should throw exception"); } catch (ExecutionException e) { assertThat(e.getCause()).isInstanceOf(DataLossException.class); } }
@Test public void testPublisherGetters() throws Exception { Publisher.Builder builder = Publisher.newBuilder(TEST_TOPIC); builder.setChannelProvider(TEST_CHANNEL_PROVIDER); builder.setExecutorProvider(SINGLE_THREAD_EXECUTOR); builder.setBatchingSettings( BatchingSettings.newBuilder() .setRequestByteThreshold(10L) .setDelayThreshold(Duration.ofMillis(11)) .setElementCountThreshold(12L) .build()); builder.setCredentialsProvider(NoCredentialsProvider.create()); Publisher publisher = builder.build(); assertEquals(TEST_TOPIC, publisher.getTopicName()); assertEquals(10, (long) publisher.getBatchingSettings().getRequestByteThreshold()); assertEquals(Duration.ofMillis(11), publisher.getBatchingSettings().getDelayThreshold()); assertEquals(12, (long) publisher.getBatchingSettings().getElementCountThreshold()); publisher.shutdown(); publisher.awaitTermination(1, TimeUnit.MINUTES); }
/** Example of creating a {@code Publisher}. */ // [TARGET newBuilder(ProjectTopicName)] // [VARIABLE "my_project"] // [VARIABLE "my_topic"] public static void newBuilder(String projectId, String topicId) throws Exception { ProjectTopicName topic = ProjectTopicName.of(projectId, topicId); Publisher publisher = Publisher.newBuilder(topic).build(); try { // ... } finally { // When finished with the publisher, make sure to shutdown to free up resources. publisher.shutdown(); publisher.awaitTermination(1, TimeUnit.MINUTES); } }
public Publisher getSingleThreadedPublisher(ProjectTopicName topicName) throws Exception { // [START pubsub_publisher_concurrency_control] // create a publisher with a single threaded executor ExecutorProvider executorProvider = InstantiatingExecutorProvider.newBuilder().setExecutorThreadCount(1).build(); Publisher publisher = Publisher.newBuilder(topicName).setExecutorProvider(executorProvider).build(); // [END pubsub_publisher_concurrency_control] return publisher; }