Refine search
publisher = Publisher.newBuilder(topicName).build(); ApiFuture<String> messageIdFuture = publisher.publish(pubsubMessage); messageIdFutures.add(messageIdFuture); publisher.shutdown(); publisher.awaitTermination(1, TimeUnit.MINUTES);
&& hasBatchingBytes() && batchedBytes + messageSize >= getMaxBatchBytes()) { batchToSend = new OutstandingBatch(messagesBatch, batchedBytes); messagesBatch = new LinkedList<>(); if (!hasBatchingBytes() || messageSize < getMaxBatchBytes()) { batchedBytes += messageSize; messagesBatch.add(outstandingPublish); if (messagesBatch.size() == getBatchingSettings().getElementCountThreshold()) { batchToSend = new OutstandingBatch(messagesBatch, batchedBytes); messagesBatch = new LinkedList<>(); setupDurationBasedPublishAlarm(); } else if (currentAlarmFuture != null) { logger.log(Level.FINER, "Cancelling alarm, no more messages"); if (hasBatchingBytes() && messageSize >= getMaxBatchBytes()) { logger.log( Level.FINER, "Message exceeds the max batch bytes, scheduling it for immediate send.");
/** Example of creating a {@code Publisher}. */ // [TARGET newBuilder(ProjectTopicName)] // [VARIABLE "my_project"] // [VARIABLE "my_topic"] public static void newBuilder(String projectId, String topicId) throws Exception { ProjectTopicName topic = ProjectTopicName.of(projectId, topicId); Publisher publisher = Publisher.newBuilder(topic).build(); try { // ... } finally { // When finished with the publisher, make sure to shutdown to free up resources. publisher.shutdown(); publisher.awaitTermination(1, TimeUnit.MINUTES); } }
/** * Constructs a new {@link Builder} using the given topic. * * <p>Example of creating a {@code Publisher}. * * <pre>{@code * String projectName = "my_project"; * String topicName = "my_topic"; * ProjectTopicName topic = ProjectTopicName.create(projectName, topicName); * Publisher publisher = Publisher.newBuilder(topic).build(); * try { * // ... * } finally { * // When finished with the publisher, make sure to shutdown to free up resources. * publisher.shutdown(); * publisher.awaitTermination(1, TimeUnit.MINUTES); * } * }</pre> */ public static Builder newBuilder(TopicName topicName) { return newBuilder(topicName.toString()); }
/** Example of publishing a message. */ // [TARGET publish(PubsubMessage)] // [VARIABLE "my_message"] public ApiFuture<String> publish(String message) { ByteString data = ByteString.copyFromUtf8(message); PubsubMessage pubsubMessage = PubsubMessage.newBuilder().setData(data).build(); ApiFuture<String> messageIdFuture = publisher.publish(pubsubMessage); ApiFutures.addCallback( messageIdFuture, new ApiFutureCallback<String>() { public void onSuccess(String messageId) { System.out.println("published with message id: " + messageId); } public void onFailure(Throwable t) { System.out.println("failed to publish: " + t); } }, Executors.newSingleThreadExecutor()); return messageIdFuture; }
publisher = Publisher.newBuilder(topicName).build(); ApiFuture<String> future = publisher.publish(pubsubMessage); futures.add(future); publisher.shutdown();
@Test(expected = ExecutionException.class) public void testPublishFailureRetries_retriesDisabled() throws Exception { Publisher publisher = getTestPublisherBuilder() .setExecutorProvider(SINGLE_THREAD_EXECUTOR) .setRetrySettings( Publisher.Builder.DEFAULT_RETRY_SETTINGS .toBuilder() .setTotalTimeout(Duration.ofSeconds(10)) .setMaxAttempts(1) .build()) .build(); testPublisherServiceImpl.addPublishError(new Throwable("Transiently failing")); ApiFuture<String> publishFuture1 = sendTestMessage(publisher, "A"); try { publishFuture1.get(); } finally { assertSame(testPublisherServiceImpl.getCapturedRequests().size(), 1); publisher.shutdown(); publisher.awaitTermination(1, TimeUnit.MINUTES); } }
@Test public void testPublisherGetters() throws Exception { Publisher.Builder builder = Publisher.newBuilder(TEST_TOPIC); builder.setChannelProvider(TEST_CHANNEL_PROVIDER); builder.setExecutorProvider(SINGLE_THREAD_EXECUTOR); builder.setBatchingSettings( BatchingSettings.newBuilder() .setRequestByteThreshold(10L) .setDelayThreshold(Duration.ofMillis(11)) .setElementCountThreshold(12L) .build()); builder.setCredentialsProvider(NoCredentialsProvider.create()); Publisher publisher = builder.build(); assertEquals(TEST_TOPIC, publisher.getTopicName()); assertEquals(10, (long) publisher.getBatchingSettings().getRequestByteThreshold()); assertEquals(Duration.ofMillis(11), publisher.getBatchingSettings().getDelayThreshold()); assertEquals(12, (long) publisher.getBatchingSettings().getElementCountThreshold()); publisher.shutdown(); publisher.awaitTermination(1, TimeUnit.MINUTES); }
@Test public void testPublishByNumBatchedMessages() throws Exception { Publisher publisher = getTestPublisherBuilder() .setBatchingSettings( Publisher.Builder.DEFAULT_BATCHING_SETTINGS .toBuilder() .setElementCountThreshold(2L) .setDelayThreshold(Duration.ofSeconds(100)) .build()) .build(); testPublisherServiceImpl .addPublishResponse(PublishResponse.newBuilder().addMessageIds("1").addMessageIds("2")) .addPublishResponse(PublishResponse.newBuilder().addMessageIds("3").addMessageIds("4")); ApiFuture<String> publishFuture1 = sendTestMessage(publisher, "A"); ApiFuture<String> publishFuture2 = sendTestMessage(publisher, "B"); ApiFuture<String> publishFuture3 = sendTestMessage(publisher, "C"); // Note we are not advancing time but message should still get published assertEquals("1", publishFuture1.get()); assertEquals("2", publishFuture2.get()); assertFalse(publishFuture3.isDone()); ApiFuture<String> publishFuture4 = publisher.publish(PubsubMessage.newBuilder().setData(ByteString.copyFromUtf8("D")).build()); assertEquals("3", publishFuture3.get()); assertEquals("4", publishFuture4.get()); assertEquals(2, testPublisherServiceImpl.getCapturedRequests().get(0).getMessagesCount()); assertEquals(2, testPublisherServiceImpl.getCapturedRequests().get(1).getMessagesCount()); publisher.shutdown(); publisher.awaitTermination(1, TimeUnit.MINUTES); }
private long getMaxBatchBytes() { return getBatchingSettings().getRequestByteThreshold(); }
private Publisher createPublisherWithCustomCredentials(ProjectTopicName topicName) throws Exception { // [START pubsub_publisher_custom_credentials] // read service account credentials from file CredentialsProvider credentialsProvider = FixedCredentialsProvider.create( ServiceAccountCredentials.fromStream(new FileInputStream("credentials.json"))); Publisher publisher = Publisher.newBuilder(topicName).setCredentialsProvider(credentialsProvider).build(); // [END pubsub_publisher_custom_credentials] return publisher; } }
@Override public void sendAsync(RowMap r, AbstractAsyncProducer.CallbackCompleter cc) throws Exception { String message = r.toJSON(outputConfig); ByteString data = ByteString.copyFromUtf8(message); PubsubMessage pubsubMessage = PubsubMessage.newBuilder().setData(data).build(); if ( r instanceof DDLMap ) { ApiFuture<String> apiFuture = ddlPubsub.publish(pubsubMessage); PubsubCallback callback = new PubsubCallback(cc, r.getNextPosition(), message, this.succeededMessageCount, this.failedMessageCount, this.succeededMessageMeter, this.failedMessageMeter, this.context); ApiFutures.addCallback(apiFuture, callback); } else { ApiFuture<String> apiFuture = pubsub.publish(pubsubMessage); PubsubCallback callback = new PubsubCallback(cc, r.getNextPosition(), message, this.succeededMessageCount, this.failedMessageCount, this.succeededMessageMeter, this.failedMessageMeter, this.context); ApiFutures.addCallback(apiFuture, callback); } }
@Test public void testPublishFailureRetries_maxRetriesSetUnlimited() throws Exception { Publisher publisher = getTestPublisherBuilder() .setExecutorProvider(SINGLE_THREAD_EXECUTOR) .setRetrySettings( Publisher.Builder.DEFAULT_RETRY_SETTINGS .toBuilder() .setTotalTimeout(Duration.ofSeconds(10)) .setMaxAttempts(0) .build()) .build(); testPublisherServiceImpl.addPublishError(new Throwable("Transiently failing")); testPublisherServiceImpl.addPublishError(new Throwable("Transiently failing")); testPublisherServiceImpl.addPublishResponse(PublishResponse.newBuilder().addMessageIds("1")); ApiFuture<String> publishFuture1 = sendTestMessage(publisher, "A"); assertEquals("1", publishFuture1.get()); assertEquals(3, testPublisherServiceImpl.getCapturedRequests().size()); publisher.shutdown(); publisher.awaitTermination(1, TimeUnit.MINUTES); }
private void shutdownPublisher() { try { if (publisher != null) { publisher.shutdown(); } } catch (Exception e) { getLogger().warn("Failed to gracefully shutdown the Google Cloud PubSub Publisher due to {}", new Object[]{e}); } }
private void setupDurationBasedPublishAlarm() { if (!activeAlarm.getAndSet(true)) { long delayThresholdMs = getBatchingSettings().getDelayThreshold().toMillis(); logger.log(Level.FINER, "Setting up alarm for the next {0} ms.", delayThresholdMs); currentAlarmFuture = executor.schedule( new Runnable() { @Override public void run() { logger.log(Level.FINER, "Sending messages based on schedule."); activeAlarm.getAndSet(false); publishAllOutstanding(); } }, delayThresholdMs, TimeUnit.MILLISECONDS); } }
publisher = Publisher.newBuilder(topicName).build(); ApiFuture<String> future = publisher.publish(pubsubMessage); if (publisher != null) { publisher.shutdown(); publisher.awaitTermination(1, TimeUnit.MINUTES);
public MaxwellPubsubProducerWorker(MaxwellContext context, String pubsubProjectId, String pubsubTopic, String ddlPubsubTopic, ArrayBlockingQueue<RowMap> queue) throws IOException { super(context); this.projectId = pubsubProjectId; this.topic = TopicName.of(pubsubProjectId, pubsubTopic); this.pubsub = Publisher.newBuilder(this.topic).build(); if ( context.getConfig().outputConfig.outputDDL == true && ddlPubsubTopic != pubsubTopic ) { this.ddlTopic = TopicName.of(pubsubProjectId, ddlPubsubTopic); this.ddlPubsub = Publisher.newBuilder(this.ddlTopic).build(); } else { this.ddlTopic = this.topic; this.ddlPubsub = this.pubsub; } Metrics metrics = context.getMetrics(); this.queue = queue; this.taskState = new StoppableTaskState("MaxwellPubsubProducerWorker"); }
private ApiFuture<String> sendTestMessage(Publisher publisher, String data) { return publisher.publish( PubsubMessage.newBuilder().setData(ByteString.copyFromUtf8(data)).build()); }
@Test public void testPublishFailureRetries_maxRetriesSetup() throws Exception { Publisher publisher = getTestPublisherBuilder() .setExecutorProvider(SINGLE_THREAD_EXECUTOR) .setRetrySettings( Publisher.Builder.DEFAULT_RETRY_SETTINGS .toBuilder() .setTotalTimeout(Duration.ofSeconds(10)) .setMaxAttempts(3) .build()) .build(); testPublisherServiceImpl.addPublishError(new Throwable("Transiently failing")); testPublisherServiceImpl.addPublishError(new Throwable("Transiently failing")); testPublisherServiceImpl.addPublishResponse(PublishResponse.newBuilder().addMessageIds("1")); ApiFuture<String> publishFuture1 = sendTestMessage(publisher, "A"); assertEquals("1", publishFuture1.get()); assertEquals(3, testPublisherServiceImpl.getCapturedRequests().size()); publisher.shutdown(); publisher.awaitTermination(1, TimeUnit.MINUTES); }