@Override public int publish(TopicPath topic, List<OutgoingMessage> outgoingMessages) throws IOException { List<PubsubMessage> pubsubMessages = new ArrayList<>(outgoingMessages.size()); for (OutgoingMessage outgoingMessage : outgoingMessages) { PubsubMessage pubsubMessage = new PubsubMessage().encodeData(outgoingMessage.elementBytes); pubsubMessage.setAttributes(getMessageAttributes(outgoingMessage)); pubsubMessages.add(pubsubMessage); } PublishRequest request = new PublishRequest().setMessages(pubsubMessages); PublishResponse response = pubsub.projects().topics().publish(topic.getPath(), request).execute(); return response.getMessageIds().size(); }
@Override public List<SubscriptionPath> listSubscriptions(ProjectPath project, TopicPath topic) throws IOException { Subscriptions.List request = pubsub.projects().subscriptions().list(project.getPath()); ListSubscriptionsResponse response = request.execute(); if (response.getSubscriptions() == null || response.getSubscriptions().isEmpty()) { return ImmutableList.of(); } List<SubscriptionPath> subscriptions = new ArrayList<>(response.getSubscriptions().size()); while (true) { for (Subscription subscription : response.getSubscriptions()) { if (subscription.getTopic().equals(topic.getPath())) { subscriptions.add(subscriptionPathFromPath(subscription.getName())); } } if (Strings.isNullOrEmpty(response.getNextPageToken())) { break; } request.setPageToken(response.getNextPageToken()); response = request.execute(); } return subscriptions; }
@Override public int publish(TopicPath topic, List<OutgoingMessage> outgoingMessages) throws IOException { PublishRequest.Builder request = PublishRequest.newBuilder().setTopic(topic.getPath()); for (OutgoingMessage outgoingMessage : outgoingMessages) { PubsubMessage.Builder message = PubsubMessage.newBuilder().setData(ByteString.copyFrom(outgoingMessage.elementBytes)); if (outgoingMessage.attributes != null) { message.putAllAttributes(outgoingMessage.attributes); } if (timestampAttribute != null) { message .getMutableAttributes() .put(timestampAttribute, String.valueOf(outgoingMessage.timestampMsSinceEpoch)); } if (idAttribute != null && !Strings.isNullOrEmpty(outgoingMessage.recordId)) { message.getMutableAttributes().put(idAttribute, outgoingMessage.recordId); } request.addMessages(message); } PublishResponse response = publisherStub().publish(request.build()); return response.getMessageIdsCount(); }
PubsubIO.readMessagesWithAttributes().fromTopic(dlqTopic.topicPath().getPath()));
@Test public void publishOneMessage() throws IOException { String expectedTopic = TOPIC.getPath(); PubsubMessage expectedPubsubMessage = PubsubMessage.newBuilder()
@Override public List<SubscriptionPath> listSubscriptions(ProjectPath project, TopicPath topic) throws IOException { ListSubscriptionsRequest.Builder request = ListSubscriptionsRequest.newBuilder() .setProject(project.getPath()) .setPageSize(LIST_BATCH_SIZE); ListSubscriptionsResponse response = subscriberStub().listSubscriptions(request.build()); if (response.getSubscriptionsCount() == 0) { return ImmutableList.of(); } List<SubscriptionPath> subscriptions = new ArrayList<>(response.getSubscriptionsCount()); while (true) { for (Subscription subscription : response.getSubscriptionsList()) { if (subscription.getTopic().equals(topic.getPath())) { subscriptions.add(subscriptionPathFromPath(subscription.getName())); } } if (response.getNextPageToken().isEmpty()) { break; } request.setPageToken(response.getNextPageToken()); response = subscriberStub().listSubscriptions(request.build()); } return subscriptions; }
@Test public void publishOneMessage() throws IOException { String expectedTopic = TOPIC.getPath(); PubsubMessage expectedPubsubMessage = new PubsubMessage() .encodeData(DATA.getBytes(StandardCharsets.UTF_8)) .setAttributes( ImmutableMap.<String, String>builder() .put(TIMESTAMP_ATTRIBUTE, String.valueOf(MESSAGE_TIME)) .put(ID_ATTRIBUTE, RECORD_ID) .put("k", "v") .build()); PublishRequest expectedRequest = new PublishRequest().setMessages(ImmutableList.of(expectedPubsubMessage)); PublishResponse expectedResponse = new PublishResponse().setMessageIds(ImmutableList.of(MESSAGE_ID)); when((Object) (mockPubsub.projects().topics().publish(expectedTopic, expectedRequest).execute())) .thenReturn(expectedResponse); Map<String, String> attrs = new HashMap<>(); attrs.put("k", "v"); OutgoingMessage actualMessage = new OutgoingMessage(DATA.getBytes(StandardCharsets.UTF_8), attrs, MESSAGE_TIME, RECORD_ID); int n = client.publish(TOPIC, ImmutableList.of(actualMessage)); assertEquals(1, n); }
@Test public void publishOneMessageWithNoTimestampOrIdAttribute() throws IOException { // For this test, create a new PubsubJsonClient without the timestamp attribute // or id attribute set. client = new PubsubJsonClient(null, null, mockPubsub); String expectedTopic = TOPIC.getPath(); PubsubMessage expectedPubsubMessage = new PubsubMessage() .encodeData(DATA.getBytes(StandardCharsets.UTF_8)) .setAttributes(ImmutableMap.<String, String>builder().put("k", "v").build()); PublishRequest expectedRequest = new PublishRequest().setMessages(ImmutableList.of(expectedPubsubMessage)); PublishResponse expectedResponse = new PublishResponse().setMessageIds(ImmutableList.of(MESSAGE_ID)); when((Object) (mockPubsub.projects().topics().publish(expectedTopic, expectedRequest).execute())) .thenReturn(expectedResponse); Map<String, String> attrs = new HashMap<>(); attrs.put("k", "v"); OutgoingMessage actualMessage = new OutgoingMessage(DATA.getBytes(StandardCharsets.UTF_8), attrs, MESSAGE_TIME, RECORD_ID); int n = client.publish(TOPIC, ImmutableList.of(actualMessage)); assertEquals(1, n); }
@Test public void publishOneMessageWithOnlyTimestampAndIdAttributes() throws IOException { String expectedTopic = TOPIC.getPath(); PubsubMessage expectedPubsubMessage = new PubsubMessage() .encodeData(DATA.getBytes(StandardCharsets.UTF_8)) .setAttributes( ImmutableMap.<String, String>builder() .put(TIMESTAMP_ATTRIBUTE, String.valueOf(MESSAGE_TIME)) .put(ID_ATTRIBUTE, RECORD_ID) .build()); PublishRequest expectedRequest = new PublishRequest().setMessages(ImmutableList.of(expectedPubsubMessage)); PublishResponse expectedResponse = new PublishResponse().setMessageIds(ImmutableList.of(MESSAGE_ID)); when((Object) (mockPubsub.projects().topics().publish(expectedTopic, expectedRequest).execute())) .thenReturn(expectedResponse); OutgoingMessage actualMessage = new OutgoingMessage( DATA.getBytes(StandardCharsets.UTF_8), ImmutableMap.of(), MESSAGE_TIME, RECORD_ID); int n = client.publish(TOPIC, ImmutableList.of(actualMessage)); assertEquals(1, n); }
@Override public POutput expand(PCollection<? extends T> input) { return input // assign a dummy key and global window, // this is needed to accumulate all observed events in the same state cell .apply(Window.into(new GlobalWindows())) .apply(WithKeys.of("dummyKey")) .apply( "checkAllEventsForSuccess", ParDo.of(new StatefulPredicateCheck<>(coder, formatter, successPredicate))) // signal the success/failure to the result topic .apply("publishSuccess", PubsubIO.writeStrings().to(resultTopicPath.getPath())); } }
@Override public PDone expand(PBegin input) { return input .apply("Start signal", Create.of(START_SIGNAL_MESSAGE)) .apply(PubsubIO.writeStrings().to(startTopicPath.getPath())); } }
@Override public void createTopic(TopicPath topic) throws IOException { pubsub .projects() .topics() .create(topic.getPath(), new Topic()) .execute(); // ignore Topic result. }
@Override public void deleteTopic(TopicPath topic) throws IOException { pubsub.projects().topics().delete(topic.getPath()).execute(); // ignore Empty result. }
private static Topic buildTopic(int i) { Topic topic = new Topic(); topic.setName(PubsubClient.topicPathFromName(PROJECT.getId(), "Topic" + i).getPath()); return topic; }
@Override public void createSubscription( TopicPath topic, SubscriptionPath subscription, int ackDeadlineSeconds) throws IOException { Subscription request = Subscription.newBuilder() .setTopic(topic.getPath()) .setName(subscription.getPath()) .setAckDeadlineSeconds(ackDeadlineSeconds) .build(); subscriberStub().createSubscription(request); // ignore Subscription result. }
@Override public void createTopic(TopicPath topic) throws IOException { Topic request = Topic.newBuilder().setName(topic.getPath()).build(); publisherStub().createTopic(request); // ignore Topic result. }
@Test public void topicPathFromNameWellFormed() { TopicPath path = PubsubClient.topicPathFromName("test", "something"); assertEquals("projects/test/topics/something", path.getPath()); assertEquals("/topics/test/something", path.getV1Beta1Path()); } }
@Override public void createSubscription( TopicPath topic, SubscriptionPath subscription, int ackDeadlineSeconds) throws IOException { Subscription request = new Subscription().setTopic(topic.getPath()).setAckDeadlineSeconds(ackDeadlineSeconds); pubsub .projects() .subscriptions() .create(subscription.getPath(), request) .execute(); // ignore Subscription result. }
@Override public void deleteTopic(TopicPath topic) throws IOException { DeleteTopicRequest request = DeleteTopicRequest.newBuilder().setTopic(topic.getPath()).build(); publisherStub().deleteTopic(request); // ignore Empty result. }
private static Subscription buildSubscription(int i) { Subscription subscription = new Subscription(); subscription.setName( PubsubClient.subscriptionPathFromName(PROJECT.getId(), "Subscription" + i).getPath()); subscription.setTopic(PubsubClient.topicPathFromName(PROJECT.getId(), "Topic" + i).getPath()); return subscription; } }