@Override public List<SubscriptionPath> listSubscriptions(ProjectPath project, TopicPath topic) throws IOException { ListSubscriptionsRequest.Builder request = ListSubscriptionsRequest.newBuilder() .setProject(project.getPath()) .setPageSize(LIST_BATCH_SIZE); ListSubscriptionsResponse response = subscriberStub().listSubscriptions(request.build()); if (response.getSubscriptionsCount() == 0) { return ImmutableList.of(); } List<SubscriptionPath> subscriptions = new ArrayList<>(response.getSubscriptionsCount()); while (true) { for (Subscription subscription : response.getSubscriptionsList()) { if (subscription.getTopic().equals(topic.getPath())) { subscriptions.add(subscriptionPathFromPath(subscription.getName())); } } if (response.getNextPageToken().isEmpty()) { break; } request.setPageToken(response.getNextPageToken()); response = subscriberStub().listSubscriptions(request.build()); } return subscriptions; }
public static TopicPath topicPathFromName(String projectId, String topicName) { return new TopicPath(String.format("projects/%s/topics/%s", projectId, topicName)); }
@Override public int publish(TopicPath topic, List<OutgoingMessage> outgoingMessages) throws IOException { synchronized (STATE) { checkState(inPublishMode(), "Can only publish in publish mode"); checkState( topic.equals(STATE.expectedTopic), "Topic %s does not match expected %s", topic, STATE.expectedTopic); for (OutgoingMessage outgoingMessage : outgoingMessages) { if (STATE.remainingFailingOutgoingMessages.remove(outgoingMessage)) { throw new RuntimeException("Simulating failure for " + outgoingMessage); } checkState( STATE.remainingExpectedOutgoingMessages.remove(outgoingMessage), "Unexpected outgoing message %s", outgoingMessage); } return outgoingMessages.size(); } }
PubsubIO.readMessagesWithAttributes().fromTopic(dlqTopic.topicPath().getPath()));
if (overriddenTransform.getTopicProvider().isAccessible()) { stepContext.addInput( PropertyNames.PUBSUB_TOPIC, overriddenTransform.getTopic().getV1Beta1Path()); } else { stepContext.addInput(
@Test public void publishOneMessage() throws IOException { String expectedTopic = TOPIC.getPath(); PubsubMessage expectedPubsubMessage = PubsubMessage.newBuilder()
if (overriddenTransform.getTopicProvider().isAccessible()) { stepContext.addInput( PropertyNames.PUBSUB_TOPIC, overriddenTransform.getTopic().getV1Beta1Path()); } else { stepContext.addInput(
@Test public void publishOneMessage() throws IOException { String expectedTopic = TOPIC.getPath(); PubsubMessage expectedPubsubMessage = new PubsubMessage() .encodeData(DATA.getBytes(StandardCharsets.UTF_8)) .setAttributes( ImmutableMap.<String, String>builder() .put(TIMESTAMP_ATTRIBUTE, String.valueOf(MESSAGE_TIME)) .put(ID_ATTRIBUTE, RECORD_ID) .put("k", "v") .build()); PublishRequest expectedRequest = new PublishRequest().setMessages(ImmutableList.of(expectedPubsubMessage)); PublishResponse expectedResponse = new PublishResponse().setMessageIds(ImmutableList.of(MESSAGE_ID)); when((Object) (mockPubsub.projects().topics().publish(expectedTopic, expectedRequest).execute())) .thenReturn(expectedResponse); Map<String, String> attrs = new HashMap<>(); attrs.put("k", "v"); OutgoingMessage actualMessage = new OutgoingMessage(DATA.getBytes(StandardCharsets.UTF_8), attrs, MESSAGE_TIME, RECORD_ID); int n = client.publish(TOPIC, ImmutableList.of(actualMessage)); assertEquals(1, n); }
@Test public void publishOneMessageWithNoTimestampOrIdAttribute() throws IOException { // For this test, create a new PubsubJsonClient without the timestamp attribute // or id attribute set. client = new PubsubJsonClient(null, null, mockPubsub); String expectedTopic = TOPIC.getPath(); PubsubMessage expectedPubsubMessage = new PubsubMessage() .encodeData(DATA.getBytes(StandardCharsets.UTF_8)) .setAttributes(ImmutableMap.<String, String>builder().put("k", "v").build()); PublishRequest expectedRequest = new PublishRequest().setMessages(ImmutableList.of(expectedPubsubMessage)); PublishResponse expectedResponse = new PublishResponse().setMessageIds(ImmutableList.of(MESSAGE_ID)); when((Object) (mockPubsub.projects().topics().publish(expectedTopic, expectedRequest).execute())) .thenReturn(expectedResponse); Map<String, String> attrs = new HashMap<>(); attrs.put("k", "v"); OutgoingMessage actualMessage = new OutgoingMessage(DATA.getBytes(StandardCharsets.UTF_8), attrs, MESSAGE_TIME, RECORD_ID); int n = client.publish(TOPIC, ImmutableList.of(actualMessage)); assertEquals(1, n); }
@Test public void publishOneMessageWithOnlyTimestampAndIdAttributes() throws IOException { String expectedTopic = TOPIC.getPath(); PubsubMessage expectedPubsubMessage = new PubsubMessage() .encodeData(DATA.getBytes(StandardCharsets.UTF_8)) .setAttributes( ImmutableMap.<String, String>builder() .put(TIMESTAMP_ATTRIBUTE, String.valueOf(MESSAGE_TIME)) .put(ID_ATTRIBUTE, RECORD_ID) .build()); PublishRequest expectedRequest = new PublishRequest().setMessages(ImmutableList.of(expectedPubsubMessage)); PublishResponse expectedResponse = new PublishResponse().setMessageIds(ImmutableList.of(MESSAGE_ID)); when((Object) (mockPubsub.projects().topics().publish(expectedTopic, expectedRequest).execute())) .thenReturn(expectedResponse); OutgoingMessage actualMessage = new OutgoingMessage( DATA.getBytes(StandardCharsets.UTF_8), ImmutableMap.of(), MESSAGE_TIME, RECORD_ID); int n = client.publish(TOPIC, ImmutableList.of(actualMessage)); assertEquals(1, n); }
@Override public List<SubscriptionPath> listSubscriptions(ProjectPath project, TopicPath topic) throws IOException { Subscriptions.List request = pubsub.projects().subscriptions().list(project.getPath()); ListSubscriptionsResponse response = request.execute(); if (response.getSubscriptions() == null || response.getSubscriptions().isEmpty()) { return ImmutableList.of(); } List<SubscriptionPath> subscriptions = new ArrayList<>(response.getSubscriptions().size()); while (true) { for (Subscription subscription : response.getSubscriptions()) { if (subscription.getTopic().equals(topic.getPath())) { subscriptions.add(subscriptionPathFromPath(subscription.getName())); } } if (Strings.isNullOrEmpty(response.getNextPageToken())) { break; } request.setPageToken(response.getNextPageToken()); response = request.execute(); } return subscriptions; }
@Override public int publish(TopicPath topic, List<OutgoingMessage> outgoingMessages) throws IOException { PublishRequest.Builder request = PublishRequest.newBuilder().setTopic(topic.getPath()); for (OutgoingMessage outgoingMessage : outgoingMessages) { PubsubMessage.Builder message = PubsubMessage.newBuilder().setData(ByteString.copyFrom(outgoingMessage.elementBytes)); if (outgoingMessage.attributes != null) { message.putAllAttributes(outgoingMessage.attributes); } if (timestampAttribute != null) { message .getMutableAttributes() .put(timestampAttribute, String.valueOf(outgoingMessage.timestampMsSinceEpoch)); } if (idAttribute != null && !Strings.isNullOrEmpty(outgoingMessage.recordId)) { message.getMutableAttributes().put(idAttribute, outgoingMessage.recordId); } request.addMessages(message); } PublishResponse response = publisherStub().publish(request.build()); return response.getMessageIdsCount(); }
@Override public POutput expand(PCollection<? extends T> input) { return input // assign a dummy key and global window, // this is needed to accumulate all observed events in the same state cell .apply(Window.into(new GlobalWindows())) .apply(WithKeys.of("dummyKey")) .apply( "checkAllEventsForSuccess", ParDo.of(new StatefulPredicateCheck<>(coder, formatter, successPredicate))) // signal the success/failure to the result topic .apply("publishSuccess", PubsubIO.writeStrings().to(resultTopicPath.getPath())); } }
public static TopicPath topicPathFromPath(String path) { return new TopicPath(path); }
@Override public int publish(TopicPath topic, List<OutgoingMessage> outgoingMessages) throws IOException { List<PubsubMessage> pubsubMessages = new ArrayList<>(outgoingMessages.size()); for (OutgoingMessage outgoingMessage : outgoingMessages) { PubsubMessage pubsubMessage = new PubsubMessage().encodeData(outgoingMessage.elementBytes); pubsubMessage.setAttributes(getMessageAttributes(outgoingMessage)); pubsubMessages.add(pubsubMessage); } PublishRequest request = new PublishRequest().setMessages(pubsubMessages); PublishResponse response = pubsub.projects().topics().publish(topic.getPath(), request).execute(); return response.getMessageIds().size(); }
/** * Create a random subscription for {@code topic}. Return the {@link SubscriptionPath}. It is the * responsibility of the caller to later delete the subscription. */ public SubscriptionPath createRandomSubscription( ProjectPath project, TopicPath topic, int ackDeadlineSeconds) throws IOException { // Create a randomized subscription derived from the topic name. String subscriptionName = topic.getName() + "_beam_" + ThreadLocalRandom.current().nextLong(); SubscriptionPath subscription = PubsubClient.subscriptionPathFromName(project.getId(), subscriptionName); createSubscription(topic, subscription, ackDeadlineSeconds); return subscription; }
@Override public void deleteTopic(TopicPath topic) throws IOException { pubsub.projects().topics().delete(topic.getPath()).execute(); // ignore Empty result. }
@Override public PDone expand(PBegin input) { return input .apply("Start signal", Create.of(START_SIGNAL_MESSAGE)) .apply(PubsubIO.writeStrings().to(startTopicPath.getPath())); } }
@Override public void createTopic(TopicPath topic) throws IOException { pubsub .projects() .topics() .create(topic.getPath(), new Topic()) .execute(); // ignore Topic result. }
private static Topic buildTopic(int i) { Topic topic = new Topic(); topic.setName(PubsubClient.topicPathFromName(PROJECT.getId(), "Topic" + i).getPath()); return topic; }