/** Publish messages to {@link #topicPath()}. */ public void publish(List<PubsubMessage> messages) throws IOException { List<PubsubClient.OutgoingMessage> outgoingMessages = messages.stream().map(this::toOutgoingMessage).collect(toList()); pubsub.publish(eventsTopicPath, outgoingMessages); }
private void tearDown() throws IOException { if (pubsub == null) { return; } try { if (resultTopicPath != null) { pubsub.deleteTopic(resultTopicPath); } } finally { pubsub.close(); pubsub = null; resultTopicPath = null; } }
private List<SubscriptionPath> listSubscriptions(ProjectPath projectPath, TopicPath topicPath) throws IOException { return pubsub.listSubscriptions(projectPath, topicPath); }
/** * Create a random subscription for {@code topic}. Return the {@link SubscriptionPath}. It is the * responsibility of the caller to later delete the subscription. */ public SubscriptionPath createRandomSubscription( ProjectPath project, TopicPath topic, int ackDeadlineSeconds) throws IOException { // Create a randomized subscription derived from the topic name. String subscriptionName = topic.getName() + "_beam_" + ThreadLocalRandom.current().nextLong(); SubscriptionPath subscription = PubsubClient.subscriptionPathFromName(project.getId(), subscriptionName); createSubscription(topic, subscription, ackDeadlineSeconds); return subscription; }
private void initializePubsub(Description description) throws IOException { pubsub = PubsubGrpcClient.FACTORY.newClient( NO_TIMESTAMP_ATTRIBUTE, NO_ID_ATTRIBUTE, pipelineOptions); TopicPath eventsTopicPathTmp = PubsubClient.topicPathFromName( pipelineOptions.getProject(), createTopicName(description, EVENTS_TOPIC_NAME)); pubsub.createTopic(eventsTopicPathTmp); eventsTopicPath = eventsTopicPathTmp; }
@Test public void noSubscriptionSplitGeneratesSubscription() throws Exception { TopicPath topicPath = PubsubClient.topicPathFromName("my_project", "my_topic"); factory = PubsubTestClient.createFactoryForCreateSubscription(); PubsubUnboundedSource source = new PubsubUnboundedSource( factory, StaticValueProvider.of(PubsubClient.projectPathFromId("my_project")), StaticValueProvider.of(topicPath), null /* subscription */, null /* timestampLabel */, null /* idLabel */, false /* needsAttributes */); assertThat(source.getSubscription(), nullValue()); assertThat(source.getSubscription(), nullValue()); PipelineOptions options = PipelineOptionsFactory.create(); List<PubsubSource> splits = (new PubsubSource(source)).split(3, options); // We have at least one returned split assertThat(splits, hasSize(greaterThan(0))); for (PubsubSource split : splits) { // Each split is equal assertThat(split, equalTo(splits.get(0))); } assertThat(splits.get(0).subscriptionPath, not(nullValue())); }
private static Subscription buildSubscription(int i) { Subscription subscription = new Subscription(); subscription.setName( PubsubClient.subscriptionPathFromName(PROJECT.getId(), "Subscription" + i).getPath()); subscription.setTopic(PubsubClient.topicPathFromName(PROJECT.getId(), "Topic" + i).getPath()); return subscription; } }
@Test public void listSubscriptions() throws Exception { ListSubscriptionsResponse expectedResponse1 = new ListSubscriptionsResponse(); expectedResponse1.setSubscriptions(Collections.singletonList(buildSubscription(1))); expectedResponse1.setNextPageToken("AVgJH3Z7aHxiDBs"); ListSubscriptionsResponse expectedResponse2 = new ListSubscriptionsResponse(); expectedResponse2.setSubscriptions(Collections.singletonList(buildSubscription(2))); Subscriptions.List request = mockPubsub.projects().subscriptions().list(PROJECT.getPath()); when((Object) (request.execute())).thenReturn(expectedResponse1, expectedResponse2); final TopicPath topic101 = PubsubClient.topicPathFromName("testProject", "Topic2"); List<SubscriptionPath> subscriptionPaths = client.listSubscriptions(PROJECT, topic101); assertEquals(1, subscriptionPaths.size()); }
@FinishBundle public void finishBundle() throws Exception { pubsubClient.close(); pubsubClient = null; }
pubsubClient.get().pull(requestTimeMsSinceEpoch, subscription, PULL_BATCH_SIZE, true); if (receivedMessages.isEmpty()) {
@Override public TopicPath apply(PubsubTopic from) { return PubsubClient.topicPathFromName(from.project, from.topic); } }
private String pollForResultForDuration( SubscriptionPath signalSubscriptionPath, Duration duration) throws IOException { List<PubsubClient.IncomingMessage> signal = null; DateTime endPolling = DateTime.now().plus(duration.getMillis()); do { try { signal = pubsub.pull(DateTime.now().getMillis(), signalSubscriptionPath, 1, false); pubsub.acknowledge( signalSubscriptionPath, signal.stream().map(m -> m.ackId).collect(toList())); break; } catch (StatusRuntimeException e) { if (!Status.DEADLINE_EXCEEDED.equals(e.getStatus())) { LOG.warn( "(Will retry) Error while polling {} for signal: {}", signalSubscriptionPath, e.getStatus()); } sleep(500); } } while (DateTime.now().isBefore(endPolling)); if (signal == null) { throw new AssertionError( String.format( "Did not receive signal on %s in %ss", signalSubscriptionPath, duration.getStandardSeconds())); } return new String(signal.get(0).elementBytes, UTF_8); }
"Cannot create subscription to topic %s because pipeline option 'project' not specified", topicPath); projectPath = PubsubClient.projectPathFromId(options.as(GcpOptions.class).getProject()); timestampAttribute, idAttribute, options.as(PubsubOptions.class))) { SubscriptionPath subscriptionPath = pubsubClient.createRandomSubscription(projectPath, topicPath, DEAULT_ACK_TIMEOUT_SEC); LOG.warn( "Created subscription {} to topic {}."
@Override public SubscriptionPath apply(PubsubSubscription from) { return PubsubClient.subscriptionPathFromName(from.project, from.subscription); } }
@Override public ProjectPath apply(PubsubSubscription from) { return PubsubClient.projectPathFromId(from.project); } }
/** * Acks the provided {@code ackIds} back to Pubsub, blocking until all of the messages are * ACKed. * * <p>CAUTION: May be invoked from a separate thread. * * <p>CAUTION: Retains {@code ackIds}. */ void ackBatch(List<String> ackIds) throws IOException { pubsubClient.get().acknowledge(subscription, ackIds); ackedIds.add(ackIds); }
@Override public boolean start() throws IOException { // Determine the ack timeout. ackTimeoutMs = pubsubClient.get().ackDeadlineSeconds(subscription) * 1000; return advance(); }
@Test public void noSubscriptionNoSplitGeneratesSubscription() throws Exception { TopicPath topicPath = PubsubClient.topicPathFromName("my_project", "my_topic"); factory = PubsubTestClient.createFactoryForCreateSubscription(); PubsubUnboundedSource source = new PubsubUnboundedSource( factory, StaticValueProvider.of(PubsubClient.projectPathFromId("my_project")), StaticValueProvider.of(topicPath),
/** Wait for a success signal for {@code duration}. */ public void waitForSuccess(Duration duration) throws IOException { SubscriptionPath resultSubscriptionPath = PubsubClient.subscriptionPathFromName( pipelineOptions.getProject(), "result-subscription-" + String.valueOf(ThreadLocalRandom.current().nextLong())); pubsub.createSubscription( resultTopicPath, resultSubscriptionPath, (int) duration.getStandardSeconds()); String result = pollForResultForDuration(resultSubscriptionPath, duration); if (!RESULT_SUCCESS_MESSAGE.equals(result)) { throw new AssertionError(result); } }
private void initializePubsub(Description description) throws IOException { pubsub = PubsubGrpcClient.FACTORY.newClient( NO_TIMESTAMP_ATTRIBUTE, NO_ID_ATTRIBUTE, pipelineOptions); // Example topic name: // integ-test-TestClassName-testMethodName-2018-12-11-23-32-333-<random-long>-result TopicPath resultTopicPathTmp = PubsubClient.topicPathFromName( pipelineOptions.getProject(), createTopicName(description, RESULT_TOPIC_NAME)); TopicPath startTopicPathTmp = PubsubClient.topicPathFromName( pipelineOptions.getProject(), createTopicName(description, START_TOPIC_NAME)); pubsub.createTopic(resultTopicPathTmp); pubsub.createTopic(startTopicPathTmp); // Set these after successful creation; this signals that they need teardown resultTopicPath = resultTopicPathTmp; startTopicPath = startTopicPathTmp; }