@Override public Coder<PubsubMessage> getOutputCoder() { return outer.getNeedsAttributes() ? PubsubMessageWithAttributesCoder.of() : PubsubMessagePayloadOnlyCoder.of(); }
public PubsubSource(PubsubUnboundedSource outer) { this(outer, outer.getSubscriptionProvider()); }
StepTranslationContext stepContext = context.addStep(transform, "ParallelRead"); stepContext.addInput(PropertyNames.FORMAT, "pubsub"); if (overriddenTransform.getTopicProvider() != null) { if (overriddenTransform.getTopicProvider().isAccessible()) { stepContext.addInput( PropertyNames.PUBSUB_TOPIC, overriddenTransform.getTopic().getV1Beta1Path()); } else { stepContext.addInput( PropertyNames.PUBSUB_TOPIC_OVERRIDE, ((NestedValueProvider) overriddenTransform.getTopicProvider()).propertyName()); if (overriddenTransform.getSubscriptionProvider() != null) { if (overriddenTransform.getSubscriptionProvider().isAccessible()) { stepContext.addInput( PropertyNames.PUBSUB_SUBSCRIPTION, overriddenTransform.getSubscription().getV1Beta1Path()); } else { stepContext.addInput( PropertyNames.PUBSUB_SUBSCRIPTION_OVERRIDE, ((NestedValueProvider) overriddenTransform.getSubscriptionProvider()).propertyName()); if (overriddenTransform.getTimestampAttribute() != null) { stepContext.addInput( PropertyNames.PUBSUB_TIMESTAMP_ATTRIBUTE, overriddenTransform.getTimestampAttribute()); if (overriddenTransform.getIdAttribute() != null) { stepContext.addInput( PropertyNames.PUBSUB_ID_ATTRIBUTE, overriddenTransform.getIdAttribute());
@Test public void noSubscriptionSplitGeneratesSubscription() throws Exception { TopicPath topicPath = PubsubClient.topicPathFromName("my_project", "my_topic"); factory = PubsubTestClient.createFactoryForCreateSubscription(); PubsubUnboundedSource source = new PubsubUnboundedSource( factory, StaticValueProvider.of(PubsubClient.projectPathFromId("my_project")), StaticValueProvider.of(topicPath), null /* subscription */, null /* timestampLabel */, null /* idLabel */, false /* needsAttributes */); assertThat(source.getSubscription(), nullValue()); assertThat(source.getSubscription(), nullValue()); PipelineOptions options = PipelineOptionsFactory.create(); List<PubsubSource> splits = (new PubsubSource(source)).split(3, options); // We have at least one returned split assertThat(splits, hasSize(greaterThan(0))); for (PubsubSource split : splits) { // Each split is equal assertThat(split, equalTo(splits.get(0))); } assertThat(splits.get(0).subscriptionPath, not(nullValue())); }
private void setupOneMessage(Iterable<IncomingMessage> incoming) { now = new AtomicLong(REQ_TIME); clock = () -> now.get(); factory = PubsubTestClient.createFactoryForPull(clock, SUBSCRIPTION, ACK_TIMEOUT_S, incoming); PubsubUnboundedSource source = new PubsubUnboundedSource( clock, factory, null, null, StaticValueProvider.of(SUBSCRIPTION), TIMESTAMP_ATTRIBUTE, ID_ATTRIBUTE, true /* needsAttributes */); primSource = new PubsubSource(source); }
factory = PubsubTestClient.createFactoryForCreateSubscription(); PubsubUnboundedSource source = new PubsubUnboundedSource( factory, StaticValueProvider.of(PubsubClient.projectPathFromId("my_project")), assertThat(source.getSubscription(), nullValue()); assertThat(source.getSubscription(), nullValue());
: NestedValueProvider.of(getSubscriptionProvider(), new SubscriptionPathTranslator()); PubsubUnboundedSource source = new PubsubUnboundedSource( FACTORY,