@ProcessElement public void processElement(ProcessContext c) throws Exception { elementCounter.inc(); PubsubMessage message = c.element(); byte[] elementBytes = message.getPayload(); Map<String, String> attributes = message.getAttributeMap(); long timestampMsSinceEpoch = c.timestamp().getMillis(); @Nullable String recordId = null; switch (recordIdMethod) { case NONE: break; case DETERMINISTIC: recordId = Hashing.murmur3_128().hashBytes(elementBytes).toString(); break; case RANDOM: // Since these elements go through a GroupByKey, any failures while sending to // Pubsub will be retried without falling back and generating a new record id. // Thus even though we may send the same message to Pubsub twice, it is guaranteed // to have the same record id. recordId = UUID.randomUUID().toString(); break; } c.output( KV.of( ThreadLocalRandom.current().nextInt(numShards), new OutgoingMessage(elementBytes, attributes, timestampMsSinceEpoch, recordId))); }
new OutgoingMessage( str.getBytes(StandardCharsets.UTF_8), ImmutableMap.of(),
try { OutgoingMessage actualMessage = new OutgoingMessage( DATA.getBytes(StandardCharsets.UTF_8), ATTRIBUTES, MESSAGE_TIME, RECORD_ID); int n = client.publish(TOPIC, ImmutableList.of(actualMessage));
String str = String.valueOf(i); outgoing.add( new OutgoingMessage( str.getBytes(StandardCharsets.UTF_8), ImmutableMap.of(),
@Test public void sendOneMessage() throws IOException { List<OutgoingMessage> outgoing = ImmutableList.of( new OutgoingMessage( DATA.getBytes(StandardCharsets.UTF_8), ATTRIBUTES, TIMESTAMP, getRecordId(DATA))); int batchSize = 1; int batchBytes = 1; try (PubsubTestClientFactory factory = PubsubTestClient.createFactoryForPublish(TOPIC, outgoing, ImmutableList.of())) { PubsubUnboundedSink sink = new PubsubUnboundedSink( factory, StaticValueProvider.of(TOPIC), TIMESTAMP_ATTRIBUTE, ID_ATTRIBUTE, NUM_SHARDS, batchSize, batchBytes, Duration.standardSeconds(2), RecordIdMethod.DETERMINISTIC); p.apply(Create.of(ImmutableList.of(DATA))).apply(ParDo.of(new Stamp(ATTRIBUTES))).apply(sink); p.run(); } // The PubsubTestClientFactory will assert fail on close if the actual published // message does not match the expected publish message. }
@Test public void sendOneMessageWithoutAttributes() throws IOException { List<OutgoingMessage> outgoing = ImmutableList.of( new OutgoingMessage( DATA.getBytes(StandardCharsets.UTF_8), null /* attributes */, TIMESTAMP, getRecordId(DATA))); try (PubsubTestClientFactory factory = PubsubTestClient.createFactoryForPublish(TOPIC, outgoing, ImmutableList.of())) { PubsubUnboundedSink sink = new PubsubUnboundedSink( factory, StaticValueProvider.of(TOPIC), TIMESTAMP_ATTRIBUTE, ID_ATTRIBUTE, NUM_SHARDS, 1 /* batchSize */, 1 /* batchBytes */, Duration.standardSeconds(2), RecordIdMethod.DETERMINISTIC); p.apply(Create.of(ImmutableList.of(DATA))) .apply(ParDo.of(new Stamp(null /* attributes */))) .apply(sink); p.run(); } // The PubsubTestClientFactory will assert fail on close if the actual published // message does not match the expected publish message. }
@Test public void publishOneMessage() throws IOException { String expectedTopic = TOPIC.getPath(); PubsubMessage expectedPubsubMessage = new PubsubMessage() .encodeData(DATA.getBytes(StandardCharsets.UTF_8)) .setAttributes( ImmutableMap.<String, String>builder() .put(TIMESTAMP_ATTRIBUTE, String.valueOf(MESSAGE_TIME)) .put(ID_ATTRIBUTE, RECORD_ID) .put("k", "v") .build()); PublishRequest expectedRequest = new PublishRequest().setMessages(ImmutableList.of(expectedPubsubMessage)); PublishResponse expectedResponse = new PublishResponse().setMessageIds(ImmutableList.of(MESSAGE_ID)); when((Object) (mockPubsub.projects().topics().publish(expectedTopic, expectedRequest).execute())) .thenReturn(expectedResponse); Map<String, String> attrs = new HashMap<>(); attrs.put("k", "v"); OutgoingMessage actualMessage = new OutgoingMessage(DATA.getBytes(StandardCharsets.UTF_8), attrs, MESSAGE_TIME, RECORD_ID); int n = client.publish(TOPIC, ImmutableList.of(actualMessage)); assertEquals(1, n); }
@Test public void publishOneMessageWithNoTimestampOrIdAttribute() throws IOException { // For this test, create a new PubsubJsonClient without the timestamp attribute // or id attribute set. client = new PubsubJsonClient(null, null, mockPubsub); String expectedTopic = TOPIC.getPath(); PubsubMessage expectedPubsubMessage = new PubsubMessage() .encodeData(DATA.getBytes(StandardCharsets.UTF_8)) .setAttributes(ImmutableMap.<String, String>builder().put("k", "v").build()); PublishRequest expectedRequest = new PublishRequest().setMessages(ImmutableList.of(expectedPubsubMessage)); PublishResponse expectedResponse = new PublishResponse().setMessageIds(ImmutableList.of(MESSAGE_ID)); when((Object) (mockPubsub.projects().topics().publish(expectedTopic, expectedRequest).execute())) .thenReturn(expectedResponse); Map<String, String> attrs = new HashMap<>(); attrs.put("k", "v"); OutgoingMessage actualMessage = new OutgoingMessage(DATA.getBytes(StandardCharsets.UTF_8), attrs, MESSAGE_TIME, RECORD_ID); int n = client.publish(TOPIC, ImmutableList.of(actualMessage)); assertEquals(1, n); }
@Test public void publishOneMessageWithOnlyTimestampAndIdAttributes() throws IOException { String expectedTopic = TOPIC.getPath(); PubsubMessage expectedPubsubMessage = new PubsubMessage() .encodeData(DATA.getBytes(StandardCharsets.UTF_8)) .setAttributes( ImmutableMap.<String, String>builder() .put(TIMESTAMP_ATTRIBUTE, String.valueOf(MESSAGE_TIME)) .put(ID_ATTRIBUTE, RECORD_ID) .build()); PublishRequest expectedRequest = new PublishRequest().setMessages(ImmutableList.of(expectedPubsubMessage)); PublishResponse expectedResponse = new PublishResponse().setMessageIds(ImmutableList.of(MESSAGE_ID)); when((Object) (mockPubsub.projects().topics().publish(expectedTopic, expectedRequest).execute())) .thenReturn(expectedResponse); OutgoingMessage actualMessage = new OutgoingMessage( DATA.getBytes(StandardCharsets.UTF_8), ImmutableMap.of(), MESSAGE_TIME, RECORD_ID); int n = client.publish(TOPIC, ImmutableList.of(actualMessage)); assertEquals(1, n); }
@Test public void publishOneMessage() throws IOException { OutgoingMessage expectedOutgoingMessage = new OutgoingMessage(DATA.getBytes(StandardCharsets.UTF_8), null, MESSAGE_TIME, MESSAGE_ID); try (PubsubTestClientFactory factory = PubsubTestClient.createFactoryForPublish( TOPIC, Sets.newHashSet(expectedOutgoingMessage), ImmutableList.of())) { try (PubsubTestClient client = (PubsubTestClient) factory.newClient(null, null, null)) { client.publish(TOPIC, ImmutableList.of(expectedOutgoingMessage)); } } } }
private PubsubClient.OutgoingMessage toOutgoingMessage(PubsubMessage message) { return new PubsubClient.OutgoingMessage( message.getPayload(), message.getAttributeMap(), DateTime.now().getMillis(), null); } }
@Override public OutgoingMessage decode(InputStream inStream) throws CoderException, IOException { byte[] elementBytes = ByteArrayCoder.of().decode(inStream); Map<String, String> attributes = ATTRIBUTES_CODER.decode(inStream); long timestampMsSinceEpoch = BigEndianLongCoder.of().decode(inStream); @Nullable String recordId = RECORD_ID_CODER.decode(inStream); return new OutgoingMessage(elementBytes, attributes, timestampMsSinceEpoch, recordId); } }
@Test public void saneCoder() throws Exception { OutgoingMessage message = new OutgoingMessage( DATA.getBytes(StandardCharsets.UTF_8), ImmutableMap.of(), TIMESTAMP, getRecordId(DATA)); CoderProperties.coderDecodeEncodeEqual(PubsubUnboundedSink.CODER, message); CoderProperties.coderSerializable(PubsubUnboundedSink.CODER); }
public int publish(TopicPath topicPath, Message message) throws IOException { List<OutgoingMessage> messages = Lists.newArrayList( new OutgoingMessage( message.toByteArray(), new HashMap<>(), System.currentTimeMillis(), null)); return getClient().publish(topicPath, messages); } }