@Override public PDone expand(PCollection<IndexedRecord> in) { PubSubDatasetProperties dataset = properties.getDatasetProperties(); PubSubDatastoreProperties datastore = dataset.getDatastoreProperties(); prepareTopicSubscription(properties); PubsubIO.Write<PubsubMessage> pubsubWrite = PubsubIO.writeMessages().to( String.format("projects/%s/topics/%s", datastore.projectName.getValue(), dataset.topic.getValue())); if (properties.idLabel.getValue() != null && !"".equals(properties.idLabel.getValue())) { pubsubWrite.withIdAttribute(properties.idLabel.getValue()); } if (properties.timestampLabel.getValue() != null && !"".equals(properties.timestampLabel.getValue())) { pubsubWrite.withTimestampAttribute(properties.timestampLabel.getValue()); } switch (dataset.valueFormat.getValue()) { case CSV: { return in.apply(MapElements.via(new FormatCsv(dataset.fieldDelimiter.getValue()))).apply(pubsubWrite); } case AVRO: { return in.apply(MapElements.via(new FormatAvro())).apply(pubsubWrite); } default: throw new RuntimeException("To be implemented: " + dataset.valueFormat.getValue()); } }
@Test public void testWriteDisplayData() { String topic = "projects/project/topics/topic"; PubsubIO.Write<?> write = PubsubIO.writeStrings() .to(topic) .withTimestampAttribute("myTimestamp") .withIdAttribute("myId"); DisplayData displayData = DisplayData.from(write); assertThat(displayData, hasDisplayItem("topic", topic)); assertThat(displayData, hasDisplayItem("timestampAttribute", "myTimestamp")); assertThat(displayData, hasDisplayItem("idAttribute", "myId")); }
private PubsubIO.Write<PubsubMessage> writeMessagesToDlq() { PubsubIO.Write<PubsubMessage> write = PubsubIO.writeMessages().to(getDeadLetterQueue()); return (getTimestampAttribute() == null) ? write : write.withTimestampAttribute(getTimestampAttribute()); }