private PubsubIO.Write<PubsubMessage> writeMessagesToDlq() { PubsubIO.Write<PubsubMessage> write = PubsubIO.writeMessages().to(getDeadLetterQueue()); return (getTimestampAttribute() == null) ? write : write.withTimestampAttribute(getTimestampAttribute()); }
@Override public PDone expand(PCollection<IndexedRecord> in) { PubSubDatasetProperties dataset = properties.getDatasetProperties(); PubSubDatastoreProperties datastore = dataset.getDatastoreProperties(); prepareTopicSubscription(properties); PubsubIO.Write<PubsubMessage> pubsubWrite = PubsubIO.writeMessages().to( String.format("projects/%s/topics/%s", datastore.projectName.getValue(), dataset.topic.getValue())); if (properties.idLabel.getValue() != null && !"".equals(properties.idLabel.getValue())) { pubsubWrite.withIdAttribute(properties.idLabel.getValue()); } if (properties.timestampLabel.getValue() != null && !"".equals(properties.timestampLabel.getValue())) { pubsubWrite.withTimestampAttribute(properties.timestampLabel.getValue()); } switch (dataset.valueFormat.getValue()) { case CSV: { return in.apply(MapElements.via(new FormatCsv(dataset.fieldDelimiter.getValue()))).apply(pubsubWrite); } case AVRO: { return in.apply(MapElements.via(new FormatAvro())).apply(pubsubWrite); } default: throw new RuntimeException("To be implemented: " + dataset.valueFormat.getValue()); } }
/** * Runs the pipeline with the supplied options. * * @param options The execution parameters to the pipeline. * @return The result of the pipeline execution. */ public static PipelineResult run(Options options) { // Create the pipeline Pipeline pipeline = Pipeline.create(options); /** * Steps: 1) Read PubSubMessage with attributes from input PubSub subscription. * 2) Apply any filters if an attribute=value pair is provided. * 3) Write each PubSubMessage to output PubSub topic. */ pipeline .apply( "Read PubSub Events", PubsubIO.readMessagesWithAttributes().fromSubscription(options.getInputSubscription())) .apply("Filter Events If Enabled", ParDo.of(new ExtractAndFilterEventsFn())) .apply("Write PubSub Events", PubsubIO.writeMessages().to(options.getOutputTopic())); // Execute the pipeline and return the result. return pipeline.run(); } }