@Override public PDone expand(PCollection<PubsubMessage> input) { return PDone.in(input.getPipeline()); }
/** * Returns a singleton {@link PCollectionList} containing the given {@link PCollection}. * * <p>Longer {@link PCollectionList PCollectionLists} can be created by calling {@link #and} on * the result. */ public static <T> PCollectionList<T> of(PCollection<T> pc) { return new PCollectionList<T>(pc.getPipeline()).and(pc); }
@Override public void finishSpecifyingOutput( String transformName, PInput input, PTransform<?, ?> transform) { this.coderOrFailure = inferCoderOrFail( input, transform, getPipeline().getCoderRegistry(), getPipeline().getSchemaRegistry()); super.finishSpecifyingOutput(transformName, input, transform); }
/** Returns a new {@code KeyedPCollectionTuple<K>} with the given tag and initial PCollection. */ public static <K, InputT> KeyedPCollectionTuple<K> of( TupleTag<InputT> tag, PCollection<KV<K, InputT>> pc) { return new KeyedPCollectionTuple<K>(pc.getPipeline()).and(tag, pc); }
@Override public PCollection<KV<K, OutputT>> expand(PCollection<KV<K, Iterable<InputT>>> input) { return PCollection.createPrimitiveOutputInternal( input.getPipeline(), input.getWindowingStrategy(), input.isBounded(), outputCoder); }
@Override public PCollection<OutputT> expand(PCollection<? extends InputT> input) { return PCollection.createPrimitiveOutputInternal( input.getPipeline(), input.getWindowingStrategy(), input.isBounded(), outputCoder, onlyOutputTag); }
@Override public PDone expand(PCollection<RedisMutation> input) { input.apply(ParDo.of(dofn)); return PDone.in(input.getPipeline()); }
@Override public PCollection<List<ElemT>> expand(PCollection<List<ElemT>> input) { return PCollection.createPrimitiveOutputInternal( input.getPipeline(), input.getWindowingStrategy(), input.isBounded(), input.getCoder()); }
@Override public PDone expand(PCollection<T> input) { input.apply(ParDo.of(new GridFsWriteFn<>(this))); return PDone.in(input.getPipeline()); } }
@Override public POutput buildIOWriter(PCollection<Row> input) { input.apply(ParDo.of(new CollectorFn(tableWithRows))); return PDone.in(input.getPipeline()); }
@Override public PCollection<ElemT> expand(PCollection<ElemT> input) { return PCollection.createPrimitiveOutputInternal( input.getPipeline(), input.getWindowingStrategy(), input.isBounded(), input.getCoder()); }
@Override public PDone expand(PCollection<T> input) { input.apply(ParDo.of(new WriteFn<>(this))); return PDone.in(input.getPipeline()); }
@SuppressWarnings({"rawtypes", "unchecked"}) @Override public PCollection<KV<K, Iterable<WindowedValue<V>>>> expand(PCollection<KV<K, V>> input) { return PCollection.createPrimitiveOutputInternal( input.getPipeline(), input.getWindowingStrategy(), input.isBounded(), (Coder) GroupByKey.getOutputKvCoder(input.getCoder())); } }
@Override public PDone expand(PCollection<T> input) { input.apply(ParDo.of(new CollectorFn<T>(getName()))); return PDone.in(input.getPipeline()); }
@Override public PDone expand(PCollection<Integer> input) { // Apply an operation so that this is a composite transform. input.apply(Count.perElement()); return PDone.in(input.getPipeline()); } }
@Override public PCollection<KV<K, Iterable<V>>> expand(PCollection<KeyedWorkItem<K, V>> input) { KeyedWorkItemCoder<K, V> inputCoder = getKeyedWorkItemCoder(input.getCoder()); return PCollection.createPrimitiveOutputInternal( input.getPipeline(), outputWindowingStrategy, input.isBounded(), KvCoder.of(inputCoder.getKeyCoder(), IterableCoder.of(inputCoder.getElementCoder()))); } }
@Override public PDone expand(PCollection<SolrInputDocument> input) { checkState(getConnectionConfiguration() != null, "withConnectionConfiguration() is required"); checkState(getCollection() != null, "to() is required"); input.apply(ParDo.of(new WriteFn(this))); return PDone.in(input.getPipeline()); }
@Override public PDone expand(PCollection<String> input) { checkArgument(getConnectionFactory() != null, "withConnectionFactory() is required"); checkArgument( getQueue() != null || getTopic() != null, "Either withQueue(queue) or withTopic(topic) is required"); checkArgument( getQueue() == null || getTopic() == null, "withQueue(queue) and withTopic(topic) are exclusive"); input.apply(ParDo.of(new WriterFn(this))); return PDone.in(input.getPipeline()); }
@Override public PDone expand(PCollection<IndexedRecord> in) { TableReference table = new TableReference(); table.setProjectId(datastore.projectName.getValue()); table.setDatasetId(dataset.bqDataset.getValue()); table.setTableId(dataset.tableName.getValue()); BigQueryIO.Write bigQueryIOPTransform = BigQueryIO.writeTableRows().to(table); bigQueryIOPTransform = setTableOperation(bigQueryIOPTransform); bigQueryIOPTransform = setWriteOperation(bigQueryIOPTransform); in.apply(ParDo.of(new IndexedRecordToTableRowFn())).apply(bigQueryIOPTransform); return PDone.in(in.getPipeline()); }
@Override public PDone expand(PCollection<IndexedRecord> in) { TableReference table = new TableReference(); table.setProjectId(datastore.projectName.getValue()); table.setDatasetId(dataset.bqDataset.getValue()); table.setTableId(dataset.tableName.getValue()); BigQueryIO.Write bigQueryIOPTransform = BigQueryIO.writeTableRows().to(table); bigQueryIOPTransform = setTableOperation(bigQueryIOPTransform); bigQueryIOPTransform = setWriteOperation(bigQueryIOPTransform); in.apply(ParDo.of(new IndexedRecordToTableRowFn())).apply(bigQueryIOPTransform); return PDone.in(in.getPipeline()); }