/** * Returns a {@link TupleTagList} containing the given {@link TupleTag TupleTags}, in order. * * <p>Longer {@link TupleTagList TupleTagLists} can be created by calling {@link #and} on the * result. */ public static TupleTagList of(List<TupleTag<?>> tags) { return empty().and(tags); }
public TupleTagList getTags() { TupleTagList list = TupleTagList.empty(); for (TupleTag<FeatureRowExtended> tag : tagMap.values()) { list = list.and(tag); } return list; }
/** * Returns a singleton {@link TupleTagList} containing the given {@link TupleTag}. * * <p>Longer {@link TupleTagList TupleTagLists} can be created by calling {@link #and} on the * result. */ public static TupleTagList of(TupleTag<?> tag) { return empty().and(tag); }
/** * Constructs a PartitionDoFn. * * @throws IllegalArgumentException if {@code numPartitions <= 0} */ public PartitionDoFn(int numPartitions, PartitionFn<? super X> partitionFn) { if (numPartitions <= 0) { throw new IllegalArgumentException("numPartitions must be > 0"); } this.numPartitions = numPartitions; this.partitionFn = partitionFn; TupleTagList buildOutputTags = TupleTagList.empty(); for (int partition = 0; partition < numPartitions; partition++) { buildOutputTags = buildOutputTags.and(new TupleTag<X>()); } outputTags = buildOutputTags; }
public static CoGbkResultSchema of(List<TupleTag<?>> tags) { TupleTagList tupleTags = TupleTagList.empty(); for (TupleTag<?> tag : tags) { tupleTags = tupleTags.and(tag); } return new CoGbkResultSchema(tupleTags); }
/** * Returns a new {@link CoGbkResult} based on this, with the given tag and given data added to it. */ public <V> CoGbkResult and(TupleTag<V> tag, List<V> data) { if (nextTestUnionId != schema.size()) { throw new IllegalArgumentException( "Attempting to call and() on a CoGbkResult apparently not created by" + " of()."); } List<Iterable<?>> valueMap = new ArrayList<>(this.valueMap); valueMap.add(data); return new CoGbkResult( new CoGbkResultSchema(schema.getTupleTagList().and(tag)), valueMap, nextTestUnionId + 1); }
/** * Returns a new {@code KeyedPCollectionTuple<K>} that is the same as this, appended with the * given PCollection. */ public <V> KeyedPCollectionTuple<K> and(TupleTag<V> tag, PCollection<KV<K, V>> pc) { if (pc.getPipeline() != getPipeline()) { throw new IllegalArgumentException("PCollections come from different Pipelines"); } TaggedKeyedPCollection<K, ?> wrapper = new TaggedKeyedPCollection<>(tag, pc); Coder<K> myKeyCoder = keyCoder == null ? getKeyCoder(pc) : keyCoder; List<TaggedKeyedPCollection<K, ?>> newKeyedCollections = copyAddLast(keyedCollections, wrapper); return new KeyedPCollectionTuple<>( getPipeline(), newKeyedCollections, schema.getTupleTagList().and(tag), myKeyCoder); }
@Override public PCollectionTuple expand(PCollection<? extends KeyedWorkItem<K, KV<K, InputT>>> input) { PCollectionTuple outputs = PCollectionTuple.ofPrimitiveOutputsInternal( input.getPipeline(), TupleTagList.of(getMainOutputTag()).and(getAdditionalOutputTags().getAll()), // TODO Collections.emptyMap(), input.getWindowingStrategy(), input.isBounded()); return outputs; } }
public static <OutputT> PCollectionTuple createPrimitiveOutputFor( PCollection<?> input, DoFn<?, OutputT> fn, TupleTag<OutputT> mainOutputTag, TupleTagList additionalOutputTags, Map<TupleTag<?>, Coder<?>> outputTagsToCoders, WindowingStrategy<?, ?> windowingStrategy) { DoFnSignature signature = DoFnSignatures.getSignature(fn.getClass()); PCollectionTuple outputs = PCollectionTuple.ofPrimitiveOutputsInternal( input.getPipeline(), TupleTagList.of(mainOutputTag).and(additionalOutputTags.getAll()), outputTagsToCoders, windowingStrategy, input.isBounded().and(signature.isBoundedPerElement())); // Set output type descriptor similarly to how ParDo.MultiOutput does it. outputs.get(mainOutputTag).setTypeDescriptor(fn.getOutputTypeDescriptor()); return outputs; }
@Test public void testMultiOutputAppliedMultipleTimesDifferentOutputs() { pipeline.enableAbandonedNodeEnforcement(false); PCollection<Long> longs = pipeline.apply(GenerateSequence.from(0)); TupleTag<Long> mainOut = new TupleTag<>(); final TupleTag<String> valueAsString = new TupleTag<>(); final TupleTag<Integer> valueAsInt = new TupleTag<>(); DoFn<Long, Long> fn = new DoFn<Long, Long>() { @ProcessElement public void processElement(ProcessContext cxt, @Element Long element) { cxt.output(cxt.element()); cxt.output(valueAsString, Long.toString(cxt.element())); cxt.output(valueAsInt, element.intValue()); } }; ParDo.MultiOutput<Long, Long> parDo = ParDo.of(fn).withOutputTags(mainOut, TupleTagList.of(valueAsString).and(valueAsInt)); PCollectionTuple firstApplication = longs.apply("first", parDo); PCollectionTuple secondApplication = longs.apply("second", parDo); assertThat(firstApplication, not(equalTo(secondApplication))); assertThat( firstApplication.getAll().keySet(), Matchers.containsInAnyOrder(mainOut, valueAsString, valueAsInt)); assertThat( secondApplication.getAll().keySet(), Matchers.containsInAnyOrder(mainOut, valueAsString, valueAsInt)); }
PCollectionTuple.ofPrimitiveOutputsInternal( input.getPipeline(), TupleTagList.of(mainOutputTag).and(additionalOutputTags.getAll()),
public void drop() {} }) .withOutputTags(tag1, TupleTagList.of(tag2).and(tag3)));
@Test @Category(ValidatesRunner.class) public void testParDoWithEmptyTaggedOutput() { TupleTag<String> mainOutputTag = new TupleTag<String>("main") {}; TupleTag<String> additionalOutputTag1 = new TupleTag<String>("additional1") {}; TupleTag<String> additionalOutputTag2 = new TupleTag<String>("additional2") {}; PCollectionTuple outputs = pipeline .apply(Create.empty(VarIntCoder.of())) .apply( ParDo.of(new TestNoOutputDoFn()) .withOutputTags( mainOutputTag, TupleTagList.of(additionalOutputTag1).and(additionalOutputTag2))); PAssert.that(outputs.get(mainOutputTag)).empty(); PAssert.that(outputs.get(additionalOutputTag1)).empty(); PAssert.that(outputs.get(additionalOutputTag2)).empty(); pipeline.run(); }
@Parameters(name = "{index}: {0}") public static Iterable<ParDo.MultiOutput<?, ?>> data() { return ImmutableList.of( ParDo.of(new DropElementsFn()).withOutputTags(new TupleTag<>(), TupleTagList.empty()), ParDo.of(new DropElementsFn()) .withOutputTags(new TupleTag<>(), TupleTagList.empty()) .withSideInputs(singletonSideInput, multimapSideInput), ParDo.of(new DropElementsFn()) .withOutputTags( new TupleTag<>(), TupleTagList.of(new TupleTag<byte[]>() {}).and(new TupleTag<Integer>() {})) .withSideInputs(singletonSideInput, multimapSideInput), ParDo.of(new DropElementsFn()) .withOutputTags( new TupleTag<>(), TupleTagList.of(new TupleTag<byte[]>() {}).and(new TupleTag<Integer>() {})), ParDo.of(new SplittableDropElementsFn()) .withOutputTags(new TupleTag<>(), TupleTagList.empty()), ParDo.of(new StateTimerDropElementsFn()) .withOutputTags(new TupleTag<>(), TupleTagList.empty())); }