@Override public String toString() { return "CoGbkResultSchema: " + tupleTagList.getAll(); } }
/** Builds a schema from a tuple of {@code TupleTag<?>}s. */ public CoGbkResultSchema(TupleTagList tupleTagList) { this.tupleTagList = tupleTagList; int index = -1; for (TupleTag<?> tag : tupleTagList.getAll()) { index++; tagMap.put(tag, index); } }
@Override public int hashCode() { return tupleTagList.getAll().hashCode(); }
/** Returns the number of columns for this schema. */ public int size() { return tupleTagList.getAll().size(); }
private CloudObject toCloudObject(CoGbkResultSchema schema) { CloudObject result = CloudObject.forClass(CoGbkResultSchema.class); List<CloudObject> tags = new ArrayList<>(schema.getTupleTagList().size()); for (TupleTag<?> tag : schema.getTupleTagList().getAll()) { CloudObject tagCloudObject = CloudObject.forClass(TupleTag.class); Structs.addString(tagCloudObject, PropertyNames.VALUE, tag.getId()); tags.add(tagCloudObject); } Structs.addList(result, PropertyNames.TUPLE_TAGS, tags); return result; }
doFn, transform.getMainOutputTag(), transform.getAdditionalOutputTags().getAll(), input.getWindowingStrategy(), sideInputs, } else { int portIndex = 0; for (TupleTag<?> tag : transform.getAdditionalOutputTags().getAll()) { if (tag.equals(output.getKey())) { ports.put(pc, operator.additionalOutputPorts[portIndex]);
@Override public PCollectionTuple expand(PCollection<? extends KeyedWorkItem<K, KV<K, InputT>>> input) { PCollectionTuple outputs = PCollectionTuple.ofPrimitiveOutputsInternal( input.getPipeline(), TupleTagList.of(getMainOutputTag()).and(getAdditionalOutputTags().getAll()), // TODO Collections.emptyMap(), input.getWindowingStrategy(), input.isBounded()); return outputs; } }
@Override public <T> TransformEvaluator<T> forApplication( AppliedPTransform<?, ?, ?> application, CommittedBundle<?> inputBundle) throws Exception { @SuppressWarnings({"unchecked", "rawtypes"}) TransformEvaluator<T> evaluator = (TransformEvaluator<T>) createEvaluator( (AppliedPTransform) application, (PCollection<InputT>) inputBundle.getPCollection(), inputBundle.getKey(), ParDoTranslation.getSideInputs(application), (TupleTag<OutputT>) ParDoTranslation.getMainOutputTag(application), ParDoTranslation.getAdditionalOutputTags(application).getAll()); return evaluator; }
@Before public void setup() { MockitoAnnotations.initMocks(this); inputPc = p.apply(Create.of(1, 2, 3)); mainOutputTag = new TupleTag<Integer>() {}; additionalOutputTags = TupleTagList.empty().getAll(); bundleFactory = ImmutableListBundleFactory.create(); }
public static <OutputT> PCollectionTuple createPrimitiveOutputFor( PCollection<?> input, DoFn<?, OutputT> fn, TupleTag<OutputT> mainOutputTag, TupleTagList additionalOutputTags, Map<TupleTag<?>, Coder<?>> outputTagsToCoders, WindowingStrategy<?, ?> windowingStrategy) { DoFnSignature signature = DoFnSignatures.getSignature(fn.getClass()); PCollectionTuple outputs = PCollectionTuple.ofPrimitiveOutputsInternal( input.getPipeline(), TupleTagList.of(mainOutputTag).and(additionalOutputTags.getAll()), outputTagsToCoders, windowingStrategy, input.isBounded().and(signature.isBoundedPerElement())); // Set output type descriptor similarly to how ParDo.MultiOutput does it. outputs.get(mainOutputTag).setTypeDescriptor(fn.getOutputTypeDescriptor()); return outputs; }
@Override public PCollectionList<T> expand(PCollection<T> in) { final TupleTagList outputTags = partitionDoFn.getOutputTags(); PCollectionTuple outputs = in.apply(ParDo.of(partitionDoFn).withOutputTags(new TupleTag<Void>() {}, outputTags)); PCollectionList<T> pcs = PCollectionList.empty(in.getPipeline()); Coder<T> coder = in.getCoder(); for (TupleTag<?> outputTag : outputTags.getAll()) { // All the tuple tags are actually TupleTag<T> // And all the collections are actually PCollection<T> @SuppressWarnings("unchecked") TupleTag<T> typedOutputTag = (TupleTag<T>) outputTag; pcs = pcs.and(outputs.get(typedOutputTag).setCoder(coder)); } return pcs; }
context.getOutputs(transform), transform.getMainOutputTag(), transform.getAdditionalOutputTags().getAll(), context, new ParDoTranslationHelper.DoFnOperatorFactory<InputT, OutputT>() {
application.getTransform().getSideInputs(), application.getTransform().getMainOutputTag(), application.getTransform().getAdditionalOutputTags().getAll());
context.getOutputs(transform), transform.getMainOutputTag(), transform.getAdditionalOutputTags().getAll(), context, new ParDoTranslationHelper.DoFnOperatorFactory<
context.getOutputs(transform), transform.getMainOutputTag(), transform.getAdditionalOutputTags().getAll(), context, (doFn,
context.getOutputs(transform), transform.getMainOutputTag(), transform.getAdditionalOutputTags().getAll(), context, (doFn,
PCollectionTuple.ofPrimitiveOutputsInternal( input.getPipeline(), TupleTagList.of(mainOutputTag).and(additionalOutputTags.getAll()),
new Add(singletonView), new TupleTag<>(), TupleTagList.empty().getAll(), WindowingStrategy.globalDefault(), Collections.singletonList(singletonView),