@SuppressWarnings("unchecked") <T extends PValue> T getInput(PTransform<T, ?> transform) { return (T) Iterables.getOnlyElement(TransformInputs.nonAdditionalInputs(currentTransform)); }
@SuppressWarnings("unchecked") public <T extends PValue> T getInput(PTransform<T, ?> transform) { return (T) Iterables.getOnlyElement(TransformInputs.nonAdditionalInputs(currentTransform)); }
@SuppressWarnings("unchecked") <T extends PValue> T getInput(PTransform<T, ?> transform) { return (T) Iterables.getOnlyElement(TransformInputs.nonAdditionalInputs(currentTransform)); }
@SuppressWarnings("unchecked") public <T extends PValue> T getInput(PTransform<T, ?> transform) { return (T) Iterables.getOnlyElement(TransformInputs.nonAdditionalInputs(currentTransform)); }
@SuppressWarnings("unchecked") <T extends PValue> T getInput(PTransform<T, ?> transform) { return (T) Iterables.getOnlyElement(TransformInputs.nonAdditionalInputs(currentTransform)); }
@SuppressWarnings("unchecked") public <T extends PValue> T getInput(PTransform<T, ?> transform) { return (T) Iterables.getOnlyElement(TransformInputs.nonAdditionalInputs(currentTransform)); }
/** * @param beamNode the beam node to be translated. * @param pipeline pipeline. * @return true if the main input bounded. */ private static boolean isMainInputBounded(final TransformHierarchy.Node beamNode, final Pipeline pipeline) { final AppliedPTransform pTransform = beamNode.toAppliedPTransform(pipeline); final PCollection<?> mainInput = (PCollection<?>) Iterables.getOnlyElement(TransformInputs.nonAdditionalInputs(pTransform)); return mainInput.isBounded() == PCollection.IsBounded.BOUNDED; } }
public <T extends PValue> T getInput(PTransform<T, ?> transform) { @SuppressWarnings("unchecked") T input = (T) Iterables.getOnlyElement(TransformInputs.nonAdditionalInputs(getCurrentTransform())); return input; }
@Override public <InputT extends PValue> InputT getInput(PTransform<InputT, ?> transform) { return (InputT) Iterables.getOnlyElement( TransformInputs.nonAdditionalInputs(getCurrentTransform(transform))); }
public <InputT extends PValue> InputT getInput() { return (InputT) Iterables.getOnlyElement(TransformInputs.nonAdditionalInputs(getCurrentTransform())); }
public PValue getInput() { return Iterables.getOnlyElement(TransformInputs.nonAdditionalInputs(getCurrentTransform())); }
/** * @param beamNode the beam node to be translated. * @param pipeline pipeline. * @return true if the main input has global window. */ private static boolean isGlobalWindow(final TransformHierarchy.Node beamNode, final Pipeline pipeline) { final AppliedPTransform pTransform = beamNode.toAppliedPTransform(pipeline); final PCollection<?> mainInput = (PCollection<?>) Iterables.getOnlyElement(TransformInputs.nonAdditionalInputs(pTransform)); return mainInput.getWindowingStrategy().getWindowFn() instanceof GlobalWindows; }
@Test public void nonAdditionalInputsWithNoInputSucceeds() { AppliedPTransform<PInput, POutput, TestTransform> transform = AppliedPTransform.of( "input-free", Collections.emptyMap(), Collections.emptyMap(), new TestTransform(), pipeline); assertThat(TransformInputs.nonAdditionalInputs(transform), Matchers.empty()); }
protected boolean shouldDefer(TransformHierarchy.Node node) { TransformInputs.nonAdditionalInputs(node.toAppliedPTransform(getPipeline())); if (nonAdditionalInputs.size() != 1) { return false;
@Test public void nonAdditionalInputsWithOneMainInputSucceeds() { PCollection<Long> input = pipeline.apply(GenerateSequence.from(1L)); AppliedPTransform<PInput, POutput, TestTransform> transform = AppliedPTransform.of( "input-single", Collections.singletonMap(new TupleTag<Long>() {}, input), Collections.emptyMap(), new TestTransform(), pipeline); assertThat(TransformInputs.nonAdditionalInputs(transform), Matchers.containsInAnyOrder(input)); }
@Test public void nonAdditionalInputsWithMultipleNonAdditionalInputsSucceeds() { Map<TupleTag<?>, PValue> allInputs = new HashMap<>(); PCollection<Integer> mainInts = pipeline.apply("MainInput", Create.of(12, 3)); allInputs.put(new TupleTag<Integer>() {}, mainInts); PCollection<Void> voids = pipeline.apply("VoidInput", Create.empty(VoidCoder.of())); allInputs.put(new TupleTag<Void>() {}, voids); AppliedPTransform<PInput, POutput, TestTransform> transform = AppliedPTransform.of( "additional-free", allInputs, Collections.emptyMap(), new TestTransform(), pipeline); assertThat( TransformInputs.nonAdditionalInputs(transform), Matchers.containsInAnyOrder(voids, mainInts)); }
private static <K, InputT, AccumT> Coder<AccumT> extractAccumulatorCoder( GlobalCombineFn<InputT, AccumT, ?> combineFn, AppliedPTransform<PCollection<KV<K, InputT>>, ?, Combine.PerKey<K, InputT, ?>> transform) throws CannotProvideCoderException { @SuppressWarnings("unchecked") PCollection<KV<K, InputT>> mainInput = (PCollection<KV<K, InputT>>) Iterables.getOnlyElement(TransformInputs.nonAdditionalInputs(transform)); KvCoder<K, InputT> inputCoder = (KvCoder<K, InputT>) mainInput.getCoder(); return AppliedCombineFn.withInputCoder( combineFn, transform.getPipeline().getCoderRegistry(), inputCoder, transform.getTransform().getSideInputs(), ((PCollection<?>) Iterables.getOnlyElement(transform.getOutputs().values())) .getWindowingStrategy()) .getAccumulatorCoder(); }
@Test public void nonAdditionalInputsWithOnlyAdditionalInputsThrows() { Map<TupleTag<?>, PValue> additionalInputs = new HashMap<>(); additionalInputs.put(new TupleTag<String>() {}, pipeline.apply(Create.of("1, 2", "3"))); additionalInputs.put(new TupleTag<Long>() {}, pipeline.apply(GenerateSequence.from(3L))); AppliedPTransform<PInput, POutput, TestTransform> transform = AppliedPTransform.of( "additional-only", additionalInputs, Collections.emptyMap(), new TestTransform(additionalInputs), pipeline); thrown.expect(IllegalArgumentException.class); thrown.expectMessage("at least one"); TransformInputs.nonAdditionalInputs(transform); }
@Test public void nonAdditionalInputsWithAdditionalInputsSucceeds() { Map<TupleTag<?>, PValue> additionalInputs = new HashMap<>(); additionalInputs.put(new TupleTag<String>() {}, pipeline.apply(Create.of("1, 2", "3"))); additionalInputs.put(new TupleTag<Long>() {}, pipeline.apply(GenerateSequence.from(3L))); Map<TupleTag<?>, PValue> allInputs = new HashMap<>(); PCollection<Integer> mainInts = pipeline.apply("MainInput", Create.of(12, 3)); allInputs.put(new TupleTag<Integer>() {}, mainInts); PCollection<Void> voids = pipeline.apply("VoidInput", Create.empty(VoidCoder.of())); allInputs.put(new TupleTag<Void>() {}, voids); allInputs.putAll(additionalInputs); AppliedPTransform<PInput, POutput, TestTransform> transform = AppliedPTransform.of( "additional", allInputs, Collections.emptyMap(), new TestTransform(additionalInputs), pipeline); assertThat( TransformInputs.nonAdditionalInputs(transform), Matchers.containsInAnyOrder(mainInts, voids)); }
/** * Create a group by key transform. * It returns GroupByKeyAndWindowDoFnTransform if window function is not default. * @param ctx translation context * @param beamNode the beam node to be translated * @return group by key transform */ private static Transform createGBKTransform( final PipelineTranslationContext ctx, final TransformHierarchy.Node beamNode) { final AppliedPTransform pTransform = beamNode.toAppliedPTransform(ctx.getPipeline()); final PCollection<?> mainInput = (PCollection<?>) Iterables.getOnlyElement(TransformInputs.nonAdditionalInputs(pTransform)); final TupleTag mainOutputTag = new TupleTag<>(); if (isGlobalWindow(beamNode, ctx.getPipeline())) { return new GroupByKeyTransform(); } else { return new GroupByKeyAndWindowDoFnTransform( getOutputCoders(pTransform), mainOutputTag, mainInput.getWindowingStrategy(), ctx.getPipelineOptions(), SystemReduceFn.buffering(mainInput.getCoder()), DisplayData.from(beamNode.getTransform())); } }