@Override public PCollectionView<T> expand(PCollection<T> input) { try { GroupByKey.applicableTo(input); } catch (IllegalStateException e) { throw new IllegalStateException("Unable to create a side-input view from input", e); } Combine.Globally<T, T> singletonCombine = Combine.globally(new SingletonCombineFn<>(hasDefault, input.getCoder(), defaultValue)); if (!hasDefault) { singletonCombine = singletonCombine.withoutDefaults(); } return input.apply(singletonCombine.asSingletonView()); } }
@Before public void setup() { MockitoAnnotations.initMocks(this); PCollection<Integer> created = p.apply(Create.of(1, 2, 3)); singletonView = created .apply(Window.into(new IdentitySideInputWindowFn())) .apply(Sum.integersGlobally().asSingletonView()); underlying = new TestDoFnRunner<>(); }
@Test public void testWithDefaultsPreservesSideInputs() { final PCollectionView<Integer> view = pipeline.apply(Create.of(1)).apply(Sum.integersGlobally().asSingletonView()); Combine.Globally<Integer, String> combine = Combine.globally(new TestCombineFnWithContext(view)) .withSideInputs(view) .withoutDefaults(); assertEquals(Collections.singletonList(view), combine.getSideInputs()); }
pipeline.apply(Create.of(1)).apply(Sum.integersGlobally().asSingletonView());
TimestampedValue.of(3, new Instant(100)))) .apply("WindowSideInput", Window.into(windowFn)) .apply("CombineSideInput", Sum.integersGlobally().asSingletonView());
sumScores.apply(Values.create()).apply(Mean.<Integer>globally().asSingletonView());
TimestampedValue.of(3, new Instant(13)))) .apply("WindowSideInput", Window.into(FixedWindows.of(Duration.millis(10)))) .apply(Sum.integersGlobally().asSingletonView());
write.withSharding(Sum.integersGlobally().asSingletonView()); assertThat( PTransformMatchers.writeWithRunnerDeterminedSharding()
@Test public void getReplacementTransformGetSideInputs() { PCollectionView<Long> sideLong = pipeline .apply("LongSideInputVals", Create.of(-1L, -2L, -4L)) .apply("SideLongView", Sum.longsGlobally().asSingletonView()); PCollectionView<List<String>> sideStrings = pipeline .apply("StringSideInputVals", Create.of("foo", "bar", "baz")) .apply("SideStringsView", View.asList()); ParDo.SingleOutput<Integer, Long> originalTransform = ParDo.of(new ToLongFn()).withSideInputs(sideLong, sideStrings); PCollection<? extends Integer> input = pipeline.apply(Create.of(1, 2, 3)); AppliedPTransform< PCollection<? extends Integer>, PCollection<Long>, ParDo.SingleOutput<Integer, Long>> application = AppliedPTransform.of( "original", input.expand(), input.apply(originalTransform).expand(), originalTransform, pipeline); PTransformReplacement<PCollection<? extends Integer>, PCollection<Long>> replacementTransform = factory.getReplacementTransform(application); ParDoSingle<Integer, Long> parDoSingle = (ParDoSingle<Integer, Long>) replacementTransform.getTransform(); assertThat(parDoSingle.getSideInputs(), containsInAnyOrder(sideStrings, sideLong)); }
@Test @Category(ValidatesRunner.class) public void testSideInputWithNullDefault() { final PCollectionView<Void> view = pipeline .apply("CreateSideInput", Create.of((Void) null).withCoder(VoidCoder.of())) .apply(Combine.globally((Iterable<Void> input) -> null).asSingletonView()); @SuppressWarnings("ObjectToString") PCollection<String> output = pipeline .apply("CreateMainInput", Create.of("")) .apply( "OutputMainAndSideInputs", ParDo.of( new DoFn<String, String>() { @ProcessElement public void processElement(ProcessContext c) { c.output(c.element() + c.sideInput(view)); } }) .withSideInputs(view)); PAssert.that(output).containsInAnyOrder("null"); pipeline.run(); }
@Test @Category(ValidatesRunner.class) public void testCombineGloballyAsSingletonView() { final PCollectionView<Integer> view = pipeline .apply("CreateEmptySideInput", Create.empty(BigEndianIntegerCoder.of())) .apply(Sum.integersGlobally().asSingletonView()); PCollection<Integer> output = pipeline .apply("CreateVoidMainInput", Create.of((Void) null)) .apply( "OutputSideInput", ParDo.of( new DoFn<Void, Integer>() { @ProcessElement public void processElement(ProcessContext c) { c.output(c.sideInput(view)); } }) .withSideInputs(view)); PAssert.thatSingleton(output).isEqualTo(0); pipeline.run(); }
@Before public void setup() { MockitoAnnotations.initMocks(this); PCollection<Integer> create = pipeline.apply("forBaseCollection", Create.of(1, 2, 3, 4)); mapView = create.apply("forKeyTypes", WithKeys.of("foo")).apply("asMapView", View.asMap()); singletonView = create.apply("forCombinedTypes", Mean.<Integer>globally().asSingletonView()); iterableView = create.apply("asIterableView", View.asIterable()); container = SideInputContainer.create(context, ImmutableList.of(iterableView, mapView, singletonView)); }
@Test public void testWithFanoutPreservesSideInputs() { final PCollectionView<Integer> view = pipeline.apply(Create.of(1)).apply(Sum.integersGlobally().asSingletonView()); Combine.Globally<Integer, String> combine = Combine.globally(new TestCombineFnWithContext(view)).withSideInputs(view).withFanout(1); assertEquals(Collections.singletonList(view), combine.getSideInputs()); } }
@Test public void testCombineGetName() { assertEquals("Combine.globally(SumInts)", Combine.globally(new SumInts()).getName()); assertEquals( "Combine.GloballyAsSingletonView", Combine.globally(new SumInts()).asSingletonView().getName()); assertEquals("Combine.perKey(Test)", Combine.perKey(new TestCombineFn()).getName()); assertEquals( "Combine.perKeyWithFanout(Test)", Combine.perKey(new TestCombineFn()).withHotKeyFanout(10).getName()); }