@Before public void setup() { MockitoAnnotations.initMocks(this); PCollection<Integer> created = p.apply(Create.of(1, 2, 3)); singletonView = created .apply(Window.into(new IdentitySideInputWindowFn())) .apply(Sum.integersGlobally().asSingletonView()); underlying = new TestDoFnRunner<>(); }
@Test public void testWithDefaultsPreservesSideInputs() { final PCollectionView<Integer> view = pipeline.apply(Create.of(1)).apply(Sum.integersGlobally().asSingletonView()); Combine.Globally<Integer, String> combine = Combine.globally(new TestCombineFnWithContext(view)) .withSideInputs(view) .withoutDefaults(); assertEquals(Collections.singletonList(view), combine.getSideInputs()); }
@Test public void countAssertsMultipleCallsIndependent() { PCollection<Integer> create = pipeline.apply("FirstCreate", Create.of(1, 2, 3)); PAssert.that(create).containsInAnyOrder(1, 2, 3); PAssert.thatSingleton(create.apply(Sum.integersGlobally())).isEqualTo(6); assertThat(PAssert.countAsserts(pipeline), equalTo(2)); PAssert.thatMap(pipeline.apply("CreateMap", Create.of(KV.of(1, 2)))) .isEqualTo(Collections.singletonMap(1, 2)); assertThat(PAssert.countAsserts(pipeline), equalTo(3)); } }
@Test public void countAssertsSucceeds() { PCollection<Integer> create = pipeline.apply("FirstCreate", Create.of(1, 2, 3)); PAssert.that(create).containsInAnyOrder(1, 2, 3); PAssert.thatSingleton(create.apply(Sum.integersGlobally())).isEqualTo(6); PAssert.thatMap(pipeline.apply("CreateMap", Create.of(KV.of(1, 2)))) .isEqualTo(Collections.singletonMap(1, 2)); assertThat(PAssert.countAsserts(pipeline), equalTo(3)); }
@Before public void setup() { PCollection<Integer> created = p.apply(Create.of(1, 2, 3)); PCollection<Integer> summed = created.apply(Sum.integersGlobally()); DirectGraphs.performDirectOverrides(p); DirectGraph graph = DirectGraphs.getGraph(p); create = graph.getProducer(created); sum = graph.getProducer(summed); }
@Test public void testWithFanoutPreservesSideInputs() { final PCollectionView<Integer> view = pipeline.apply(Create.of(1)).apply(Sum.integersGlobally().asSingletonView()); Combine.Globally<Integer, String> combine = Combine.globally(new TestCombineFnWithContext(view)).withSideInputs(view).withFanout(1); assertEquals(Collections.singletonList(view), combine.getSideInputs()); } }
@Override public PCollectionTuple expand(PCollection<Integer> input) { PCollection<Integer> sum = input.apply(Sum.integersGlobally()); // Fails here when attempting to construct a tuple with an unbound object. return PCollectionTuple.of(sumTag, sum) .and( doneTag, PCollection.createPrimitiveOutputInternal( input.getPipeline(), WindowingStrategy.globalDefault(), input.isBounded(), VoidCoder.of())); } }
@Test @Category(ValidatesRunner.class) public void testGlobalCombineWithDefaultsAndTriggers() { PCollection<Integer> input = pipeline.apply(Create.of(1, 1)); PCollection<String> output = input .apply( Window.<Integer>into(new GlobalWindows()) .triggering(Repeatedly.forever(AfterPane.elementCountAtLeast(1))) .accumulatingFiredPanes() .withAllowedLateness(new Duration(0), ClosingBehavior.FIRE_ALWAYS)) .apply(Sum.integersGlobally()) .apply(ParDo.of(new FormatPaneInfo())); // The actual elements produced are nondeterministic. Could be one, could be two. // But it should certainly have a final element with the correct final sum. PAssert.that(output) .satisfies( input1 -> { assertThat(input1, hasItem("2: true")); return null; }); pipeline.run(); }
write.withSharding(Sum.integersGlobally().asSingletonView()); assertThat( PTransformMatchers.writeWithRunnerDeterminedSharding()
@Test @Category(ValidatesRunner.class) public void testHotKeyCombiningWithAccumulationMode() { PCollection<Integer> input = pipeline.apply(Create.of(1, 2, 3, 4, 5)); PCollection<Integer> output = input .apply( Window.<Integer>into(new GlobalWindows()) .triggering(Repeatedly.forever(AfterPane.elementCountAtLeast(1))) .accumulatingFiredPanes() .withAllowedLateness(new Duration(0), ClosingBehavior.FIRE_ALWAYS)) .apply(Sum.integersGlobally().withoutDefaults().withFanout(2)) .apply(ParDo.of(new GetLast())); PAssert.that(output) .satisfies( input1 -> { assertThat(input1, hasItem(15)); return null; }); pipeline.run(); }
@Test public void testSumGetNames() { assertEquals("Combine.globally(SumInteger)", Sum.integersGlobally().getName()); assertEquals("Combine.globally(SumDouble)", Sum.doublesGlobally().getName()); assertEquals("Combine.globally(SumLong)", Sum.longsGlobally().getName()); assertEquals("Combine.perKey(SumInteger)", Sum.integersPerKey().getName()); assertEquals("Combine.perKey(SumDouble)", Sum.doublesPerKey().getName()); assertEquals("Combine.perKey(SumLong)", Sum.longsPerKey().getName()); }
.apply(Sum.integersGlobally()) .apply("FormatResults", ParDo.of(new DoFn<Integer, String>() { @ProcessElement
@Test @Category(ValidatesRunner.class) public void testCombineGloballyAsSingletonView() { final PCollectionView<Integer> view = pipeline .apply("CreateEmptySideInput", Create.empty(BigEndianIntegerCoder.of())) .apply(Sum.integersGlobally().asSingletonView()); PCollection<Integer> output = pipeline .apply("CreateVoidMainInput", Create.of((Void) null)) .apply( "OutputSideInput", ParDo.of( new DoFn<Void, Integer>() { @ProcessElement public void processElement(ProcessContext c) { c.output(c.sideInput(view)); } }) .withSideInputs(view)); PAssert.thatSingleton(output).isEqualTo(0); pipeline.run(); }
.apply(Sum.integersGlobally()) .apply("FormatResults", ParDo.of(new DoFn<Integer, String>() { @ProcessElement
TimestampedValue.of(3, new Instant(13)))) .apply("WindowSideInput", Window.into(FixedWindows.of(Duration.millis(10)))) .apply(Sum.integersGlobally().asSingletonView());
/** * Tests that when a processing time timers comes in after a window is expired it does not cause * a spurious output. */ @Test @Category({ValidatesRunner.class, UsesTestStream.class}) public void testCombiningAccumulatingProcessingTime() throws Exception { PCollection<Integer> triggeredSums = p.apply( TestStream.create(VarIntCoder.of()) .advanceWatermarkTo(new Instant(0)) .addElements( TimestampedValue.of(2, new Instant(2)), TimestampedValue.of(5, new Instant(5))) .advanceWatermarkTo(new Instant(100)) .advanceProcessingTime(Duration.millis(10)) .advanceWatermarkToInfinity()) .apply( Window.<Integer>into(FixedWindows.of(Duration.millis(100))) .withTimestampCombiner(TimestampCombiner.EARLIEST) .accumulatingFiredPanes() .withAllowedLateness(Duration.ZERO) .triggering( Repeatedly.forever( AfterProcessingTime.pastFirstElementInPane() .plusDelayOf(Duration.millis(10))))) .apply(Sum.integersGlobally().withoutDefaults()); PAssert.that(triggeredSums).containsInAnyOrder(7); p.run(); }
TimestampedValue.of(3, new Instant(13)))) .apply("WindowSideInput", Window.into(new GlobalWindows())) .apply(Sum.integersGlobally()) .apply(View.asSingleton());
TimestampedValue.of(3, new Instant(13)))) .apply("WindowSideInput", Window.into(FixedWindows.of(Duration.millis(10)))) .apply(Sum.integersGlobally().withoutDefaults()) .apply(View.asSingleton());
TimestampedValue.of(3, new Instant(100)))) .apply("WindowSideInput", Window.into(windowFn)) .apply("CombineSideInput", Sum.integersGlobally().asSingletonView());
globallyInput.apply("Sum", Sum.integersGlobally().withoutDefaults());