@Test public void testPaneInfoSkipToFinish() throws Exception { ReduceFnTester<Integer, Iterable<Integer>, IntervalWindow> tester = ReduceFnTester.nonCombining( FixedWindows.of(Duration.millis(10)), mockTriggerStateMachine, AccumulationMode.DISCARDING_FIRED_PANES, Duration.millis(100), ClosingBehavior.FIRE_IF_NON_EMPTY); tester.advanceInputWatermark(new Instant(0)); when(mockTriggerStateMachine.shouldFire(anyTriggerContext())).thenReturn(true); triggerShouldFinish(mockTriggerStateMachine); injectElement(tester, 1); assertThat( tester.extractOutput(), contains(WindowMatchers.valueWithPaneInfo(PaneInfo.createPane(true, true, Timing.EARLY)))); }
@Test public void testPaneInfoSkipToNonSpeculativeAndFinish() throws Exception { ReduceFnTester<Integer, Iterable<Integer>, IntervalWindow> tester = ReduceFnTester.nonCombining( FixedWindows.of(Duration.millis(10)), mockTriggerStateMachine, AccumulationMode.DISCARDING_FIRED_PANES, Duration.millis(100), ClosingBehavior.FIRE_IF_NON_EMPTY); tester.advanceInputWatermark(new Instant(15)); when(mockTriggerStateMachine.shouldFire(anyTriggerContext())).thenReturn(true); triggerShouldFinish(mockTriggerStateMachine); injectElement(tester, 1); assertThat( tester.extractOutput(), contains(WindowMatchers.valueWithPaneInfo(PaneInfo.createPane(true, true, Timing.LATE)))); }
@Test public void testOnElementBufferingAccumulating() throws Exception { // Test basic execution of a trigger using a non-combining window set and accumulating mode. ReduceFnTester<Integer, Iterable<Integer>, IntervalWindow> tester = ReduceFnTester.nonCombining( FixedWindows.of(Duration.millis(10)), mockTriggerStateMachine, AccumulationMode.ACCUMULATING_FIRED_PANES, Duration.millis(100), ClosingBehavior.FIRE_IF_NON_EMPTY); injectElement(tester, 1); // Fires {1, 2} when(mockTriggerStateMachine.shouldFire(anyTriggerContext())).thenReturn(true); injectElement(tester, 2); // Fires {1, 2, 3} because we are in accumulating mode when(mockTriggerStateMachine.shouldFire(anyTriggerContext())).thenReturn(true); triggerShouldFinish(mockTriggerStateMachine); injectElement(tester, 3); // This element shouldn't be seen, because the trigger has finished injectElement(tester, 4); assertThat( tester.extractOutput(), contains( isSingleWindowedValue(containsInAnyOrder(1, 2), 1, 0, 10), isSingleWindowedValue(containsInAnyOrder(1, 2, 3), 3, 0, 10))); assertTrue(tester.isMarkedFinished(firstWindow)); tester.assertHasOnlyGlobalAndFinishedSetsFor(firstWindow); }
List<WindowedValue<Iterable<Integer>>> output = tester.extractOutput(); assertThat(output.size(), equalTo(1)); assertThat(
/** * We should fire a non-empty ON_TIME pane in the GlobalWindow when the watermark moves to * end-of-time. */ @Test public void fireNonEmptyOnDrainInGlobalWindow() throws Exception { ReduceFnTester<Integer, Iterable<Integer>, GlobalWindow> tester = ReduceFnTester.nonCombining( WindowingStrategy.of(new GlobalWindows()) .withTrigger(Repeatedly.forever(AfterPane.elementCountAtLeast(3))) .withMode(AccumulationMode.DISCARDING_FIRED_PANES)); tester.advanceInputWatermark(new Instant(0)); final int n = 20; for (int i = 0; i < n; i++) { tester.injectElements(TimestampedValue.of(i, new Instant(i))); } List<WindowedValue<Iterable<Integer>>> output = tester.extractOutput(); assertEquals(n / 3, output.size()); for (int i = 0; i < output.size(); i++) { assertEquals(Timing.EARLY, output.get(i).getPane().getTiming()); assertEquals(i, output.get(i).getPane().getIndex()); assertEquals(3, Iterables.size(output.get(i).getValue())); } tester.advanceInputWatermark(BoundedWindow.TIMESTAMP_MAX_VALUE); output = tester.extractOutput(); assertEquals(1, output.size()); assertEquals(Timing.ON_TIME, output.get(0).getPane().getTiming()); assertEquals(n / 3, output.get(0).getPane().getIndex()); assertEquals(n - ((n / 3) * 3), Iterables.size(output.get(0).getValue())); }
@Test public void testOnElementCombiningDiscarding() throws Exception { // Test basic execution of a trigger using a non-combining window set and discarding mode. WindowingStrategy<?, IntervalWindow> strategy = WindowingStrategy.of((WindowFn<?, IntervalWindow>) FixedWindows.of(Duration.millis(10))) .withTimestampCombiner(TimestampCombiner.EARLIEST) .withMode(AccumulationMode.DISCARDING_FIRED_PANES) .withAllowedLateness(Duration.millis(100)); ReduceFnTester<Integer, Integer, IntervalWindow> tester = ReduceFnTester.combining( strategy, mockTriggerStateMachine, Sum.ofIntegers(), VarIntCoder.of()); injectElement(tester, 2); when(mockTriggerStateMachine.shouldFire(anyTriggerContext())).thenReturn(true); injectElement(tester, 3); when(mockTriggerStateMachine.shouldFire(anyTriggerContext())).thenReturn(true); triggerShouldFinish(mockTriggerStateMachine); injectElement(tester, 4); // This element shouldn't be seen, because the trigger has finished injectElement(tester, 6); assertThat( tester.extractOutput(), contains( isSingleWindowedValue(equalTo(5), 2, 0, 10), isSingleWindowedValue(equalTo(4), 4, 0, 10))); assertTrue(tester.isMarkedFinished(firstWindow)); tester.assertHasOnlyGlobalAndFinishedSetsFor(firstWindow); }
@Test public void testOnElementCombiningAccumulating() throws Exception { // Test basic execution of a trigger using a non-combining window set and accumulating mode. WindowingStrategy<?, IntervalWindow> strategy = WindowingStrategy.of((WindowFn<?, IntervalWindow>) FixedWindows.of(Duration.millis(10))) .withTimestampCombiner(TimestampCombiner.EARLIEST) .withMode(AccumulationMode.ACCUMULATING_FIRED_PANES) .withAllowedLateness(Duration.millis(100)); ReduceFnTester<Integer, Integer, IntervalWindow> tester = ReduceFnTester.combining( strategy, mockTriggerStateMachine, Sum.ofIntegers(), VarIntCoder.of()); injectElement(tester, 1); when(mockTriggerStateMachine.shouldFire(anyTriggerContext())).thenReturn(true); injectElement(tester, 2); when(mockTriggerStateMachine.shouldFire(anyTriggerContext())).thenReturn(true); triggerShouldFinish(mockTriggerStateMachine); injectElement(tester, 3); // This element shouldn't be seen, because the trigger has finished injectElement(tester, 4); assertThat( tester.extractOutput(), contains( isSingleWindowedValue(equalTo(3), 1, 0, 10), isSingleWindowedValue(equalTo(6), 3, 0, 10))); assertTrue(tester.isMarkedFinished(firstWindow)); tester.assertHasOnlyGlobalAndFinishedSetsFor(firstWindow); }
/** * When the watermark passes the end-of-window and window expiration time in a single update, this * tests that it does not crash. */ @Test public void testFixedWindowsEowAndGcTogether() throws Exception { ReduceFnTester<Integer, Iterable<Integer>, IntervalWindow> tester = ReduceFnTester.nonCombining( FixedWindows.of(Duration.millis(10)), DefaultTriggerStateMachine.of(), AccumulationMode.ACCUMULATING_FIRED_PANES, Duration.millis(50), ClosingBehavior.FIRE_ALWAYS); tester.setAutoAdvanceOutputWatermark(true); tester.advanceInputWatermark(new Instant(0)); injectElement(tester, 1); tester.advanceInputWatermark(new Instant(100)); assertThat( tester.extractOutput(), contains( isSingleWindowedValue( contains(1), 1, 0, 10, PaneInfo.createPane(true, true, Timing.ON_TIME)))); }
/** * When the watermark passes the end-of-window and window expiration time in a single update, this * tests that it does not crash. */ @Test public void testFixedWindowsEowAndGcTogetherFireIfNonEmpty() throws Exception { ReduceFnTester<Integer, Iterable<Integer>, IntervalWindow> tester = ReduceFnTester.nonCombining( FixedWindows.of(Duration.millis(10)), DefaultTriggerStateMachine.of(), AccumulationMode.ACCUMULATING_FIRED_PANES, Duration.millis(50), ClosingBehavior.FIRE_IF_NON_EMPTY); tester.setAutoAdvanceOutputWatermark(true); tester.advanceInputWatermark(new Instant(0)); injectElement(tester, 1); tester.advanceInputWatermark(new Instant(100)); List<WindowedValue<Iterable<Integer>>> output = tester.extractOutput(); assertThat( output, contains( isSingleWindowedValue( contains(1), 1, 0, 10, PaneInfo.createPane(true, true, Timing.ON_TIME)))); }
/** * When the watermark passes the end-of-window and window expiration time in a single update, this * tests that it does not crash. */ @Test public void testSessionEowAndGcTogether() throws Exception { ReduceFnTester<Integer, Iterable<Integer>, IntervalWindow> tester = ReduceFnTester.nonCombining( Sessions.withGapDuration(Duration.millis(10)), DefaultTriggerStateMachine.of(), AccumulationMode.ACCUMULATING_FIRED_PANES, Duration.millis(50), ClosingBehavior.FIRE_ALWAYS); tester.setAutoAdvanceOutputWatermark(true); tester.advanceInputWatermark(new Instant(0)); injectElement(tester, 1); tester.advanceInputWatermark(new Instant(100)); assertThat( tester.extractOutput(), contains( isSingleWindowedValue( contains(1), 1, 1, 11, PaneInfo.createPane(true, true, Timing.ON_TIME)))); }
@Test public void testPaneInfoFinalAndOnTime() throws Exception { ReduceFnTester<Integer, Iterable<Integer>, IntervalWindow> tester = ReduceFnTester.nonCombining( WindowingStrategy.of(FixedWindows.of(Duration.millis(10))) .withTrigger( Repeatedly.forever(AfterPane.elementCountAtLeast(2)) .orFinally(AfterWatermark.pastEndOfWindow())) .withMode(AccumulationMode.DISCARDING_FIRED_PANES) .withAllowedLateness(Duration.millis(100)) .withClosingBehavior(ClosingBehavior.FIRE_ALWAYS)); tester.advanceInputWatermark(new Instant(0)); // Should trigger due to element count tester.injectElements( TimestampedValue.of(1, new Instant(1)), TimestampedValue.of(2, new Instant(2))); assertThat( tester.extractOutput(), contains( WindowMatchers.valueWithPaneInfo( PaneInfo.createPane(true, false, Timing.EARLY, 0, -1)))); tester.advanceInputWatermark(new Instant(150)); assertThat( tester.extractOutput(), contains( WindowMatchers.valueWithPaneInfo( PaneInfo.createPane(false, true, Timing.ON_TIME, 1, 0)))); }
/** * Tests that if end-of-window and GC timers come in together, that the pane is correctly marked * as final. */ @Test public void testCombiningAccumulatingEventTime() throws Exception { WindowingStrategy<?, IntervalWindow> strategy = WindowingStrategy.of((WindowFn<?, IntervalWindow>) FixedWindows.of(Duration.millis(100))) .withTimestampCombiner(TimestampCombiner.EARLIEST) .withMode(AccumulationMode.ACCUMULATING_FIRED_PANES) .withAllowedLateness(Duration.millis(1)) .withTrigger(Repeatedly.forever(AfterWatermark.pastEndOfWindow())); ReduceFnTester<Integer, Integer, IntervalWindow> tester = ReduceFnTester.combining(strategy, Sum.ofIntegers(), VarIntCoder.of()); injectElement(tester, 2); // processing timer @ 5000 + 10; EOW timer @ 100 injectElement(tester, 5); tester.advanceInputWatermark(new Instant(1000)); assertThat( tester.extractOutput(), contains( isSingleWindowedValue( equalTo(7), 2, 0, 100, PaneInfo.createPane(true, true, Timing.ON_TIME, 0, 0)))); }
@Test public void noEmptyPanesFinalAlways() throws Exception { ReduceFnTester<Integer, Iterable<Integer>, IntervalWindow> tester = ReduceFnTester.nonCombining( WindowingStrategy.of(FixedWindows.of(Duration.millis(10))) .withTrigger( Repeatedly.forever( AfterFirst.of( AfterPane.elementCountAtLeast(2), AfterWatermark.pastEndOfWindow()))) .withMode(AccumulationMode.ACCUMULATING_FIRED_PANES) .withAllowedLateness(Duration.millis(100)) .withTimestampCombiner(TimestampCombiner.EARLIEST) .withClosingBehavior(ClosingBehavior.FIRE_ALWAYS)); tester.advanceInputWatermark(new Instant(0)); tester.injectElements( TimestampedValue.of(1, new Instant(1)), TimestampedValue.of(2, new Instant(2))); tester.advanceInputWatermark(new Instant(20)); tester.advanceInputWatermark(new Instant(250)); List<WindowedValue<Iterable<Integer>>> output = tester.extractOutput(); assertThat( output, contains( // Trigger with 2 elements isSingleWindowedValue(containsInAnyOrder(1, 2), 1, 0, 10), // Trigger for the empty on time pane isSingleWindowedValue(containsInAnyOrder(1, 2), 9, 0, 10), // Trigger for the final pane isSingleWindowedValue(containsInAnyOrder(1, 2), 9, 0, 10))); }
@Test public void noEmptyPanesFinalIfNonEmpty() throws Exception { ReduceFnTester<Integer, Iterable<Integer>, IntervalWindow> tester = ReduceFnTester.nonCombining( WindowingStrategy.of(FixedWindows.of(Duration.millis(10))) .withTrigger( Repeatedly.forever( AfterFirst.of( AfterPane.elementCountAtLeast(2), AfterWatermark.pastEndOfWindow()))) .withMode(AccumulationMode.ACCUMULATING_FIRED_PANES) .withAllowedLateness(Duration.millis(100)) .withTimestampCombiner(TimestampCombiner.EARLIEST) .withClosingBehavior(ClosingBehavior.FIRE_IF_NON_EMPTY)); tester.advanceInputWatermark(new Instant(0)); tester.injectElements( TimestampedValue.of(1, new Instant(1)), TimestampedValue.of(2, new Instant(2))); tester.advanceInputWatermark(new Instant(20)); tester.advanceInputWatermark(new Instant(250)); List<WindowedValue<Iterable<Integer>>> output = tester.extractOutput(); assertThat( output, contains( // Trigger with 2 elements isSingleWindowedValue(containsInAnyOrder(1, 2), 1, 0, 10), // Trigger for the empty on time pane isSingleWindowedValue(containsInAnyOrder(1, 2), 9, 0, 10))); }
/** * Tests that when a processing time timers comes in after a window is expired and GC'd it does * not cause a spurious output. */ @Test public void testCombiningAccumulatingProcessingTimeSeparateBundles() throws Exception { WindowingStrategy<?, IntervalWindow> strategy = WindowingStrategy.of((WindowFn<?, IntervalWindow>) FixedWindows.of(Duration.millis(100))) .withTimestampCombiner(TimestampCombiner.EARLIEST) .withMode(AccumulationMode.ACCUMULATING_FIRED_PANES) .withAllowedLateness(Duration.ZERO) .withTrigger( Repeatedly.forever( AfterProcessingTime.pastFirstElementInPane().plusDelayOf(Duration.millis(10)))); ReduceFnTester<Integer, Integer, IntervalWindow> tester = ReduceFnTester.combining(strategy, Sum.ofIntegers(), VarIntCoder.of()); tester.advanceProcessingTime(new Instant(5000)); injectElement(tester, 2); // processing timer @ 5000 + 10; EOW timer @ 100 injectElement(tester, 5); tester.advanceInputWatermark(new Instant(100)); tester.advanceProcessingTime(new Instant(5011)); assertThat( tester.extractOutput(), contains( isSingleWindowedValue( equalTo(7), 2, 0, 100, PaneInfo.createPane(true, true, Timing.ON_TIME, 0, 0)))); }
List<WindowedValue<Integer>> output = tester.extractOutput(); assertEquals(2, output.size());
/** Tests that a processing time timer does not cause window GC. */ @Test public void testProcessingTimeTimerDoesNotGc() throws Exception { WindowingStrategy<?, IntervalWindow> strategy = WindowingStrategy.of((WindowFn<?, IntervalWindow>) FixedWindows.of(Duration.millis(100))) .withTimestampCombiner(TimestampCombiner.EARLIEST) .withMode(AccumulationMode.ACCUMULATING_FIRED_PANES) .withAllowedLateness(Duration.ZERO) .withTrigger( Repeatedly.forever( AfterProcessingTime.pastFirstElementInPane().plusDelayOf(Duration.millis(10)))); ReduceFnTester<Integer, Integer, IntervalWindow> tester = ReduceFnTester.combining(strategy, Sum.ofIntegers(), VarIntCoder.of()); tester.advanceProcessingTime(new Instant(5000)); injectElement(tester, 2); // processing timer @ 5000 + 10; EOW timer @ 100 injectElement(tester, 5); tester.advanceProcessingTime(new Instant(10000)); tester.assertHasOnlyGlobalAndStateFor(new IntervalWindow(new Instant(0), new Instant(100))); assertThat( tester.extractOutput(), contains( isSingleWindowedValue( equalTo(7), 2, 0, 100, PaneInfo.createPane(true, false, Timing.EARLY, 0, 0)))); }
tester.extractOutput(), contains( isSingleWindowedValue(
List<WindowedValue<Iterable<Integer>>> output = tester.extractOutput(); assertEquals(2, output.size());
/** * Tests that when a processing time timer comes in after a window is expired it is just ignored. */ @Test public void testLateProcessingTimeTimer() throws Exception { WindowingStrategy<?, IntervalWindow> strategy = WindowingStrategy.of((WindowFn<?, IntervalWindow>) FixedWindows.of(Duration.millis(100))) .withTimestampCombiner(TimestampCombiner.EARLIEST) .withMode(AccumulationMode.ACCUMULATING_FIRED_PANES) .withAllowedLateness(Duration.ZERO) .withTrigger( Repeatedly.forever( AfterProcessingTime.pastFirstElementInPane().plusDelayOf(Duration.millis(10)))); ReduceFnTester<Integer, Integer, IntervalWindow> tester = ReduceFnTester.combining(strategy, Sum.ofIntegers(), VarIntCoder.of()); tester.advanceProcessingTime(new Instant(5000)); injectElement(tester, 2); // processing timer @ 5000 + 10; EOW timer @ 100 injectElement(tester, 5); // After this advancement, the window is expired and only the GC process // should be allowed to touch it tester.advanceInputWatermarkNoTimers(new Instant(100)); // This should not output tester.advanceProcessingTime(new Instant(6000)); assertThat(tester.extractOutput(), emptyIterable()); }