/** * Inject all the timestamped values (after passing through the window function) as if they * arrived in a single chunk of a bundle (or work-unit). */ @SafeVarargs public final void injectElements(TimestampedValue<InputT>... values) throws Exception { injectElements(Arrays.asList(values)); }
private void injectElements( ReduceFnTester<Integer, ?, IntervalWindow> tester, Iterable<Integer> values) throws Exception { doNothing().when(mockTriggerStateMachine).onElement(anyElementContext()); List<TimestampedValue<Integer>> timestampedValues = new ArrayList<>(); for (int value : values) { timestampedValues.add(TimestampedValue.of(value, new Instant(value))); } tester.injectElements(timestampedValues); }
private void injectElement(ReduceFnTester<Integer, ?, IntervalWindow> tester, int element) throws Exception { doNothing().when(mockTriggerStateMachine).onElement(anyElementContext()); tester.injectElements(TimestampedValue.of(element, new Instant(element))); }
@Test public void testMergeBeforeFinalizing() throws Exception { // Verify that we merge windows before producing output so users don't see undesired // unmerged windows. ReduceFnTester<Integer, Iterable<Integer>, IntervalWindow> tester = ReduceFnTester.nonCombining( Sessions.withGapDuration(Duration.millis(10)), mockTriggerStateMachine, AccumulationMode.DISCARDING_FIRED_PANES, Duration.ZERO, ClosingBehavior.FIRE_IF_NON_EMPTY); // All on time data, verify watermark hold. // These two windows should pre-merge immediately to [1, 20) tester.injectElements( TimestampedValue.of(1, new Instant(1)), // in [1, 11) TimestampedValue.of(10, new Instant(10))); // in [10, 20) // And this should fire the end-of-window timer tester.advanceInputWatermark(new Instant(100)); List<WindowedValue<Iterable<Integer>>> output = tester.extractOutput(); assertThat(output.size(), equalTo(1)); assertThat( output.get(0), isSingleWindowedValue( containsInAnyOrder(1, 10), 1, // timestamp 1, // window start 20)); // window end assertThat( output.get(0).getPane(), equalTo(PaneInfo.createPane(true, true, Timing.ON_TIME, 0, 0))); }
tester.injectElements(TimestampedValue.of(1, new Instant(1))); // in [1, 11), gc at 21. tester.injectElements(TimestampedValue.of(1, new Instant(1))); // in [1, 11), gc at 21.
tester.injectElements( TimestampedValue.of(1, new Instant(1)), // in [1, 11) TimestampedValue.of(10, new Instant(10))); // in [10, 20)
tester.injectElements( TimestampedValue.of(1, new Instant(1)), // in [1, 11), gc at 21. TimestampedValue.of(8, new Instant(8))); // in [8, 18), gc at 28. tester.injectElements( TimestampedValue.of(1, new Instant(1)), // in [1, 11), gc at 21. TimestampedValue.of(2, new Instant(2)), // in [2, 12), gc at 22.
tester.injectElements(TimestampedValue.of(value1, new Instant(1))); tester.injectElements(TimestampedValue.of(value2, new Instant(3)));
/** * We should fire a non-empty ON_TIME pane in the GlobalWindow when the watermark moves to * end-of-time. */ @Test public void fireNonEmptyOnDrainInGlobalWindow() throws Exception { ReduceFnTester<Integer, Iterable<Integer>, GlobalWindow> tester = ReduceFnTester.nonCombining( WindowingStrategy.of(new GlobalWindows()) .withTrigger(Repeatedly.forever(AfterPane.elementCountAtLeast(3))) .withMode(AccumulationMode.DISCARDING_FIRED_PANES)); tester.advanceInputWatermark(new Instant(0)); final int n = 20; for (int i = 0; i < n; i++) { tester.injectElements(TimestampedValue.of(i, new Instant(i))); } List<WindowedValue<Iterable<Integer>>> output = tester.extractOutput(); assertEquals(n / 3, output.size()); for (int i = 0; i < output.size(); i++) { assertEquals(Timing.EARLY, output.get(i).getPane().getTiming()); assertEquals(i, output.get(i).getPane().getIndex()); assertEquals(3, Iterables.size(output.get(i).getValue())); } tester.advanceInputWatermark(BoundedWindow.TIMESTAMP_MAX_VALUE); output = tester.extractOutput(); assertEquals(1, output.size()); assertEquals(Timing.ON_TIME, output.get(0).getPane().getTiming()); assertEquals(n / 3, output.get(0).getPane().getIndex()); assertEquals(n - ((n / 3) * 3), Iterables.size(output.get(0).getValue())); }
for (int i = 0; i < n; i++) { tester.advanceProcessingTime(new Instant(i)); tester.injectElements(TimestampedValue.of(i, new Instant(i)));
/** Ensure a closed trigger has its state recorded in the merge result window. */ @Test public void testMergingWithCloseTrigger() throws Exception { Duration allowedLateness = Duration.millis(50); ReduceFnTester<Integer, Iterable<Integer>, IntervalWindow> tester = ReduceFnTester.nonCombining( Sessions.withGapDuration(Duration.millis(10)), mockTriggerStateMachine, AccumulationMode.DISCARDING_FIRED_PANES, allowedLateness, ClosingBehavior.FIRE_IF_NON_EMPTY); // Create a new merged session window. IntervalWindow mergedWindow = new IntervalWindow(new Instant(1), new Instant(12)); tester.injectElements( TimestampedValue.of(1, new Instant(1)), TimestampedValue.of(2, new Instant(2))); // Force the trigger to be closed for the merged window. when(mockTriggerStateMachine.shouldFire(anyTriggerContext())).thenReturn(true); triggerShouldFinish(mockTriggerStateMachine); // Fire and end-of-window timer as though the trigger set it tester.advanceInputWatermark(new Instant(13)); tester.fireTimer(mergedWindow, mergedWindow.maxTimestamp(), TimeDomain.EVENT_TIME); // Trigger is now closed. assertTrue(tester.isMarkedFinished(mergedWindow)); when(mockTriggerStateMachine.shouldFire(anyTriggerContext())).thenReturn(false); // Revisit the same session window. tester.injectElements( TimestampedValue.of(1, new Instant(1)), TimestampedValue.of(2, new Instant(2))); // Trigger is still closed. assertTrue(tester.isMarkedFinished(mergedWindow)); }
tester.injectElements( TimestampedValue.of(1, new Instant(1)), TimestampedValue.of(2, new Instant(2)));
@Test public void noEmptyPanesFinalAlways() throws Exception { ReduceFnTester<Integer, Iterable<Integer>, IntervalWindow> tester = ReduceFnTester.nonCombining( WindowingStrategy.of(FixedWindows.of(Duration.millis(10))) .withTrigger( Repeatedly.forever( AfterFirst.of( AfterPane.elementCountAtLeast(2), AfterWatermark.pastEndOfWindow()))) .withMode(AccumulationMode.ACCUMULATING_FIRED_PANES) .withAllowedLateness(Duration.millis(100)) .withTimestampCombiner(TimestampCombiner.EARLIEST) .withClosingBehavior(ClosingBehavior.FIRE_ALWAYS)); tester.advanceInputWatermark(new Instant(0)); tester.injectElements( TimestampedValue.of(1, new Instant(1)), TimestampedValue.of(2, new Instant(2))); tester.advanceInputWatermark(new Instant(20)); tester.advanceInputWatermark(new Instant(250)); List<WindowedValue<Iterable<Integer>>> output = tester.extractOutput(); assertThat( output, contains( // Trigger with 2 elements isSingleWindowedValue(containsInAnyOrder(1, 2), 1, 0, 10), // Trigger for the empty on time pane isSingleWindowedValue(containsInAnyOrder(1, 2), 9, 0, 10), // Trigger for the final pane isSingleWindowedValue(containsInAnyOrder(1, 2), 9, 0, 10))); }
@Test public void noEmptyPanesFinalIfNonEmpty() throws Exception { ReduceFnTester<Integer, Iterable<Integer>, IntervalWindow> tester = ReduceFnTester.nonCombining( WindowingStrategy.of(FixedWindows.of(Duration.millis(10))) .withTrigger( Repeatedly.forever( AfterFirst.of( AfterPane.elementCountAtLeast(2), AfterWatermark.pastEndOfWindow()))) .withMode(AccumulationMode.ACCUMULATING_FIRED_PANES) .withAllowedLateness(Duration.millis(100)) .withTimestampCombiner(TimestampCombiner.EARLIEST) .withClosingBehavior(ClosingBehavior.FIRE_IF_NON_EMPTY)); tester.advanceInputWatermark(new Instant(0)); tester.injectElements( TimestampedValue.of(1, new Instant(1)), TimestampedValue.of(2, new Instant(2))); tester.advanceInputWatermark(new Instant(20)); tester.advanceInputWatermark(new Instant(250)); List<WindowedValue<Iterable<Integer>>> output = tester.extractOutput(); assertThat( output, contains( // Trigger with 2 elements isSingleWindowedValue(containsInAnyOrder(1, 2), 1, 0, 10), // Trigger for the empty on time pane isSingleWindowedValue(containsInAnyOrder(1, 2), 9, 0, 10))); }
tester.injectElements( TimestampedValue.of(1, new Instant(1)), TimestampedValue.of(2, new Instant(2)));
tester.advanceProcessingTime(new Instant(0)); tester.injectElements(TimestampedValue.of(1, new Instant(1))); tester.injectElements(TimestampedValue.of(3, new Instant(3)));
tester.advanceProcessingTime(new Instant(0)); tester.injectElements(TimestampedValue.of(1, new Instant(1)));
tester.injectElements(TimestampedValue.of(1, new Instant(1))); tester.injectElements(TimestampedValue.of(2, new Instant(2)));
tester.injectElements( TimestampedValue.of(1, new Instant(1)), TimestampedValue.of(1, new Instant(3)),
@Test public void testPaneInfoFinalAndOnTime() throws Exception { ReduceFnTester<Integer, Iterable<Integer>, IntervalWindow> tester = ReduceFnTester.nonCombining( WindowingStrategy.of(FixedWindows.of(Duration.millis(10))) .withTrigger( Repeatedly.forever(AfterPane.elementCountAtLeast(2)) .orFinally(AfterWatermark.pastEndOfWindow())) .withMode(AccumulationMode.DISCARDING_FIRED_PANES) .withAllowedLateness(Duration.millis(100)) .withClosingBehavior(ClosingBehavior.FIRE_ALWAYS)); tester.advanceInputWatermark(new Instant(0)); // Should trigger due to element count tester.injectElements( TimestampedValue.of(1, new Instant(1)), TimestampedValue.of(2, new Instant(2))); assertThat( tester.extractOutput(), contains( WindowMatchers.valueWithPaneInfo( PaneInfo.createPane(true, false, Timing.EARLY, 0, -1)))); tester.advanceInputWatermark(new Instant(150)); assertThat( tester.extractOutput(), contains( WindowMatchers.valueWithPaneInfo( PaneInfo.createPane(false, true, Timing.ON_TIME, 1, 0)))); }