@Override protected Trigger getContinuationTrigger(List<Trigger> continuationTriggers) { // Use OrFinallyTrigger instead of AfterFirst because the continuation of ACTUAL // may not be a OnceTrigger. return Repeatedly.forever( new OrFinallyTrigger( continuationTriggers.get(ACTUAL), (Trigger.OnceTrigger) continuationTriggers.get(UNTIL))); }
@Override protected Trigger getContinuationTrigger(List<Trigger> continuationTriggers) { return Repeatedly.forever(new AfterFirst(continuationTriggers)); }
@Test public void testToString() { Trigger trigger = Repeatedly.forever( new StubTrigger() { @Override public String toString() { return "innerTrigger"; } }); assertEquals("Repeatedly.forever(innerTrigger)", trigger.toString()); } }
@Test public void testContinuation() throws Exception { Trigger trigger = AfterProcessingTime.pastFirstElementInPane(); Trigger repeatedly = Repeatedly.forever(trigger); assertEquals( Repeatedly.forever(trigger.getContinuationTrigger()), repeatedly.getContinuationTrigger()); assertEquals( Repeatedly.forever(trigger.getContinuationTrigger().getContinuationTrigger()), repeatedly.getContinuationTrigger().getContinuationTrigger()); }
/** Tests that the watermark that guarantees firing is that of the subtrigger. */ @Test public void testFireDeadline() throws Exception { setUp(FixedWindows.of(Duration.millis(10))); IntervalWindow window = new IntervalWindow(new Instant(0), new Instant(10)); Instant arbitraryInstant = new Instant(34957849); when(mockTrigger.getWatermarkThatGuaranteesFiring(Mockito.<IntervalWindow>any())) .thenReturn(arbitraryInstant); assertThat( Repeatedly.forever(mockTrigger).getWatermarkThatGuaranteesFiring(window), equalTo(arbitraryInstant)); }
@Override public PCollection<KV<String, Integer>> expand(PCollection<GameActionInfo> input) { return input .apply( "LeaderboardUserGlobalWindow", Window.<GameActionInfo>into(new GlobalWindows()) // Get periodic results every ten minutes. .triggering( Repeatedly.forever( AfterProcessingTime.pastFirstElementInPane().plusDelayOf(TEN_MINUTES))) .accumulatingFiredPanes() .withAllowedLateness(allowedLateness)) // Extract and sum username/score pairs from the event data. .apply("ExtractUserScore", new ExtractAndSumScore("user")); } }
@Override public PCollection<String> expand(PCollection<KV<String, Integer>> input) { return input .apply(Window.<KV<String, Integer>>into(FixedWindows.of(TWO_MINUTES)) .triggering(Repeatedly.forever(AfterPane.elementCountAtLeast(1))) .withAllowedLateness(Duration.standardDays(1000)) .accumulatingFiredPanes()) .apply(Sum.integersPerKey()) .apply(ParDo.of(new FormatAsStrings())); }
@Override public PCollection<String> expand(PCollection<KV<String, Integer>> input) { return input .apply(Window.<KV<String, Integer>>into(FixedWindows.of(TWO_MINUTES)) .triggering(Repeatedly.forever(AfterProcessingTime.pastFirstElementInPane().plusDelayOf(TWO_MINUTES))) .withAllowedLateness(Duration.standardDays(1000)) .accumulatingFiredPanes()) .apply(Sum.integersPerKey()) .apply(ParDo.of(new FormatAsStrings())); }
@Test public void testTriggerToString() throws Exception { assertEquals("AfterWatermark.pastEndOfWindow()", AfterWatermark.pastEndOfWindow().toString()); assertEquals( "Repeatedly.forever(AfterWatermark.pastEndOfWindow())", Repeatedly.forever(AfterWatermark.pastEndOfWindow()).toString()); }
@Override public PCollection<String> expand(PCollection<KV<String, Integer>> input) { return input .apply(Window.<KV<String, Integer>>into(FixedWindows.of(TWO_MINUTES)) .triggering(Repeatedly.forever(AfterProcessingTime .pastFirstElementInPane() .alignedTo(TWO_MINUTES, Utils.parseTime("12:05:00")))) .withAllowedLateness(Duration.standardDays(1000)) .accumulatingFiredPanes()) .apply(Sum.integersPerKey()) .apply(ParDo.of(new FormatAsStrings())); }
@Test public void testContinuation() throws Exception { OnceTrigger triggerA = AfterProcessingTime.pastFirstElementInPane(); OnceTrigger triggerB = AfterWatermark.pastEndOfWindow(); Trigger aOrFinallyB = triggerA.orFinally(triggerB); Trigger bOrFinallyA = triggerB.orFinally(triggerA); assertEquals( Repeatedly.forever( triggerA.getContinuationTrigger().orFinally(triggerB.getContinuationTrigger())), aOrFinallyB.getContinuationTrigger()); assertEquals( Repeatedly.forever( triggerB.getContinuationTrigger().orFinally(triggerA.getContinuationTrigger())), bOrFinallyA.getContinuationTrigger()); }
@Test public void testContinuation() throws Exception { OnceTrigger trigger1 = AfterProcessingTime.pastFirstElementInPane(); OnceTrigger trigger2 = AfterWatermark.pastEndOfWindow(); Trigger afterEach = AfterEach.inOrder(trigger1, trigger2); assertEquals( Repeatedly.forever( AfterFirst.of(trigger1.getContinuationTrigger(), trigger2.getContinuationTrigger())), afterEach.getContinuationTrigger()); }
@Test @Category(ValidatesRunner.class) public void testHotKeyCombiningWithAccumulationMode() { PCollection<Integer> input = pipeline.apply(Create.of(1, 2, 3, 4, 5)); PCollection<Integer> output = input .apply( Window.<Integer>into(new GlobalWindows()) .triggering(Repeatedly.forever(AfterPane.elementCountAtLeast(1))) .accumulatingFiredPanes() .withAllowedLateness(new Duration(0), ClosingBehavior.FIRE_ALWAYS)) .apply(Sum.integersGlobally().withoutDefaults().withFanout(2)) .apply(ParDo.of(new GetLast())); PAssert.that(output) .satisfies( input1 -> { assertThat(input1, hasItem(15)); return null; }); pipeline.run(); }
@Test public void testWatermarkHoldForLateNewWindow() throws Exception { Duration allowedLateness = Duration.standardMinutes(1); Duration gapDuration = Duration.millis(10); ReduceFnTester<Integer, Iterable<Integer>, IntervalWindow> tester = ReduceFnTester.nonCombining( WindowingStrategy.of(Sessions.withGapDuration(gapDuration)) .withMode(AccumulationMode.DISCARDING_FIRED_PANES) .withTrigger( Repeatedly.forever( AfterWatermark.pastEndOfWindow() .withLateFirings(AfterPane.elementCountAtLeast(1)))) .withAllowedLateness(allowedLateness)); tester.setAutoAdvanceOutputWatermark(false); assertEquals(null, tester.getWatermarkHold()); assertEquals(null, tester.getOutputWatermark()); tester.advanceInputWatermark(new Instant(40)); injectElements(tester, 1); assertThat(tester.getWatermarkHold(), nullValue()); injectElements(tester, 10); assertThat(tester.getWatermarkHold(), nullValue()); }
@Test public void testMissingLateness() { FixedWindows fixed10 = FixedWindows.of(Duration.standardMinutes(10)); Repeatedly trigger = Repeatedly.forever(AfterPane.elementCountAtLeast(5)); thrown.expect(IllegalArgumentException.class); thrown.expectMessage("requires that the allowed lateness"); pipeline .apply(Create.of("hello", "world").withCoder(StringUtf8Coder.of())) .apply("Mode", Window.<String>configure().accumulatingFiredPanes()) .apply("Window", Window.into(fixed10)) .apply("Trigger", Window.<String>configure().triggering(trigger)); }
@Test public void testMissingMode() { FixedWindows fixed10 = FixedWindows.of(Duration.standardMinutes(10)); Repeatedly trigger = Repeatedly.forever(AfterPane.elementCountAtLeast(5)); PCollection<String> input = pipeline .apply(Create.of("hello", "world").withCoder(StringUtf8Coder.of())) .apply("Window", Window.into(fixed10)); thrown.expect(IllegalArgumentException.class); thrown.expectMessage("requires that the accumulation mode"); input.apply( "Triggering", Window.<String>configure() .withAllowedLateness(Duration.standardDays(1)) .triggering(trigger)); }
@Test public void testWindowIntoTriggersAndAccumulating() { FixedWindows fixed10 = FixedWindows.of(Duration.standardMinutes(10)); Repeatedly trigger = Repeatedly.forever(AfterPane.elementCountAtLeast(5)); WindowingStrategy<?, ?> strategy = pipeline .apply(Create.of("hello", "world").withCoder(StringUtf8Coder.of())) .apply( Window.<String>into(fixed10) .triggering(trigger) .accumulatingFiredPanes() .withAllowedLateness(Duration.ZERO)) .getWindowingStrategy(); assertEquals(fixed10, strategy.getWindowFn()); assertEquals(trigger, strategy.getTrigger()); assertEquals(AccumulationMode.ACCUMULATING_FIRED_PANES, strategy.getMode()); }
@Test public void testWindowIntoPropagatesLateness() { FixedWindows fixed10 = FixedWindows.of(Duration.standardMinutes(10)); FixedWindows fixed25 = FixedWindows.of(Duration.standardMinutes(25)); WindowingStrategy<?, ?> strategy = pipeline .apply(Create.of("hello", "world").withCoder(StringUtf8Coder.of())) .apply( "WindowInto10", Window.<String>into(fixed10) .withAllowedLateness(Duration.standardDays(1)) .triggering(Repeatedly.forever(AfterPane.elementCountAtLeast(5))) .accumulatingFiredPanes()) .apply("WindowInto25", Window.into(fixed25)) .getWindowingStrategy(); assertEquals(Duration.standardDays(1), strategy.getAllowedLateness()); assertEquals(fixed25, strategy.getWindowFn()); }
/** * Tests that when a processing time timer comes in after a window is expired it is just ignored. */ @Test public void testLateProcessingTimeTimer() throws Exception { WindowingStrategy<?, IntervalWindow> strategy = WindowingStrategy.of((WindowFn<?, IntervalWindow>) FixedWindows.of(Duration.millis(100))) .withTimestampCombiner(TimestampCombiner.EARLIEST) .withMode(AccumulationMode.ACCUMULATING_FIRED_PANES) .withAllowedLateness(Duration.ZERO) .withTrigger( Repeatedly.forever( AfterProcessingTime.pastFirstElementInPane().plusDelayOf(Duration.millis(10)))); ReduceFnTester<Integer, Integer, IntervalWindow> tester = ReduceFnTester.combining(strategy, Sum.ofIntegers(), VarIntCoder.of()); tester.advanceProcessingTime(new Instant(5000)); injectElement(tester, 2); // processing timer @ 5000 + 10; EOW timer @ 100 injectElement(tester, 5); // After this advancement, the window is expired and only the GC process // should be allowed to touch it tester.advanceInputWatermarkNoTimers(new Instant(100)); // This should not output tester.advanceProcessingTime(new Instant(6000)); assertThat(tester.extractOutput(), emptyIterable()); }
@Test public void testWindowPropagatesEachPart() { FixedWindows fixed10 = FixedWindows.of(Duration.standardMinutes(10)); Repeatedly trigger = Repeatedly.forever(AfterPane.elementCountAtLeast(5)); WindowingStrategy<?, ?> strategy = pipeline .apply(Create.of("hello", "world").withCoder(StringUtf8Coder.of())) .apply("Mode", Window.<String>configure().accumulatingFiredPanes()) .apply( "Lateness", Window.<String>configure().withAllowedLateness(Duration.standardDays(1))) .apply("Trigger", Window.<String>configure().triggering(trigger)) .apply("Window", Window.into(fixed10)) .getWindowingStrategy(); assertEquals(fixed10, strategy.getWindowFn()); assertEquals(trigger, strategy.getTrigger()); assertEquals(AccumulationMode.ACCUMULATING_FIRED_PANES, strategy.getMode()); assertEquals(Duration.standardDays(1), strategy.getAllowedLateness()); }