@Test public void testFinishBundleExceptionsWrappedAsUserCodeException() { ThrowingDoFn fn = new ThrowingDoFn(); DoFnRunner<String, String> runner = new SimpleDoFnRunner<>( null, fn, NullSideInputReader.empty(), null, null, Collections.emptyList(), mockStepContext, null, Collections.emptyMap(), WindowingStrategy.of(new GlobalWindows())); thrown.expect(UserCodeException.class); thrown.expectCause(is(fn.exceptionToThrow)); runner.finishBundle(); }
@Test public void testStartBundleExceptionsWrappedAsUserCodeException() { ThrowingDoFn fn = new ThrowingDoFn(); DoFnRunner<String, String> runner = new SimpleDoFnRunner<>( null, fn, NullSideInputReader.empty(), null, null, Collections.emptyList(), mockStepContext, null, Collections.emptyMap(), WindowingStrategy.of(new GlobalWindows())); thrown.expect(UserCodeException.class); thrown.expectCause(is(fn.exceptionToThrow)); runner.startBundle(); }
@Test public void testProcessElementExceptionsWrappedAsUserCodeException() { ThrowingDoFn fn = new ThrowingDoFn(); DoFnRunner<String, String> runner = new SimpleDoFnRunner<>( null, fn, NullSideInputReader.empty(), null, null, Collections.emptyList(), mockStepContext, null, Collections.emptyMap(), WindowingStrategy.of(new GlobalWindows())); thrown.expect(UserCodeException.class); thrown.expectCause(is(fn.exceptionToThrow)); runner.processElement(WindowedValue.valueInGlobalWindow("anyValue")); }
@Test public void testOnTimerExceptionsWrappedAsUserCodeException() { ThrowingDoFn fn = new ThrowingDoFn(); DoFnRunner<String, String> runner = new SimpleDoFnRunner<>( null, fn, NullSideInputReader.empty(), null, null, Collections.emptyList(), mockStepContext, null, Collections.emptyMap(), WindowingStrategy.of(new GlobalWindows())); thrown.expect(UserCodeException.class); thrown.expectCause(is(fn.exceptionToThrow)); runner.onTimer( ThrowingDoFn.TIMER_ID, GlobalWindow.INSTANCE, new Instant(0), TimeDomain.EVENT_TIME); }
public static <W extends BoundedWindow> ReduceFnTester<Integer, Iterable<Integer>, W> nonCombining( WindowFn<?, W> windowFn, TriggerStateMachine triggerStateMachine, AccumulationMode mode, Duration allowedDataLateness, ClosingBehavior closingBehavior) throws Exception { WindowingStrategy<?, W> strategy = WindowingStrategy.of(windowFn) .withTimestampCombiner(TimestampCombiner.EARLIEST) .withMode(mode) .withAllowedLateness(allowedDataLateness) .withClosingBehavior(closingBehavior); return nonCombining(strategy, triggerStateMachine); }
@Test public void multipleCallbacksShouldFireFires() throws Exception { CountDownLatch latch = new CountDownLatch(2); WindowFn<Object, IntervalWindow> windowFn = FixedWindows.of(Duration.standardMinutes(10)); IntervalWindow window = new IntervalWindow(new Instant(0L), new Instant(0L).plus(Duration.standardMinutes(10))); executor.callOnGuaranteedFiring( create, window, WindowingStrategy.of(windowFn), new CountDownLatchCallback(latch)); executor.callOnGuaranteedFiring( create, window, WindowingStrategy.of(windowFn), new CountDownLatchCallback(latch)); executor.fireForWatermark(create, new Instant(0L).plus(Duration.standardMinutes(10))); assertThat(latch.await(500, TimeUnit.MILLISECONDS), equalTo(true)); }
@Test public void multipleCallbacksShouldFireFires() throws Exception { CountDownLatch latch = new CountDownLatch(2); WindowFn<Object, IntervalWindow> windowFn = FixedWindows.of(Duration.standardMinutes(10)); IntervalWindow window = new IntervalWindow(new Instant(0L), new Instant(0L).plus(Duration.standardMinutes(10))); executor.callOnGuaranteedFiring( create, window, WindowingStrategy.of(windowFn), new CountDownLatchCallback(latch)); executor.callOnGuaranteedFiring( create, window, WindowingStrategy.of(windowFn), new CountDownLatchCallback(latch)); executor.fireForWatermark(create, new Instant(0L).plus(Duration.standardMinutes(10))); assertThat(latch.await(500, TimeUnit.MILLISECONDS), equalTo(true)); }
@Test public void unrelatedStepShouldNotFire() throws Exception { CountDownLatch latch = new CountDownLatch(1); WindowFn<Object, IntervalWindow> windowFn = FixedWindows.of(Duration.standardMinutes(10)); IntervalWindow window = new IntervalWindow(new Instant(0L), new Instant(0L).plus(Duration.standardMinutes(10))); executor.callOnGuaranteedFiring( sum, window, WindowingStrategy.of(windowFn), new CountDownLatchCallback(latch)); executor.fireForWatermark(create, new Instant(0L).plus(Duration.standardMinutes(20))); assertThat(latch.await(500, TimeUnit.MILLISECONDS), equalTo(false)); }
@Test public void noCallbacksShouldFire() throws Exception { CountDownLatch latch = new CountDownLatch(1); WindowFn<Object, IntervalWindow> windowFn = FixedWindows.of(Duration.standardMinutes(10)); IntervalWindow window = new IntervalWindow(new Instant(0L), new Instant(0L).plus(Duration.standardMinutes(10))); executor.callOnGuaranteedFiring( create, window, WindowingStrategy.of(windowFn), new CountDownLatchCallback(latch)); executor.fireForWatermark(create, new Instant(0L).plus(Duration.standardMinutes(5))); assertThat(latch.await(500, TimeUnit.MILLISECONDS), equalTo(false)); }
@Test public void noCallbacksShouldFire() throws Exception { CountDownLatch latch = new CountDownLatch(1); WindowFn<Object, IntervalWindow> windowFn = FixedWindows.of(Duration.standardMinutes(10)); IntervalWindow window = new IntervalWindow(new Instant(0L), new Instant(0L).plus(Duration.standardMinutes(10))); executor.callOnGuaranteedFiring( create, window, WindowingStrategy.of(windowFn), new CountDownLatchCallback(latch)); executor.fireForWatermark(create, new Instant(0L).plus(Duration.standardMinutes(5))); assertThat(latch.await(500, TimeUnit.MILLISECONDS), equalTo(false)); }
@Test public void unrelatedStepShouldNotFire() throws Exception { CountDownLatch latch = new CountDownLatch(1); WindowFn<Object, IntervalWindow> windowFn = FixedWindows.of(Duration.standardMinutes(10)); IntervalWindow window = new IntervalWindow(new Instant(0L), new Instant(0L).plus(Duration.standardMinutes(10))); executor.callOnGuaranteedFiring( sum, window, WindowingStrategy.of(windowFn), new CountDownLatchCallback(latch)); executor.fireForWatermark(create, new Instant(0L).plus(Duration.standardMinutes(20))); assertThat(latch.await(500, TimeUnit.MILLISECONDS), equalTo(false)); }
@Test public void testOnElementCombiningAccumulating() throws Exception { // Test basic execution of a trigger using a non-combining window set and accumulating mode. WindowingStrategy<?, IntervalWindow> strategy = WindowingStrategy.of((WindowFn<?, IntervalWindow>) FixedWindows.of(Duration.millis(10))) .withTimestampCombiner(TimestampCombiner.EARLIEST) .withMode(AccumulationMode.ACCUMULATING_FIRED_PANES) .withAllowedLateness(Duration.millis(100)); ReduceFnTester<Integer, Integer, IntervalWindow> tester = ReduceFnTester.combining( strategy, mockTriggerStateMachine, Sum.ofIntegers(), VarIntCoder.of()); injectElement(tester, 1); when(mockTriggerStateMachine.shouldFire(anyTriggerContext())).thenReturn(true); injectElement(tester, 2); when(mockTriggerStateMachine.shouldFire(anyTriggerContext())).thenReturn(true); triggerShouldFinish(mockTriggerStateMachine); injectElement(tester, 3); // This element shouldn't be seen, because the trigger has finished injectElement(tester, 4); assertThat( tester.extractOutput(), contains( isSingleWindowedValue(equalTo(3), 1, 0, 10), isSingleWindowedValue(equalTo(6), 3, 0, 10))); assertTrue(tester.isMarkedFinished(firstWindow)); tester.assertHasOnlyGlobalAndFinishedSetsFor(firstWindow); }
@Parameters(name = "{index}: {0}") public static Iterable<ToProtoAndBackSpec> data() { return ImmutableList.of( toProtoAndBackSpec(WindowingStrategy.globalDefault()), toProtoAndBackSpec( WindowingStrategy.of( FixedWindows.of(Duration.millis(11)).withOffset(Duration.millis(3)))), toProtoAndBackSpec( WindowingStrategy.of( SlidingWindows.of(Duration.millis(37)) .every(Duration.millis(3)) .withOffset(Duration.millis(2)))), toProtoAndBackSpec(WindowingStrategy.of(Sessions.withGapDuration(Duration.millis(389)))), toProtoAndBackSpec( WindowingStrategy.of(REPRESENTATIVE_WINDOW_FN) .withClosingBehavior(ClosingBehavior.FIRE_ALWAYS) .withMode(AccumulationMode.ACCUMULATING_FIRED_PANES) .withTrigger(REPRESENTATIVE_TRIGGER) .withAllowedLateness(Duration.millis(71)) .withTimestampCombiner(TimestampCombiner.EARLIEST)), toProtoAndBackSpec( WindowingStrategy.of(REPRESENTATIVE_WINDOW_FN) .withClosingBehavior(ClosingBehavior.FIRE_IF_NON_EMPTY) .withMode(AccumulationMode.DISCARDING_FIRED_PANES) .withTrigger(REPRESENTATIVE_TRIGGER) .withAllowedLateness(Duration.millis(93)) .withTimestampCombiner(TimestampCombiner.LATEST))); }
/** * Converts from {@link RunnerApi.WindowingStrategy} to the SDK's {@link WindowingStrategy} using * the provided components to dereferences identifiers found in the proto. */ public static WindowingStrategy<?, ?> fromProto( RunnerApi.WindowingStrategy proto, RehydratedComponents components) throws InvalidProtocolBufferException { SdkFunctionSpec windowFnSpec = proto.getWindowFn(); WindowFn<?, ?> windowFn = windowFnFromProto(windowFnSpec); TimestampCombiner timestampCombiner = timestampCombinerFromProto(proto.getOutputTime()); AccumulationMode accumulationMode = fromProto(proto.getAccumulationMode()); Trigger trigger = TriggerTranslation.fromProto(proto.getTrigger()); ClosingBehavior closingBehavior = fromProto(proto.getClosingBehavior()); Duration allowedLateness = Duration.millis(proto.getAllowedLateness()); OnTimeBehavior onTimeBehavior = fromProto(proto.getOnTimeBehavior()); return WindowingStrategy.of(windowFn) .withAllowedLateness(allowedLateness) .withMode(accumulationMode) .withTrigger(trigger) .withTimestampCombiner(timestampCombiner) .withClosingBehavior(closingBehavior) .withOnTimeBehavior(onTimeBehavior); }
@Test public void testWindowingStrategy() throws Exception { SdkComponents sdkComponents = SdkComponents.create(); sdkComponents.registerEnvironment(Environments.createDockerEnvironment("java")); WindowingStrategy windowingStrategy = WindowingStrategy.of(FixedWindows.of(Duration.millis(1))) .withAllowedLateness(Duration.standardSeconds(4)); String id = sdkComponents.registerWindowingStrategy(windowingStrategy); RehydratedComponents rehydratedComponents = RehydratedComponents.forComponents(sdkComponents.toComponents()); WindowingStrategy<?, ?> rehydratedStrategy = rehydratedComponents.getWindowingStrategy(id); assertThat(rehydratedStrategy, equalTo((WindowingStrategy) windowingStrategy.fixDefaults())); assertThat( rehydratedComponents.getWindowingStrategy(id), theInstance((WindowingStrategy) rehydratedStrategy)); }
@Test public void noEmptyPanesFinalAlways() throws Exception { ReduceFnTester<Integer, Iterable<Integer>, IntervalWindow> tester = ReduceFnTester.nonCombining( WindowingStrategy.of(FixedWindows.of(Duration.millis(10))) .withTrigger( Repeatedly.forever( AfterFirst.of( AfterPane.elementCountAtLeast(2), AfterWatermark.pastEndOfWindow()))) .withMode(AccumulationMode.ACCUMULATING_FIRED_PANES) .withAllowedLateness(Duration.millis(100)) .withTimestampCombiner(TimestampCombiner.EARLIEST) .withClosingBehavior(ClosingBehavior.FIRE_ALWAYS)); tester.advanceInputWatermark(new Instant(0)); tester.injectElements( TimestampedValue.of(1, new Instant(1)), TimestampedValue.of(2, new Instant(2))); tester.advanceInputWatermark(new Instant(20)); tester.advanceInputWatermark(new Instant(250)); List<WindowedValue<Iterable<Integer>>> output = tester.extractOutput(); assertThat( output, contains( // Trigger with 2 elements isSingleWindowedValue(containsInAnyOrder(1, 2), 1, 0, 10), // Trigger for the empty on time pane isSingleWindowedValue(containsInAnyOrder(1, 2), 9, 0, 10), // Trigger for the final pane isSingleWindowedValue(containsInAnyOrder(1, 2), 9, 0, 10))); }
/** * Tests that if end-of-window and GC timers come in together, that the pane is correctly marked * as final. */ @Test public void testCombiningAccumulatingEventTime() throws Exception { WindowingStrategy<?, IntervalWindow> strategy = WindowingStrategy.of((WindowFn<?, IntervalWindow>) FixedWindows.of(Duration.millis(100))) .withTimestampCombiner(TimestampCombiner.EARLIEST) .withMode(AccumulationMode.ACCUMULATING_FIRED_PANES) .withAllowedLateness(Duration.millis(1)) .withTrigger(Repeatedly.forever(AfterWatermark.pastEndOfWindow())); ReduceFnTester<Integer, Integer, IntervalWindow> tester = ReduceFnTester.combining(strategy, Sum.ofIntegers(), VarIntCoder.of()); injectElement(tester, 2); // processing timer @ 5000 + 10; EOW timer @ 100 injectElement(tester, 5); tester.advanceInputWatermark(new Instant(1000)); assertThat( tester.extractOutput(), contains( isSingleWindowedValue( equalTo(7), 2, 0, 100, PaneInfo.createPane(true, true, Timing.ON_TIME, 0, 0)))); }
@Test public void testWatermarkHoldForLateNewWindow() throws Exception { Duration allowedLateness = Duration.standardMinutes(1); Duration gapDuration = Duration.millis(10); ReduceFnTester<Integer, Iterable<Integer>, IntervalWindow> tester = ReduceFnTester.nonCombining( WindowingStrategy.of(Sessions.withGapDuration(gapDuration)) .withMode(AccumulationMode.DISCARDING_FIRED_PANES) .withTrigger( Repeatedly.forever( AfterWatermark.pastEndOfWindow() .withLateFirings(AfterPane.elementCountAtLeast(1)))) .withAllowedLateness(allowedLateness)); tester.setAutoAdvanceOutputWatermark(false); assertEquals(null, tester.getWatermarkHold()); assertEquals(null, tester.getOutputWatermark()); tester.advanceInputWatermark(new Instant(40)); injectElements(tester, 1); assertThat(tester.getWatermarkHold(), nullValue()); injectElements(tester, 10); assertThat(tester.getWatermarkHold(), nullValue()); }
/** Tests that a processing time timer does not cause window GC. */ @Test public void testProcessingTimeTimerDoesNotGc() throws Exception { WindowingStrategy<?, IntervalWindow> strategy = WindowingStrategy.of((WindowFn<?, IntervalWindow>) FixedWindows.of(Duration.millis(100))) .withTimestampCombiner(TimestampCombiner.EARLIEST) .withMode(AccumulationMode.ACCUMULATING_FIRED_PANES) .withAllowedLateness(Duration.ZERO) .withTrigger( Repeatedly.forever( AfterProcessingTime.pastFirstElementInPane().plusDelayOf(Duration.millis(10)))); ReduceFnTester<Integer, Integer, IntervalWindow> tester = ReduceFnTester.combining(strategy, Sum.ofIntegers(), VarIntCoder.of()); tester.advanceProcessingTime(new Instant(5000)); injectElement(tester, 2); // processing timer @ 5000 + 10; EOW timer @ 100 injectElement(tester, 5); tester.advanceProcessingTime(new Instant(10000)); tester.assertHasOnlyGlobalAndStateFor(new IntervalWindow(new Instant(0), new Instant(100))); assertThat( tester.extractOutput(), contains( isSingleWindowedValue( equalTo(7), 2, 0, 100, PaneInfo.createPane(true, false, Timing.EARLY, 0, 0)))); }
/** * Tests that when a processing time timer comes in after a window is expired it is just ignored. */ @Test public void testLateProcessingTimeTimer() throws Exception { WindowingStrategy<?, IntervalWindow> strategy = WindowingStrategy.of((WindowFn<?, IntervalWindow>) FixedWindows.of(Duration.millis(100))) .withTimestampCombiner(TimestampCombiner.EARLIEST) .withMode(AccumulationMode.ACCUMULATING_FIRED_PANES) .withAllowedLateness(Duration.ZERO) .withTrigger( Repeatedly.forever( AfterProcessingTime.pastFirstElementInPane().plusDelayOf(Duration.millis(10)))); ReduceFnTester<Integer, Integer, IntervalWindow> tester = ReduceFnTester.combining(strategy, Sum.ofIntegers(), VarIntCoder.of()); tester.advanceProcessingTime(new Instant(5000)); injectElement(tester, 2); // processing timer @ 5000 + 10; EOW timer @ 100 injectElement(tester, 5); // After this advancement, the window is expired and only the GC process // should be allowed to touch it tester.advanceInputWatermarkNoTimers(new Instant(100)); // This should not output tester.advanceProcessingTime(new Instant(6000)); assertThat(tester.extractOutput(), emptyIterable()); }