/** * Verify that firing event-time timers see the state of the key that was active * when the timer was set. */ @Test public void testEventTimeTimersDontInterfere() throws Exception { TestOperator testOperator = new TestOperator(); KeyedOneInputStreamOperatorTestHarness<Integer, Tuple2<Integer, String>, String> testHarness = new KeyedOneInputStreamOperatorTestHarness<>(testOperator, new TestKeySelector(), BasicTypeInfo.INT_TYPE_INFO); testHarness.open(); testHarness.processWatermark(0L); testHarness.processElement(new Tuple2<>(1, "SET_EVENT_TIME_TIMER:20"), 0); testHarness.processElement(new Tuple2<>(0, "SET_STATE:HELLO"), 0); testHarness.processElement(new Tuple2<>(1, "SET_STATE:CIAO"), 0); testHarness.processElement(new Tuple2<>(0, "SET_EVENT_TIME_TIMER:10"), 0); testHarness.processWatermark(10L); assertThat( extractResult(testHarness), contains("ON_EVENT_TIME:HELLO")); testHarness.processWatermark(20L); assertThat( extractResult(testHarness), contains("ON_EVENT_TIME:CIAO")); }
/** * Ensure that we get some output from the given operator when pushing in an element and * setting watermark and processing time to {@code Long.MAX_VALUE}. */ private static <K, IN, OUT> void processElementAndEnsureOutput( OneInputStreamOperator<IN, OUT> operator, KeySelector<IN, K> keySelector, TypeInformation<K> keyType, IN element) throws Exception { KeyedOneInputStreamOperatorTestHarness<K, IN, OUT> testHarness = new KeyedOneInputStreamOperatorTestHarness<>( operator, keySelector, keyType); testHarness.open(); testHarness.setProcessingTime(0); testHarness.processWatermark(Long.MIN_VALUE); testHarness.processElement(new StreamRecord<>(element, 0)); // provoke any processing-time/event-time triggers testHarness.setProcessingTime(Long.MAX_VALUE); testHarness.processWatermark(Long.MAX_VALUE); // we at least get the two watermarks and should also see an output element assertTrue(testHarness.getOutput().size() >= 3); testHarness.close(); }
/** * Verify that timers for the different time domains don't clash. */ @Test public void testProcessingTimeAndEventTimeDontInterfere() throws Exception { TestOperator testOperator = new TestOperator(); KeyedOneInputStreamOperatorTestHarness<Integer, Tuple2<Integer, String>, String> testHarness = new KeyedOneInputStreamOperatorTestHarness<>(testOperator, new TestKeySelector(), BasicTypeInfo.INT_TYPE_INFO); testHarness.open(); testHarness.setProcessingTime(0L); testHarness.processWatermark(0L); testHarness.processElement(new Tuple2<>(0, "SET_PROC_TIME_TIMER:10"), 0); testHarness.processElement(new Tuple2<>(0, "SET_EVENT_TIME_TIMER:20"), 0); testHarness.processElement(new Tuple2<>(0, "SET_STATE:HELLO"), 0); testHarness.processWatermark(20L); assertThat( extractResult(testHarness), contains("ON_EVENT_TIME:HELLO")); testHarness.setProcessingTime(10L); assertThat( extractResult(testHarness), contains("ON_PROC_TIME:HELLO")); }
@Test public void testProcessingElementsWithinAllowedLateness() throws Exception { WindowAssigner<Integer, TimeWindow> mockAssigner = mockTimeWindowAssigner(); Trigger<Integer, TimeWindow> mockTrigger = mockTrigger(); InternalWindowFunction<Iterable<Integer>, Void, Integer, TimeWindow> mockWindowFunction = mockWindowFunction(); KeyedOneInputStreamOperatorTestHarness<Integer, Integer, Void> testHarness = createWindowOperator(mockAssigner, mockTrigger, 20L, mockWindowFunction); testHarness.open(); when(mockAssigner.assignWindows(anyInt(), anyLong(), anyAssignerContext())) .thenReturn(Arrays.asList(new TimeWindow(0, 2))); assertEquals(0, testHarness.getOutput().size()); assertEquals(0, testHarness.numKeyedStateEntries()); shouldFireOnElement(mockTrigger); // 20 is just at the limit, window.maxTime() is 1 and allowed lateness is 20 testHarness.processWatermark(new Watermark(20)); testHarness.processElement(new StreamRecord<>(0, 0L)); verify(mockWindowFunction, times(1)).process(eq(0), eq(new TimeWindow(0, 2)), anyInternalWindowContext(), intIterable(0), WindowOperatorContractTest.<Void>anyCollector()); // clear is only called at cleanup time/GC time verify(mockTrigger, never()).clear(anyTimeWindow(), anyTriggerContext()); // FIRE should not purge contents assertEquals(1, testHarness.numKeyedStateEntries()); // window contents plus trigger state assertEquals(1, testHarness.numEventTimeTimers()); // just the GC timer }
/** * Ensure that we get some output from the given operator when pushing in an element and * setting watermark and processing time to {@code Long.MAX_VALUE}. */ private static <K, IN, OUT> void processElementAndEnsureOutput( OneInputStreamOperator<IN, OUT> operator, KeySelector<IN, K> keySelector, TypeInformation<K> keyType, IN element) throws Exception { KeyedOneInputStreamOperatorTestHarness<K, IN, OUT> testHarness = new KeyedOneInputStreamOperatorTestHarness<>( operator, keySelector, keyType); if (operator instanceof OutputTypeConfigurable) { // use a dummy type since window functions just need the ExecutionConfig // this is also only needed for Fold, which we're getting rid off soon. ((OutputTypeConfigurable) operator).setOutputType(BasicTypeInfo.STRING_TYPE_INFO, new ExecutionConfig()); } testHarness.open(); testHarness.setProcessingTime(0); testHarness.processWatermark(Long.MIN_VALUE); testHarness.processElement(new StreamRecord<>(element, 0)); // provoke any processing-time/event-time triggers testHarness.setProcessingTime(Long.MAX_VALUE); testHarness.processWatermark(Long.MAX_VALUE); // we at least get the two watermarks and should also see an output element assertTrue(testHarness.getOutput().size() >= 3); testHarness.close(); }
testHarness.processWatermark(0L); testHarness.setProcessingTime(0L); testHarness1.open(); testHarness1.processWatermark(10L); testHarness1.processWatermark(20L); testHarness2.open(); testHarness2.processWatermark(10L); testHarness2.processWatermark(20L);
@Test public void testLateWindowDropping() throws Exception { WindowAssigner<Integer, TimeWindow> mockAssigner = mockTimeWindowAssigner(); Trigger<Integer, TimeWindow> mockTrigger = mockTrigger(); InternalWindowFunction<Iterable<Integer>, Void, Integer, TimeWindow> mockWindowFunction = mockWindowFunction(); KeyedOneInputStreamOperatorTestHarness<Integer, Integer, Void> testHarness = createWindowOperator(mockAssigner, mockTrigger, 20L, mockWindowFunction); testHarness.open(); when(mockAssigner.assignWindows(anyInt(), anyLong(), anyAssignerContext())) .thenReturn(Arrays.asList(new TimeWindow(0, 2))); assertEquals(0, testHarness.getOutput().size()); assertEquals(0, testHarness.numKeyedStateEntries()); shouldFireOnElement(mockTrigger); // window.maxTime() == 1 plus 20L of allowed lateness testHarness.processWatermark(new Watermark(21)); testHarness.processElement(new StreamRecord<>(0, 0L)); // there should be nothing assertEquals(0, testHarness.numKeyedStateEntries()); assertEquals(0, testHarness.numEventTimeTimers()); assertEquals(0, testHarness.numProcessingTimeTimers()); // there should be two elements now assertEquals(0, testHarness.extractOutputStreamRecords().size()); }
testHarness.processWatermark(new Watermark(20L));
testHarness1.open(); testHarness1.processWatermark(0L); testHarness1.setProcessingTime(0L); testHarness2.open(); testHarness2.processWatermark(0L); testHarness2.setProcessingTime(0L); testHarness3.open(); testHarness3.processWatermark(30L); assertThat(extractResult(testHarness3), contains("ON_EVENT_TIME:HELLO")); assertTrue(extractResult(testHarness3).isEmpty()); testHarness3.processWatermark(40L); assertThat(extractResult(testHarness3), contains("ON_EVENT_TIME:CIAO")); assertTrue(extractResult(testHarness3).isEmpty());
harness.processWatermark(new Watermark(watermarkTimestamp1)); harness.processWatermark(new Watermark(watermarkTimestamp2));
testHarness.processWatermark(0L); testHarness.processElement( Item.builder().key(1L).timestamp(1L).value(100L).window(window).build().toStreamRecord()); testHarness.processWatermark(200L);
testHarness.processWatermark(0); testHarness.processWatermark( window1 .maxTimestamp()
testHarness.processWatermark(0); testHarness.processWatermark( window1 .maxTimestamp()
testHarness.processWatermark(0); testHarness.processWatermark( window1.maxTimestamp() .plus(windowingStrategy.getAllowedLateness())