@Test public void testMergeWindowsIsCalled() throws Exception { MergingWindowAssigner<Integer, TimeWindow> mockAssigner = mockMergingAssigner(); Trigger<Integer, TimeWindow> mockTrigger = mockTrigger(); InternalWindowFunction<Iterable<Integer>, Void, Integer, TimeWindow> mockWindowFunction = mockWindowFunction(); KeyedOneInputStreamOperatorTestHarness<Integer, Integer, Void> testHarness = createWindowOperator(mockAssigner, mockTrigger, 0L, mockWindowFunction); testHarness.open(); when(mockAssigner.assignWindows(anyInt(), anyLong(), anyAssignerContext())) .thenReturn(Arrays.asList(new TimeWindow(2, 4), new TimeWindow(0, 2))); assertEquals(0, testHarness.getOutput().size()); assertEquals(0, testHarness.numKeyedStateEntries()); testHarness.processElement(new StreamRecord<>(0, 0L)); verify(mockAssigner).mergeWindows(eq(Collections.singletonList(new TimeWindow(2, 4))), anyMergeCallback()); verify(mockAssigner).mergeWindows(eq(Collections.singletonList(new TimeWindow(2, 4))), anyMergeCallback()); verify(mockAssigner, times(2)).mergeWindows(anyCollection(), anyMergeCallback()); }
@Test public void testOnElementPurgeDoesNotCleanupMergingSet() throws Exception { MergingWindowAssigner<Integer, TimeWindow> mockAssigner = mockMergingAssigner(); Trigger<Integer, TimeWindow> mockTrigger = mockTrigger(); InternalWindowFunction<Iterable<Integer>, Void, Integer, TimeWindow> mockWindowFunction = mockWindowFunction(); KeyedOneInputStreamOperatorTestHarness<Integer, Integer, Void> testHarness = createWindowOperator(mockAssigner, mockTrigger, 0L, mockWindowFunction); testHarness.open(); when(mockAssigner.assignWindows(anyInt(), anyLong(), anyAssignerContext())) .thenReturn(Arrays.asList(new TimeWindow(0, 2))); assertEquals(0, testHarness.getOutput().size()); assertEquals(0, testHarness.numKeyedStateEntries()); doAnswer(new Answer<TriggerResult>() { @Override public TriggerResult answer(InvocationOnMock invocation) throws Exception { return TriggerResult.PURGE; } }).when(mockTrigger).onElement(Matchers.<Integer>anyObject(), anyLong(), anyTimeWindow(), anyTriggerContext()); testHarness.processElement(new StreamRecord<>(0, 0L)); assertEquals(1, testHarness.numKeyedStateEntries()); // the merging window set assertEquals(1, testHarness.numEventTimeTimers()); // one cleanup timer assertEquals(0, testHarness.getOutput().size()); }
@Test public void testProcessingElementsWithinAllowedLateness() throws Exception { WindowAssigner<Integer, TimeWindow> mockAssigner = mockTimeWindowAssigner(); Trigger<Integer, TimeWindow> mockTrigger = mockTrigger(); InternalWindowFunction<Iterable<Integer>, Void, Integer, TimeWindow> mockWindowFunction = mockWindowFunction(); KeyedOneInputStreamOperatorTestHarness<Integer, Integer, Void> testHarness = createWindowOperator(mockAssigner, mockTrigger, 20L, mockWindowFunction); testHarness.open(); when(mockAssigner.assignWindows(anyInt(), anyLong(), anyAssignerContext())) .thenReturn(Arrays.asList(new TimeWindow(0, 2))); assertEquals(0, testHarness.getOutput().size()); assertEquals(0, testHarness.numKeyedStateEntries()); shouldFireOnElement(mockTrigger); // 20 is just at the limit, window.maxTime() is 1 and allowed lateness is 20 testHarness.processWatermark(new Watermark(20)); testHarness.processElement(new StreamRecord<>(0, 0L)); verify(mockWindowFunction, times(1)).process(eq(0), eq(new TimeWindow(0, 2)), anyInternalWindowContext(), intIterable(0), WindowOperatorContractTest.<Void>anyCollector()); // clear is only called at cleanup time/GC time verify(mockTrigger, never()).clear(anyTimeWindow(), anyTriggerContext()); // FIRE should not purge contents assertEquals(1, testHarness.numKeyedStateEntries()); // window contents plus trigger state assertEquals(1, testHarness.numEventTimeTimers()); // just the GC timer }
.thenReturn(Arrays.asList(new TimeWindow(2, 4), new TimeWindow(0, 2))); assertEquals(0, testHarness.getOutput().size()); assertEquals(0, testHarness.numKeyedStateEntries());
.thenReturn(Arrays.asList(new TimeWindow(2, 4), new TimeWindow(0, 2))); assertEquals(0, testHarness.getOutput().size()); assertEquals(0, testHarness.numKeyedStateEntries());
private void testOnTimePurgeDoesNotCleanupMergingSet(TimeDomainAdaptor timeAdaptor) throws Exception { MergingWindowAssigner<Integer, TimeWindow> mockAssigner = mockMergingAssigner(); timeAdaptor.setIsEventTime(mockAssigner); Trigger<Integer, TimeWindow> mockTrigger = mockTrigger(); InternalWindowFunction<Iterable<Integer>, Void, Integer, TimeWindow> mockWindowFunction = mockWindowFunction(); KeyedOneInputStreamOperatorTestHarness<Integer, Integer, Void> testHarness = createWindowOperator(mockAssigner, mockTrigger, 0L, mockWindowFunction); testHarness.open(); when(mockAssigner.assignWindows(anyInt(), anyLong(), anyAssignerContext())) .thenReturn(Arrays.asList(new TimeWindow(0, 4))); assertEquals(0, testHarness.getOutput().size()); assertEquals(0, testHarness.numKeyedStateEntries()); timeAdaptor.shouldRegisterTimerOnElement(mockTrigger, 1L); testHarness.processElement(new StreamRecord<>(0, 0L)); timeAdaptor.shouldPurgeOnTime(mockTrigger); assertEquals(2, testHarness.numKeyedStateEntries()); // the merging window set + window contents assertEquals(2, timeAdaptor.numTimers(testHarness)); // one cleanup timer + timer assertEquals(0, testHarness.getOutput().size()); timeAdaptor.advanceTime(testHarness, 1L); assertEquals(1, testHarness.numKeyedStateEntries()); // the merging window set assertEquals(1, timeAdaptor.numTimers(testHarness)); // one cleanup timer assertEquals(0, testHarness.extractOutputStreamRecords().size()); }
@Test public void testProcessingTimeGarbageCollectionTimerIsAlwaysWindowMaxTimestamp() throws Exception { WindowAssigner<Integer, TimeWindow> mockAssigner = mockTimeWindowAssigner(); when(mockAssigner.isEventTime()).thenReturn(false); Trigger<Integer, TimeWindow> mockTrigger = mockTrigger(); InternalWindowFunction<Iterable<Integer>, Void, Integer, TimeWindow> mockWindowFunction = mockWindowFunction(); KeyedOneInputStreamOperatorTestHarness<Integer, Integer, Void> testHarness = createWindowOperator(mockAssigner, mockTrigger, 20L, mockWindowFunction); testHarness.open(); when(mockAssigner.assignWindows(anyInt(), anyLong(), anyAssignerContext())) .thenReturn(Arrays.asList(new TimeWindow(0, Long.MAX_VALUE - 10))); assertEquals(0, testHarness.getOutput().size()); assertEquals(0, testHarness.numKeyedStateEntries()); testHarness.processElement(new StreamRecord<>(0, 0L)); // just the window contents assertEquals(1, testHarness.numKeyedStateEntries()); // no GC timer assertEquals(0, testHarness.numEventTimeTimers()); assertEquals(1, testHarness.numProcessingTimeTimers()); verify(mockTrigger, never()).clear(anyTimeWindow(), anyTriggerContext()); testHarness.setProcessingTime(Long.MAX_VALUE - 10); verify(mockTrigger, times(1)).clear(anyTimeWindow(), anyTriggerContext()); assertEquals(0, testHarness.numEventTimeTimers()); assertEquals(0, testHarness.numProcessingTimeTimers()); }
@Test public void testNoEventTimeGarbageCollectionTimerForLongMax() throws Exception { WindowAssigner<Integer, TimeWindow> mockAssigner = mockTimeWindowAssigner(); Trigger<Integer, TimeWindow> mockTrigger = mockTrigger(); InternalWindowFunction<Iterable<Integer>, Void, Integer, TimeWindow> mockWindowFunction = mockWindowFunction(); KeyedOneInputStreamOperatorTestHarness<Integer, Integer, Void> testHarness = createWindowOperator(mockAssigner, mockTrigger, 20L, mockWindowFunction); testHarness.open(); when(mockAssigner.assignWindows(anyInt(), anyLong(), anyAssignerContext())) .thenReturn(Arrays.asList(new TimeWindow(0, Long.MAX_VALUE - 10))); assertEquals(0, testHarness.getOutput().size()); assertEquals(0, testHarness.numKeyedStateEntries()); testHarness.processElement(new StreamRecord<>(0, 0L)); // just the window contents assertEquals(1, testHarness.numKeyedStateEntries()); // no GC timer assertEquals(0, testHarness.numEventTimeTimers()); assertEquals(0, testHarness.numProcessingTimeTimers()); }
.thenReturn(Arrays.asList(new TimeWindow(0, 20))); assertEquals(0, testHarness.getOutput().size()); assertEquals(0, testHarness.numKeyedStateEntries());
assertEquals(0, testHarness.getOutput().size()); assertEquals(0, testHarness.numKeyedStateEntries());
private void testMergingWindowSetClearedAtGarbageCollection(TimeDomainAdaptor timeAdaptor) throws Exception { WindowAssigner<Integer, TimeWindow> mockAssigner = mockMergingAssigner(); timeAdaptor.setIsEventTime(mockAssigner); Trigger<Integer, TimeWindow> mockTrigger = mockTrigger(); InternalWindowFunction<Iterable<Integer>, Void, Integer, TimeWindow> mockWindowFunction = mockWindowFunction(); KeyedOneInputStreamOperatorTestHarness<Integer, Integer, Void> testHarness = createWindowOperator(mockAssigner, mockTrigger, 20L, mockWindowFunction); testHarness.open(); when(mockAssigner.assignWindows(anyInt(), anyLong(), anyAssignerContext())) .thenReturn(Arrays.asList(new TimeWindow(0, 20))); assertEquals(0, testHarness.getOutput().size()); assertEquals(0, testHarness.numKeyedStateEntries()); testHarness.processElement(new StreamRecord<>(0, 0L)); assertEquals(2, testHarness.numKeyedStateEntries()); // window contents plus merging window set assertEquals(1, timeAdaptor.numTimers(testHarness)); // gc timers timeAdaptor.advanceTime(testHarness, 19 + 20); // 19 is maxTime of the window assertEquals(0, testHarness.numKeyedStateEntries()); assertEquals(0, timeAdaptor.numTimers(testHarness)); }
.thenReturn(Arrays.asList(new TimeWindow(0, 20))); assertEquals(0, testHarness.getOutput().size()); assertEquals(0, testHarness.numKeyedStateEntries());
@Test public void testLateWindowDropping() throws Exception { WindowAssigner<Integer, TimeWindow> mockAssigner = mockTimeWindowAssigner(); Trigger<Integer, TimeWindow> mockTrigger = mockTrigger(); InternalWindowFunction<Iterable<Integer>, Void, Integer, TimeWindow> mockWindowFunction = mockWindowFunction(); KeyedOneInputStreamOperatorTestHarness<Integer, Integer, Void> testHarness = createWindowOperator(mockAssigner, mockTrigger, 20L, mockWindowFunction); testHarness.open(); when(mockAssigner.assignWindows(anyInt(), anyLong(), anyAssignerContext())) .thenReturn(Arrays.asList(new TimeWindow(0, 2))); assertEquals(0, testHarness.getOutput().size()); assertEquals(0, testHarness.numKeyedStateEntries()); shouldFireOnElement(mockTrigger); // window.maxTime() == 1 plus 20L of allowed lateness testHarness.processWatermark(new Watermark(21)); testHarness.processElement(new StreamRecord<>(0, 0L)); // there should be nothing assertEquals(0, testHarness.numKeyedStateEntries()); assertEquals(0, testHarness.numEventTimeTimers()); assertEquals(0, testHarness.numProcessingTimeTimers()); // there should be two elements now assertEquals(0, testHarness.extractOutputStreamRecords().size()); }
/** * Ensure that we get some output from the given operator when pushing in an element and * setting watermark and processing time to {@code Long.MAX_VALUE}. */ private static <K, IN, OUT> void processElementAndEnsureOutput( OneInputStreamOperator<IN, OUT> operator, KeySelector<IN, K> keySelector, TypeInformation<K> keyType, IN element) throws Exception { KeyedOneInputStreamOperatorTestHarness<K, IN, OUT> testHarness = new KeyedOneInputStreamOperatorTestHarness<>( operator, keySelector, keyType); testHarness.open(); testHarness.setProcessingTime(0); testHarness.processWatermark(Long.MIN_VALUE); testHarness.processElement(new StreamRecord<>(element, 0)); // provoke any processing-time/event-time triggers testHarness.setProcessingTime(Long.MAX_VALUE); testHarness.processWatermark(Long.MAX_VALUE); // we at least get the two watermarks and should also see an output element assertTrue(testHarness.getOutput().size() >= 3); testHarness.close(); }
.thenReturn(Arrays.asList(new TimeWindow(0, 20))); assertEquals(0, testHarness.getOutput().size()); assertEquals(0, testHarness.numKeyedStateEntries());
.thenReturn(Arrays.asList(new TimeWindow(0, 20))); assertEquals(0, testHarness.getOutput().size()); assertEquals(0, testHarness.numKeyedStateEntries());
/** * Ensure that we get some output from the given operator when pushing in an element and * setting watermark and processing time to {@code Long.MAX_VALUE}. */ private static <K, IN, OUT> void processElementAndEnsureOutput( OneInputStreamOperator<IN, OUT> operator, KeySelector<IN, K> keySelector, TypeInformation<K> keyType, IN element) throws Exception { KeyedOneInputStreamOperatorTestHarness<K, IN, OUT> testHarness = new KeyedOneInputStreamOperatorTestHarness<>( operator, keySelector, keyType); if (operator instanceof OutputTypeConfigurable) { // use a dummy type since window functions just need the ExecutionConfig // this is also only needed for Fold, which we're getting rid off soon. ((OutputTypeConfigurable) operator).setOutputType(BasicTypeInfo.STRING_TYPE_INFO, new ExecutionConfig()); } testHarness.open(); testHarness.setProcessingTime(0); testHarness.processWatermark(Long.MIN_VALUE); testHarness.processElement(new StreamRecord<>(element, 0)); // provoke any processing-time/event-time triggers testHarness.setProcessingTime(Long.MAX_VALUE); testHarness.processWatermark(Long.MAX_VALUE); // we at least get the two watermarks and should also see an output element assertTrue(testHarness.getOutput().size() >= 3); testHarness.close(); }
private void testNoGarbageCollectionTimerForGlobalWindow(TimeDomainAdaptor timeAdaptor) throws Exception { WindowAssigner<Integer, GlobalWindow> mockAssigner = mockGlobalWindowAssigner(); timeAdaptor.setIsEventTime(mockAssigner); Trigger<Integer, GlobalWindow> mockTrigger = mockTrigger(); InternalWindowFunction<Iterable<Integer>, Void, Integer, GlobalWindow> mockWindowFunction = mockWindowFunction(); // this needs to be true for the test to succeed assertEquals(Long.MAX_VALUE, GlobalWindow.get().maxTimestamp()); KeyedOneInputStreamOperatorTestHarness<Integer, Integer, Void> testHarness = createWindowOperator(mockAssigner, mockTrigger, 0L, mockWindowFunction); testHarness.open(); assertEquals(0, testHarness.getOutput().size()); assertEquals(0, testHarness.numKeyedStateEntries()); testHarness.processElement(new StreamRecord<>(0, 0L)); // just the window contents assertEquals(1, testHarness.numKeyedStateEntries()); // verify we have no timers for either time domain assertEquals(0, testHarness.numEventTimeTimers()); assertEquals(0, testHarness.numProcessingTimeTimers()); }