@Test public void testMergeWindowsIsCalled() throws Exception { MergingWindowAssigner<Integer, TimeWindow> mockAssigner = mockMergingAssigner(); Trigger<Integer, TimeWindow> mockTrigger = mockTrigger(); InternalWindowFunction<Iterable<Integer>, Void, Integer, TimeWindow> mockWindowFunction = mockWindowFunction(); KeyedOneInputStreamOperatorTestHarness<Integer, Integer, Void> testHarness = createWindowOperator(mockAssigner, mockTrigger, 0L, mockWindowFunction); testHarness.open(); when(mockAssigner.assignWindows(anyInt(), anyLong(), anyAssignerContext())) .thenReturn(Arrays.asList(new TimeWindow(2, 4), new TimeWindow(0, 2))); assertEquals(0, testHarness.getOutput().size()); assertEquals(0, testHarness.numKeyedStateEntries()); testHarness.processElement(new StreamRecord<>(0, 0L)); verify(mockAssigner).mergeWindows(eq(Collections.singletonList(new TimeWindow(2, 4))), anyMergeCallback()); verify(mockAssigner).mergeWindows(eq(Collections.singletonList(new TimeWindow(2, 4))), anyMergeCallback()); verify(mockAssigner, times(2)).mergeWindows(anyCollection(), anyMergeCallback()); }
private void testMergingWindowSetClearedAtGarbageCollection(TimeDomainAdaptor timeAdaptor) throws Exception { WindowAssigner<Integer, TimeWindow> mockAssigner = mockMergingAssigner(); timeAdaptor.setIsEventTime(mockAssigner); Trigger<Integer, TimeWindow> mockTrigger = mockTrigger(); InternalWindowFunction<Iterable<Integer>, Void, Integer, TimeWindow> mockWindowFunction = mockWindowFunction(); KeyedOneInputStreamOperatorTestHarness<Integer, Integer, Void> testHarness = createWindowOperator(mockAssigner, mockTrigger, 20L, mockWindowFunction); testHarness.open(); when(mockAssigner.assignWindows(anyInt(), anyLong(), anyAssignerContext())) .thenReturn(Arrays.asList(new TimeWindow(0, 20))); assertEquals(0, testHarness.getOutput().size()); assertEquals(0, testHarness.numKeyedStateEntries()); testHarness.processElement(new StreamRecord<>(0, 0L)); assertEquals(2, testHarness.numKeyedStateEntries()); // window contents plus merging window set assertEquals(1, timeAdaptor.numTimers(testHarness)); // gc timers timeAdaptor.advanceTime(testHarness, 19 + 20); // 19 is maxTime of the window assertEquals(0, testHarness.numKeyedStateEntries()); assertEquals(0, timeAdaptor.numTimers(testHarness)); }
@Test public void testProcessingElementsWithinAllowedLateness() throws Exception { WindowAssigner<Integer, TimeWindow> mockAssigner = mockTimeWindowAssigner(); Trigger<Integer, TimeWindow> mockTrigger = mockTrigger(); InternalWindowFunction<Iterable<Integer>, Void, Integer, TimeWindow> mockWindowFunction = mockWindowFunction(); KeyedOneInputStreamOperatorTestHarness<Integer, Integer, Void> testHarness = createWindowOperator(mockAssigner, mockTrigger, 20L, mockWindowFunction); testHarness.open(); when(mockAssigner.assignWindows(anyInt(), anyLong(), anyAssignerContext())) .thenReturn(Arrays.asList(new TimeWindow(0, 2))); assertEquals(0, testHarness.getOutput().size()); assertEquals(0, testHarness.numKeyedStateEntries()); shouldFireOnElement(mockTrigger); // 20 is just at the limit, window.maxTime() is 1 and allowed lateness is 20 testHarness.processWatermark(new Watermark(20)); testHarness.processElement(new StreamRecord<>(0, 0L)); verify(mockWindowFunction, times(1)).process(eq(0), eq(new TimeWindow(0, 2)), anyInternalWindowContext(), intIterable(0), WindowOperatorContractTest.<Void>anyCollector()); // clear is only called at cleanup time/GC time verify(mockTrigger, never()).clear(anyTimeWindow(), anyTriggerContext()); // FIRE should not purge contents assertEquals(1, testHarness.numKeyedStateEntries()); // window contents plus trigger state assertEquals(1, testHarness.numEventTimeTimers()); // just the GC timer }
@Test public void testNoEventTimeGarbageCollectionTimerForLongMax() throws Exception { WindowAssigner<Integer, TimeWindow> mockAssigner = mockTimeWindowAssigner(); Trigger<Integer, TimeWindow> mockTrigger = mockTrigger(); InternalWindowFunction<Iterable<Integer>, Void, Integer, TimeWindow> mockWindowFunction = mockWindowFunction(); KeyedOneInputStreamOperatorTestHarness<Integer, Integer, Void> testHarness = createWindowOperator(mockAssigner, mockTrigger, 20L, mockWindowFunction); testHarness.open(); when(mockAssigner.assignWindows(anyInt(), anyLong(), anyAssignerContext())) .thenReturn(Arrays.asList(new TimeWindow(0, Long.MAX_VALUE - 10))); assertEquals(0, testHarness.getOutput().size()); assertEquals(0, testHarness.numKeyedStateEntries()); testHarness.processElement(new StreamRecord<>(0, 0L)); // just the window contents assertEquals(1, testHarness.numKeyedStateEntries()); // no GC timer assertEquals(0, testHarness.numEventTimeTimers()); assertEquals(0, testHarness.numProcessingTimeTimers()); }
@Test public void testOnElementPurgeDoesNotCleanupMergingSet() throws Exception { MergingWindowAssigner<Integer, TimeWindow> mockAssigner = mockMergingAssigner(); Trigger<Integer, TimeWindow> mockTrigger = mockTrigger(); InternalWindowFunction<Iterable<Integer>, Void, Integer, TimeWindow> mockWindowFunction = mockWindowFunction(); KeyedOneInputStreamOperatorTestHarness<Integer, Integer, Void> testHarness = createWindowOperator(mockAssigner, mockTrigger, 0L, mockWindowFunction); testHarness.open(); when(mockAssigner.assignWindows(anyInt(), anyLong(), anyAssignerContext())) .thenReturn(Arrays.asList(new TimeWindow(0, 2))); assertEquals(0, testHarness.getOutput().size()); assertEquals(0, testHarness.numKeyedStateEntries()); doAnswer(new Answer<TriggerResult>() { @Override public TriggerResult answer(InvocationOnMock invocation) throws Exception { return TriggerResult.PURGE; } }).when(mockTrigger).onElement(Matchers.<Integer>anyObject(), anyLong(), anyTimeWindow(), anyTriggerContext()); testHarness.processElement(new StreamRecord<>(0, 0L)); assertEquals(1, testHarness.numKeyedStateEntries()); // the merging window set assertEquals(1, testHarness.numEventTimeTimers()); // one cleanup timer assertEquals(0, testHarness.getOutput().size()); }
private void testOnTimePurgeDoesNotCleanupMergingSet(TimeDomainAdaptor timeAdaptor) throws Exception { MergingWindowAssigner<Integer, TimeWindow> mockAssigner = mockMergingAssigner(); timeAdaptor.setIsEventTime(mockAssigner); Trigger<Integer, TimeWindow> mockTrigger = mockTrigger(); InternalWindowFunction<Iterable<Integer>, Void, Integer, TimeWindow> mockWindowFunction = mockWindowFunction(); KeyedOneInputStreamOperatorTestHarness<Integer, Integer, Void> testHarness = createWindowOperator(mockAssigner, mockTrigger, 0L, mockWindowFunction); testHarness.open(); when(mockAssigner.assignWindows(anyInt(), anyLong(), anyAssignerContext())) .thenReturn(Arrays.asList(new TimeWindow(0, 4))); assertEquals(0, testHarness.getOutput().size()); assertEquals(0, testHarness.numKeyedStateEntries()); timeAdaptor.shouldRegisterTimerOnElement(mockTrigger, 1L); testHarness.processElement(new StreamRecord<>(0, 0L)); timeAdaptor.shouldPurgeOnTime(mockTrigger); assertEquals(2, testHarness.numKeyedStateEntries()); // the merging window set + window contents assertEquals(2, timeAdaptor.numTimers(testHarness)); // one cleanup timer + timer assertEquals(0, testHarness.getOutput().size()); timeAdaptor.advanceTime(testHarness, 1L); assertEquals(1, testHarness.numKeyedStateEntries()); // the merging window set assertEquals(1, timeAdaptor.numTimers(testHarness)); // one cleanup timer assertEquals(0, testHarness.extractOutputStreamRecords().size()); }
assertEquals(0, testHarness.numKeyedStateEntries()); assertEquals(4, testHarness.numKeyedStateEntries()); // window contents plus trigger state assertEquals(4, testHarness.numEventTimeTimers()); // window timers/gc timers
assertEquals(2, testHarness.numKeyedStateEntries()); assertEquals(2, testHarness.numEventTimeTimers());
assertEquals(0, testHarness.numKeyedStateEntries()); assertEquals(2, testHarness.numKeyedStateEntries()); // trigger state will stick around until GC time
@Test public void testProcessingTimeGarbageCollectionTimerIsAlwaysWindowMaxTimestamp() throws Exception { WindowAssigner<Integer, TimeWindow> mockAssigner = mockTimeWindowAssigner(); when(mockAssigner.isEventTime()).thenReturn(false); Trigger<Integer, TimeWindow> mockTrigger = mockTrigger(); InternalWindowFunction<Iterable<Integer>, Void, Integer, TimeWindow> mockWindowFunction = mockWindowFunction(); KeyedOneInputStreamOperatorTestHarness<Integer, Integer, Void> testHarness = createWindowOperator(mockAssigner, mockTrigger, 20L, mockWindowFunction); testHarness.open(); when(mockAssigner.assignWindows(anyInt(), anyLong(), anyAssignerContext())) .thenReturn(Arrays.asList(new TimeWindow(0, Long.MAX_VALUE - 10))); assertEquals(0, testHarness.getOutput().size()); assertEquals(0, testHarness.numKeyedStateEntries()); testHarness.processElement(new StreamRecord<>(0, 0L)); // just the window contents assertEquals(1, testHarness.numKeyedStateEntries()); // no GC timer assertEquals(0, testHarness.numEventTimeTimers()); assertEquals(1, testHarness.numProcessingTimeTimers()); verify(mockTrigger, never()).clear(anyTimeWindow(), anyTriggerContext()); testHarness.setProcessingTime(Long.MAX_VALUE - 10); verify(mockTrigger, times(1)).clear(anyTimeWindow(), anyTriggerContext()); assertEquals(0, testHarness.numEventTimeTimers()); assertEquals(0, testHarness.numProcessingTimeTimers()); }
assertEquals(0, testHarness.numKeyedStateEntries()); assertEquals(4, testHarness.numKeyedStateEntries()); // window-contents and trigger state for two windows assertEquals(4, timeAdaptor.numTimers(testHarness)); // timers/gc timers for two windows assertEquals(4, testHarness.numKeyedStateEntries()); assertEquals(2, timeAdaptor.numTimers(testHarness)); // only gc timers left
assertEquals(0, testHarness.numKeyedStateEntries()); assertEquals(4, testHarness.numKeyedStateEntries()); // window-contents and trigger state for two windows assertEquals(4, timeAdaptor.numTimers(testHarness)); // timers/gc timers for two windows assertEquals(2, testHarness.numKeyedStateEntries()); // trigger state stays until GC time assertEquals(2, timeAdaptor.numTimers(testHarness)); // gc timers are still there
assertEquals(0, testHarness.numKeyedStateEntries()); assertEquals(4, testHarness.numKeyedStateEntries()); // window contents plus trigger state assertEquals(4, testHarness.numEventTimeTimers()); // window timers/gc timers
assertEquals(0, testHarness.numKeyedStateEntries()); assertEquals(2, testHarness.numKeyedStateEntries()); // trigger state will stick around until GC time
@Test public void testLateWindowDropping() throws Exception { WindowAssigner<Integer, TimeWindow> mockAssigner = mockTimeWindowAssigner(); Trigger<Integer, TimeWindow> mockTrigger = mockTrigger(); InternalWindowFunction<Iterable<Integer>, Void, Integer, TimeWindow> mockWindowFunction = mockWindowFunction(); KeyedOneInputStreamOperatorTestHarness<Integer, Integer, Void> testHarness = createWindowOperator(mockAssigner, mockTrigger, 20L, mockWindowFunction); testHarness.open(); when(mockAssigner.assignWindows(anyInt(), anyLong(), anyAssignerContext())) .thenReturn(Arrays.asList(new TimeWindow(0, 2))); assertEquals(0, testHarness.getOutput().size()); assertEquals(0, testHarness.numKeyedStateEntries()); shouldFireOnElement(mockTrigger); // window.maxTime() == 1 plus 20L of allowed lateness testHarness.processWatermark(new Watermark(21)); testHarness.processElement(new StreamRecord<>(0, 0L)); // there should be nothing assertEquals(0, testHarness.numKeyedStateEntries()); assertEquals(0, testHarness.numEventTimeTimers()); assertEquals(0, testHarness.numProcessingTimeTimers()); // there should be two elements now assertEquals(0, testHarness.extractOutputStreamRecords().size()); }
assertEquals(0, testHarness.numKeyedStateEntries()); assertEquals(1, testHarness.numKeyedStateEntries());
assertEquals(0, testHarness.numKeyedStateEntries()); assertEquals(4, testHarness.numKeyedStateEntries()); // window-contents and trigger state for two windows assertEquals(4, timeAdaptor.numTimers(testHarness)); // timers/gc timers for two windows assertEquals(2, testHarness.numKeyedStateEntries()); // trigger state will stick around assertEquals(2, timeAdaptor.numTimers(testHarness)); // gc timers are still there
assertEquals(0, testHarness.numKeyedStateEntries()); assertEquals(4, testHarness.numKeyedStateEntries()); // window-contents plus trigger state for two windows assertEquals(4, timeAdaptor.numTimers(testHarness)); // timers/gc timers for two windows assertEquals(4, testHarness.numKeyedStateEntries()); assertEquals(2, timeAdaptor.numTimers(testHarness)); // only gc timers left
assertEquals(0, testHarness.numKeyedStateEntries()); assertEquals(1, testHarness.numKeyedStateEntries());
private void testNoGarbageCollectionTimerForGlobalWindow(TimeDomainAdaptor timeAdaptor) throws Exception { WindowAssigner<Integer, GlobalWindow> mockAssigner = mockGlobalWindowAssigner(); timeAdaptor.setIsEventTime(mockAssigner); Trigger<Integer, GlobalWindow> mockTrigger = mockTrigger(); InternalWindowFunction<Iterable<Integer>, Void, Integer, GlobalWindow> mockWindowFunction = mockWindowFunction(); // this needs to be true for the test to succeed assertEquals(Long.MAX_VALUE, GlobalWindow.get().maxTimestamp()); KeyedOneInputStreamOperatorTestHarness<Integer, Integer, Void> testHarness = createWindowOperator(mockAssigner, mockTrigger, 0L, mockWindowFunction); testHarness.open(); assertEquals(0, testHarness.getOutput().size()); assertEquals(0, testHarness.numKeyedStateEntries()); testHarness.processElement(new StreamRecord<>(0, 0L)); // just the window contents assertEquals(1, testHarness.numKeyedStateEntries()); // verify we have no timers for either time domain assertEquals(0, testHarness.numEventTimeTimers()); assertEquals(0, testHarness.numProcessingTimeTimers()); }