static <T> WindowAssigner<T, GlobalWindow> mockGlobalWindowAssigner() throws Exception { @SuppressWarnings("unchecked") WindowAssigner<T, GlobalWindow> mockAssigner = mock(WindowAssigner.class); when(mockAssigner.getWindowSerializer(Mockito.<ExecutionConfig>any())).thenReturn(new GlobalWindow.Serializer()); when(mockAssigner.isEventTime()).thenReturn(true); when(mockAssigner.assignWindows(Mockito.<T>any(), anyLong(), anyAssignerContext())).thenReturn(Collections.singletonList(GlobalWindow.get())); return mockAssigner; }
@Test public void testAssignerIsInvokedOncePerElement() throws Exception { WindowAssigner<Integer, TimeWindow> mockAssigner = mockTimeWindowAssigner(); Trigger<Integer, TimeWindow> mockTrigger = mockTrigger(); InternalWindowFunction<Iterable<Integer>, Void, Integer, TimeWindow> mockWindowFunction = mockWindowFunction(); OneInputStreamOperatorTestHarness<Integer, Void> testHarness = createWindowOperator(mockAssigner, mockTrigger, 0L, mockWindowFunction); testHarness.open(); when(mockAssigner.assignWindows(anyInt(), anyLong(), anyAssignerContext())) .thenReturn(Collections.singletonList(new TimeWindow(0, 0))); testHarness.processElement(new StreamRecord<>(0, 0L)); verify(mockAssigner, times(1)).assignWindows(eq(0), eq(0L), anyAssignerContext()); testHarness.processElement(new StreamRecord<>(0, 0L)); verify(mockAssigner, times(2)).assignWindows(eq(0), eq(0L), anyAssignerContext()); }
@Test public void testLateSideOutput() throws Exception { OutputTag<Integer> lateOutputTag = new OutputTag<Integer>("late"){}; WindowAssigner<Integer, TimeWindow> mockAssigner = mockTimeWindowAssigner(); Trigger<Integer, TimeWindow> mockTrigger = mockTrigger(); InternalWindowFunction<Iterable<Integer>, Void, Integer, TimeWindow> mockWindowFunction = mockWindowFunction(); OneInputStreamOperatorTestHarness<Integer, Void> testHarness = createWindowOperator(mockAssigner, mockTrigger, 0L, mockWindowFunction, lateOutputTag); testHarness.open(); when(mockAssigner.assignWindows(anyInt(), anyLong(), anyAssignerContext())) .thenReturn(Collections.singletonList(new TimeWindow(0, 0))); testHarness.processWatermark(20); testHarness.processElement(new StreamRecord<>(0, 5L)); verify(mockAssigner, times(1)).assignWindows(eq(0), eq(5L), anyAssignerContext()); assertThat(testHarness.getSideOutput(lateOutputTag), contains(isStreamRecord(0, 5L))); // we should also see side output if the WindowAssigner assigns no windows when(mockAssigner.assignWindows(anyInt(), anyLong(), anyAssignerContext())) .thenReturn(Collections.<TimeWindow>emptyList()); testHarness.processElement(new StreamRecord<>(0, 10L)); verify(mockAssigner, times(1)).assignWindows(eq(0), eq(5L), anyAssignerContext()); verify(mockAssigner, times(1)).assignWindows(eq(0), eq(10L), anyAssignerContext()); assertThat(testHarness.getSideOutput(lateOutputTag), contains(isStreamRecord(0, 5L), isStreamRecord(0, 10L))); }
@Test public void testOnElementCalledPerWindow() throws Exception { WindowAssigner<Integer, TimeWindow> mockAssigner = mockTimeWindowAssigner(); Trigger<Integer, TimeWindow> mockTrigger = mockTrigger(); InternalWindowFunction<Iterable<Integer>, Void, Integer, TimeWindow> mockWindowFunction = mockWindowFunction(); OneInputStreamOperatorTestHarness<Integer, Void> testHarness = createWindowOperator(mockAssigner, mockTrigger, 0L, mockWindowFunction); testHarness.open(); when(mockAssigner.assignWindows(anyInt(), anyLong(), anyAssignerContext())) .thenReturn(Arrays.asList(new TimeWindow(2, 4), new TimeWindow(0, 2))); testHarness.processElement(new StreamRecord<>(42, 1L)); verify(mockTrigger).onElement(eq(42), eq(1L), eq(new TimeWindow(2, 4)), anyTriggerContext()); verify(mockTrigger).onElement(eq(42), eq(1L), eq(new TimeWindow(0, 2)), anyTriggerContext()); verify(mockTrigger, times(2)).onElement(anyInt(), anyLong(), anyTimeWindow(), anyTriggerContext()); }
/** * Verify that there is no late-data side output if the {@code WindowAssigner} does * not assign any windows. */ @Test public void testNoLateSideOutputForSkippedWindows() throws Exception { OutputTag<Integer> lateOutputTag = new OutputTag<Integer>("late"){}; WindowAssigner<Integer, TimeWindow> mockAssigner = mockTimeWindowAssigner(); Trigger<Integer, TimeWindow> mockTrigger = mockTrigger(); InternalWindowFunction<Iterable<Integer>, Void, Integer, TimeWindow> mockWindowFunction = mockWindowFunction(); OneInputStreamOperatorTestHarness<Integer, Void> testHarness = createWindowOperator(mockAssigner, mockTrigger, 0L, mockWindowFunction, lateOutputTag); testHarness.open(); when(mockAssigner.assignWindows(anyInt(), anyLong(), anyAssignerContext())) .thenReturn(Collections.<TimeWindow>emptyList()); testHarness.processWatermark(0); testHarness.processElement(new StreamRecord<>(0, 5L)); verify(mockAssigner, times(1)).assignWindows(eq(0), eq(5L), anyAssignerContext()); assertTrue(testHarness.getSideOutput(lateOutputTag) == null || testHarness.getSideOutput(lateOutputTag).isEmpty()); }
@Test public void testAssignerWithMultipleWindows() throws Exception { WindowAssigner<Integer, TimeWindow> mockAssigner = mockTimeWindowAssigner(); Trigger<Integer, TimeWindow> mockTrigger = mockTrigger(); InternalWindowFunction<Iterable<Integer>, Void, Integer, TimeWindow> mockWindowFunction = mockWindowFunction(); OneInputStreamOperatorTestHarness<Integer, Void> testHarness = createWindowOperator(mockAssigner, mockTrigger, 0L, mockWindowFunction); testHarness.open(); when(mockAssigner.assignWindows(anyInt(), anyLong(), anyAssignerContext())) .thenReturn(Arrays.asList(new TimeWindow(2, 4), new TimeWindow(0, 2))); shouldFireOnElement(mockTrigger); testHarness.processElement(new StreamRecord<>(0, 0L)); verify(mockWindowFunction, times(2)).process(eq(0), anyTimeWindow(), anyInternalWindowContext(), anyIntIterable(), WindowOperatorContractTest.<Void>anyCollector()); verify(mockWindowFunction, times(1)).process(eq(0), eq((new TimeWindow(0, 2))), anyInternalWindowContext(), intIterable(0), WindowOperatorContractTest.<Void>anyCollector()); verify(mockWindowFunction, times(1)).process(eq(0), eq(new TimeWindow(2, 4)), anyInternalWindowContext(), intIterable(0), WindowOperatorContractTest.<Void>anyCollector()); }
when(mockAssigner.assignWindows(anyInt(), anyLong(), anyAssignerContext())) .thenReturn(Collections.singletonList(new TimeWindow(0, 2))); when(mockAssigner.assignWindows(anyInt(), anyLong(), anyAssignerContext())) .thenReturn(Collections.singletonList(new TimeWindow(0, 1))); when(mockAssigner.assignWindows(anyInt(), anyLong(), anyAssignerContext())) .thenReturn(Collections.singletonList(new TimeWindow(0, 1))); when(mockAssigner.assignWindows(anyInt(), anyLong(), anyAssignerContext())) .thenReturn(Collections.singletonList(new TimeWindow(0, 2)));
when(mockAssigner.assignWindows(anyInt(), anyLong(), anyAssignerContext())) .thenReturn(Collections.singletonList(new TimeWindow(0, 2)));
@Test public void testWindowStateNotAvailableToMergingWindows() throws Exception { WindowAssigner<Integer, TimeWindow> mockAssigner = mockMergingAssigner(); Trigger<Integer, TimeWindow> mockTrigger = mockTrigger(); InternalWindowFunction<Iterable<Integer>, Void, Integer, TimeWindow> mockWindowFunction = mockWindowFunction(); KeyedOneInputStreamOperatorTestHarness<Integer, Integer, Void> testHarness = createWindowOperator(mockAssigner, mockTrigger, 20L, mockWindowFunction); testHarness.open(); when(mockTrigger.onElement(anyInt(), anyLong(), anyTimeWindow(), anyTriggerContext())) .thenReturn(TriggerResult.FIRE); when(mockAssigner.assignWindows(anyInt(), anyLong(), anyAssignerContext())) .thenReturn(Arrays.asList(new TimeWindow(0, 20))); doAnswer(new Answer<Object>() { @Override public Object answer(InvocationOnMock invocationOnMock) throws Throwable { InternalWindowFunction.InternalWindowContext context = (InternalWindowFunction.InternalWindowContext) invocationOnMock.getArguments()[2]; context.windowState().getState(valueStateDescriptor).update("hello"); return null; } }).when(mockWindowFunction).process(anyInt(), anyTimeWindow(), anyInternalWindowContext(), anyIntIterable(), WindowOperatorContractTest.<Void>anyCollector()); expectedException.expect(UnsupportedOperationException.class); expectedException.expectMessage("Per-window state is not allowed when using merging windows."); testHarness.processElement(new StreamRecord<>(0, 0L)); }
private void testEmittingFromWindowFunction(TimeDomainAdaptor timeAdaptor) throws Exception { WindowAssigner<Integer, TimeWindow> mockAssigner = mockTimeWindowAssigner(); Trigger<Integer, TimeWindow> mockTrigger = mockTrigger(); InternalWindowFunction<Iterable<Integer>, String, Integer, TimeWindow> mockWindowFunction = mockWindowFunction(); KeyedOneInputStreamOperatorTestHarness<Integer, Integer, String> testHarness = createWindowOperator(mockAssigner, mockTrigger, 0L, mockWindowFunction); testHarness.open(); when(mockAssigner.assignWindows(anyInt(), anyLong(), anyAssignerContext())) .thenReturn(Collections.singletonList(new TimeWindow(0, 2))); doAnswer(new Answer<Void>() { @Override public Void answer(InvocationOnMock invocation) throws Exception { @SuppressWarnings("unchecked") Collector<String> out = invocation.getArgument(4); out.collect("Hallo"); out.collect("Ciao"); return null; } }).when(mockWindowFunction).process(eq(0), eq(new TimeWindow(0, 2)), anyInternalWindowContext(), intIterable(0), WindowOperatorContractTest.<String>anyCollector()); timeAdaptor.shouldRegisterTimerOnElement(mockTrigger, 1); testHarness.processElement(new StreamRecord<>(0, 0L)); verify(mockWindowFunction, never()).process(anyInt(), anyTimeWindow(), anyInternalWindowContext(), anyIntIterable(), WindowOperatorContractTest.<String>anyCollector()); assertTrue(testHarness.extractOutputStreamRecords().isEmpty()); timeAdaptor.shouldFireOnTime(mockTrigger); timeAdaptor.advanceTime(testHarness, 1L); verify(mockWindowFunction, times(1)).process(eq(0), eq(new TimeWindow(0, 2)), anyInternalWindowContext(), intIterable(0), WindowOperatorContractTest.<String>anyCollector()); assertThat(testHarness.extractOutputStreamRecords(), contains(isStreamRecord("Hallo", 1L), isStreamRecord("Ciao", 1L))); }
when(mockAssigner.assignWindows(anyInt(), anyLong(), anyAssignerContext())) .thenReturn(Arrays.asList(new TimeWindow(2, 4), new TimeWindow(0, 2)));
@Test public void testProcessingElementsWithinAllowedLateness() throws Exception { WindowAssigner<Integer, TimeWindow> mockAssigner = mockTimeWindowAssigner(); Trigger<Integer, TimeWindow> mockTrigger = mockTrigger(); InternalWindowFunction<Iterable<Integer>, Void, Integer, TimeWindow> mockWindowFunction = mockWindowFunction(); KeyedOneInputStreamOperatorTestHarness<Integer, Integer, Void> testHarness = createWindowOperator(mockAssigner, mockTrigger, 20L, mockWindowFunction); testHarness.open(); when(mockAssigner.assignWindows(anyInt(), anyLong(), anyAssignerContext())) .thenReturn(Arrays.asList(new TimeWindow(0, 2))); assertEquals(0, testHarness.getOutput().size()); assertEquals(0, testHarness.numKeyedStateEntries()); shouldFireOnElement(mockTrigger); // 20 is just at the limit, window.maxTime() is 1 and allowed lateness is 20 testHarness.processWatermark(new Watermark(20)); testHarness.processElement(new StreamRecord<>(0, 0L)); verify(mockWindowFunction, times(1)).process(eq(0), eq(new TimeWindow(0, 2)), anyInternalWindowContext(), intIterable(0), WindowOperatorContractTest.<Void>anyCollector()); // clear is only called at cleanup time/GC time verify(mockTrigger, never()).clear(anyTimeWindow(), anyTriggerContext()); // FIRE should not purge contents assertEquals(1, testHarness.numKeyedStateEntries()); // window contents plus trigger state assertEquals(1, testHarness.numEventTimeTimers()); // just the GC timer }
@Test public void testProcessingTimeGarbageCollectionTimerIsAlwaysWindowMaxTimestamp() throws Exception { WindowAssigner<Integer, TimeWindow> mockAssigner = mockTimeWindowAssigner(); when(mockAssigner.isEventTime()).thenReturn(false); Trigger<Integer, TimeWindow> mockTrigger = mockTrigger(); InternalWindowFunction<Iterable<Integer>, Void, Integer, TimeWindow> mockWindowFunction = mockWindowFunction(); KeyedOneInputStreamOperatorTestHarness<Integer, Integer, Void> testHarness = createWindowOperator(mockAssigner, mockTrigger, 20L, mockWindowFunction); testHarness.open(); when(mockAssigner.assignWindows(anyInt(), anyLong(), anyAssignerContext())) .thenReturn(Arrays.asList(new TimeWindow(0, Long.MAX_VALUE - 10))); assertEquals(0, testHarness.getOutput().size()); assertEquals(0, testHarness.numKeyedStateEntries()); testHarness.processElement(new StreamRecord<>(0, 0L)); // just the window contents assertEquals(1, testHarness.numKeyedStateEntries()); // no GC timer assertEquals(0, testHarness.numEventTimeTimers()); assertEquals(1, testHarness.numProcessingTimeTimers()); verify(mockTrigger, never()).clear(anyTimeWindow(), anyTriggerContext()); testHarness.setProcessingTime(Long.MAX_VALUE - 10); verify(mockTrigger, times(1)).clear(anyTimeWindow(), anyTriggerContext()); assertEquals(0, testHarness.numEventTimeTimers()); assertEquals(0, testHarness.numProcessingTimeTimers()); }
@Test public void testNoEventTimeGarbageCollectionTimerForLongMax() throws Exception { WindowAssigner<Integer, TimeWindow> mockAssigner = mockTimeWindowAssigner(); Trigger<Integer, TimeWindow> mockTrigger = mockTrigger(); InternalWindowFunction<Iterable<Integer>, Void, Integer, TimeWindow> mockWindowFunction = mockWindowFunction(); KeyedOneInputStreamOperatorTestHarness<Integer, Integer, Void> testHarness = createWindowOperator(mockAssigner, mockTrigger, 20L, mockWindowFunction); testHarness.open(); when(mockAssigner.assignWindows(anyInt(), anyLong(), anyAssignerContext())) .thenReturn(Arrays.asList(new TimeWindow(0, Long.MAX_VALUE - 10))); assertEquals(0, testHarness.getOutput().size()); assertEquals(0, testHarness.numKeyedStateEntries()); testHarness.processElement(new StreamRecord<>(0, 0L)); // just the window contents assertEquals(1, testHarness.numKeyedStateEntries()); // no GC timer assertEquals(0, testHarness.numEventTimeTimers()); assertEquals(0, testHarness.numProcessingTimeTimers()); }
when(mockAssigner.assignWindows(anyInt(), anyLong(), anyAssignerContext())) .thenReturn(Arrays.asList(new TimeWindow(2, 4), new TimeWindow(0, 2)));
when(mockAssigner.assignWindows(anyInt(), anyLong(), anyAssignerContext())) .thenReturn(Arrays.asList(new TimeWindow(2, 4), new TimeWindow(0, 2)));
private void testMergingWindowSetClearedAtGarbageCollection(TimeDomainAdaptor timeAdaptor) throws Exception { WindowAssigner<Integer, TimeWindow> mockAssigner = mockMergingAssigner(); timeAdaptor.setIsEventTime(mockAssigner); Trigger<Integer, TimeWindow> mockTrigger = mockTrigger(); InternalWindowFunction<Iterable<Integer>, Void, Integer, TimeWindow> mockWindowFunction = mockWindowFunction(); KeyedOneInputStreamOperatorTestHarness<Integer, Integer, Void> testHarness = createWindowOperator(mockAssigner, mockTrigger, 20L, mockWindowFunction); testHarness.open(); when(mockAssigner.assignWindows(anyInt(), anyLong(), anyAssignerContext())) .thenReturn(Arrays.asList(new TimeWindow(0, 20))); assertEquals(0, testHarness.getOutput().size()); assertEquals(0, testHarness.numKeyedStateEntries()); testHarness.processElement(new StreamRecord<>(0, 0L)); assertEquals(2, testHarness.numKeyedStateEntries()); // window contents plus merging window set assertEquals(1, timeAdaptor.numTimers(testHarness)); // gc timers timeAdaptor.advanceTime(testHarness, 19 + 20); // 19 is maxTime of the window assertEquals(0, testHarness.numKeyedStateEntries()); assertEquals(0, timeAdaptor.numTimers(testHarness)); }
@Test public void testLateWindowDropping() throws Exception { WindowAssigner<Integer, TimeWindow> mockAssigner = mockTimeWindowAssigner(); Trigger<Integer, TimeWindow> mockTrigger = mockTrigger(); InternalWindowFunction<Iterable<Integer>, Void, Integer, TimeWindow> mockWindowFunction = mockWindowFunction(); KeyedOneInputStreamOperatorTestHarness<Integer, Integer, Void> testHarness = createWindowOperator(mockAssigner, mockTrigger, 20L, mockWindowFunction); testHarness.open(); when(mockAssigner.assignWindows(anyInt(), anyLong(), anyAssignerContext())) .thenReturn(Arrays.asList(new TimeWindow(0, 2))); assertEquals(0, testHarness.getOutput().size()); assertEquals(0, testHarness.numKeyedStateEntries()); shouldFireOnElement(mockTrigger); // window.maxTime() == 1 plus 20L of allowed lateness testHarness.processWatermark(new Watermark(21)); testHarness.processElement(new StreamRecord<>(0, 0L)); // there should be nothing assertEquals(0, testHarness.numKeyedStateEntries()); assertEquals(0, testHarness.numEventTimeTimers()); assertEquals(0, testHarness.numProcessingTimeTimers()); // there should be two elements now assertEquals(0, testHarness.extractOutputStreamRecords().size()); }
private void testDeletedTimerDoesNotFire(TimeDomainAdaptor timeAdaptor) throws Exception { WindowAssigner<Integer, TimeWindow> mockAssigner = mockTimeWindowAssigner(); timeAdaptor.setIsEventTime(mockAssigner); Trigger<Integer, TimeWindow> mockTrigger = mockTrigger(); InternalWindowFunction<Iterable<Integer>, Void, Integer, TimeWindow> mockWindowFunction = mockWindowFunction(); KeyedOneInputStreamOperatorTestHarness<Integer, Integer, Void> testHarness = createWindowOperator(mockAssigner, mockTrigger, 0L, mockWindowFunction); testHarness.open(); when(mockAssigner.assignWindows(anyInt(), anyLong(), anyAssignerContext())) .thenReturn(Arrays.asList(new TimeWindow(0, 100))); assertEquals(0, timeAdaptor.numTimers(testHarness)); timeAdaptor.shouldRegisterTimerOnElement(mockTrigger, 1); testHarness.processElement(new StreamRecord<>(0, 0L)); assertEquals(2, timeAdaptor.numTimers(testHarness)); // +1 for the GC timer timeAdaptor.shouldDeleteTimerOnElement(mockTrigger, 1); testHarness.processElement(new StreamRecord<>(0, 0L)); assertEquals(1, timeAdaptor.numTimers(testHarness)); // +1 for the GC timer timeAdaptor.shouldRegisterTimerOnElement(mockTrigger, 2); testHarness.processElement(new StreamRecord<>(0, 0L)); assertEquals(2, timeAdaptor.numTimers(testHarness)); // +1 for the GC timer timeAdaptor.advanceTime(testHarness, 50L); timeAdaptor.verifyTriggerCallback(mockTrigger, times(0), 1L, null); timeAdaptor.verifyTriggerCallback(mockTrigger, times(1), 2L, new TimeWindow(0, 100)); timeAdaptor.verifyTriggerCallback(mockTrigger, times(1), null, null); assertEquals(1, timeAdaptor.numTimers(testHarness)); // +1 for the GC timer }
private void testTimerCreationAndDeletion(TimeDomainAdaptor timeAdaptor) throws Exception { WindowAssigner<Integer, TimeWindow> mockAssigner = mockTimeWindowAssigner(); timeAdaptor.setIsEventTime(mockAssigner); Trigger<Integer, TimeWindow> mockTrigger = mockTrigger(); InternalWindowFunction<Iterable<Integer>, Void, Integer, TimeWindow> mockWindowFunction = mockWindowFunction(); KeyedOneInputStreamOperatorTestHarness<Integer, Integer, Void> testHarness = createWindowOperator(mockAssigner, mockTrigger, 0L, mockWindowFunction); testHarness.open(); when(mockAssigner.assignWindows(anyInt(), anyLong(), anyAssignerContext())) .thenReturn(Arrays.asList(new TimeWindow(0, 2))); assertEquals(0, timeAdaptor.numTimers(testHarness)); timeAdaptor.shouldRegisterTimerOnElement(mockTrigger, 17); testHarness.processElement(new StreamRecord<>(0, 0L)); assertEquals(2, timeAdaptor.numTimers(testHarness)); // +1 because of the GC timer of the window timeAdaptor.shouldRegisterTimerOnElement(mockTrigger, 42); testHarness.processElement(new StreamRecord<>(0, 0L)); assertEquals(3, timeAdaptor.numTimers(testHarness)); // +1 because of the GC timer of the window timeAdaptor.shouldDeleteTimerOnElement(mockTrigger, 42); testHarness.processElement(new StreamRecord<>(0, 0L)); timeAdaptor.shouldDeleteTimerOnElement(mockTrigger, 17); testHarness.processElement(new StreamRecord<>(0, 0L)); assertEquals(1, timeAdaptor.numTimers(testHarness)); // +1 because of the GC timer of the window }