@Test public void testAssignerIsInvokedOncePerElement() throws Exception { WindowAssigner<Integer, TimeWindow> mockAssigner = mockTimeWindowAssigner(); Trigger<Integer, TimeWindow> mockTrigger = mockTrigger(); InternalWindowFunction<Iterable<Integer>, Void, Integer, TimeWindow> mockWindowFunction = mockWindowFunction(); OneInputStreamOperatorTestHarness<Integer, Void> testHarness = createWindowOperator(mockAssigner, mockTrigger, 0L, mockWindowFunction); testHarness.open(); when(mockAssigner.assignWindows(anyInt(), anyLong(), anyAssignerContext())) .thenReturn(Collections.singletonList(new TimeWindow(0, 0))); testHarness.processElement(new StreamRecord<>(0, 0L)); verify(mockAssigner, times(1)).assignWindows(eq(0), eq(0L), anyAssignerContext()); testHarness.processElement(new StreamRecord<>(0, 0L)); verify(mockAssigner, times(2)).assignWindows(eq(0), eq(0L), anyAssignerContext()); }
WindowOperator<Integer, Integer, ACC, OUT, W> operator = new WindowOperator<>( assigner, assigner.getWindowSerializer(new ExecutionConfig()), keySelector, IntSerializer.INSTANCE,
/** * Decide if a record is currently late, based on current watermark and allowed lateness. * * @param element The element to check * @return The element for which should be considered when sideoutputs */ protected boolean isElementLate(StreamRecord<IN> element){ return (windowAssigner.isEventTime()) && (element.getTimestamp() + allowedLateness <= internalTimerService.currentWatermark()); }
static <T> WindowAssigner<T, TimeWindow> mockTimeWindowAssigner() throws Exception { @SuppressWarnings("unchecked") WindowAssigner<T, TimeWindow> mockAssigner = mock(WindowAssigner.class); when(mockAssigner.getWindowSerializer(Mockito.<ExecutionConfig>any())).thenReturn(new TimeWindow.Serializer()); when(mockAssigner.isEventTime()).thenReturn(true); return mockAssigner; }
static <T> WindowAssigner<T, GlobalWindow> mockGlobalWindowAssigner() throws Exception { @SuppressWarnings("unchecked") WindowAssigner<T, GlobalWindow> mockAssigner = mock(WindowAssigner.class); when(mockAssigner.getWindowSerializer(Mockito.<ExecutionConfig>any())).thenReturn(new GlobalWindow.Serializer()); when(mockAssigner.isEventTime()).thenReturn(true); when(mockAssigner.assignWindows(Mockito.<T>any(), anyLong(), anyAssignerContext())).thenReturn(Collections.singletonList(GlobalWindow.get())); return mockAssigner; }
@Test public void testProcessingTimeGarbageCollectionTimerIsAlwaysWindowMaxTimestamp() throws Exception { WindowAssigner<Integer, TimeWindow> mockAssigner = mockTimeWindowAssigner(); when(mockAssigner.isEventTime()).thenReturn(false); Trigger<Integer, TimeWindow> mockTrigger = mockTrigger(); InternalWindowFunction<Iterable<Integer>, Void, Integer, TimeWindow> mockWindowFunction = mockWindowFunction(); KeyedOneInputStreamOperatorTestHarness<Integer, Integer, Void> testHarness = createWindowOperator(mockAssigner, mockTrigger, 20L, mockWindowFunction); testHarness.open(); when(mockAssigner.assignWindows(anyInt(), anyLong(), anyAssignerContext())) .thenReturn(Arrays.asList(new TimeWindow(0, Long.MAX_VALUE - 10))); assertEquals(0, testHarness.getOutput().size()); assertEquals(0, testHarness.numKeyedStateEntries()); testHarness.processElement(new StreamRecord<>(0, 0L)); // just the window contents assertEquals(1, testHarness.numKeyedStateEntries()); // no GC timer assertEquals(0, testHarness.numEventTimeTimers()); assertEquals(1, testHarness.numProcessingTimeTimers()); verify(mockTrigger, never()).clear(anyTimeWindow(), anyTriggerContext()); testHarness.setProcessingTime(Long.MAX_VALUE - 10); verify(mockTrigger, times(1)).clear(anyTimeWindow(), anyTriggerContext()); assertEquals(0, testHarness.numEventTimeTimers()); assertEquals(0, testHarness.numProcessingTimeTimers()); }
@PublicEvolving public WindowedStream(KeyedStream<T, K> input, WindowAssigner<? super T, W> windowAssigner) { this.input = input; this.windowAssigner = windowAssigner; this.trigger = windowAssigner.getDefaultTrigger(input.getExecutionEnvironment()); }
@PublicEvolving public AllWindowedStream(DataStream<T> input, WindowAssigner<? super T, W> windowAssigner) { this.input = input.keyBy(new NullByteKeySelector<T>()); this.windowAssigner = windowAssigner; this.trigger = windowAssigner.getDefaultTrigger(input.getExecutionEnvironment()); }
@Test public void testLateSideOutput() throws Exception { OutputTag<Integer> lateOutputTag = new OutputTag<Integer>("late"){}; WindowAssigner<Integer, TimeWindow> mockAssigner = mockTimeWindowAssigner(); Trigger<Integer, TimeWindow> mockTrigger = mockTrigger(); InternalWindowFunction<Iterable<Integer>, Void, Integer, TimeWindow> mockWindowFunction = mockWindowFunction(); OneInputStreamOperatorTestHarness<Integer, Void> testHarness = createWindowOperator(mockAssigner, mockTrigger, 0L, mockWindowFunction, lateOutputTag); testHarness.open(); when(mockAssigner.assignWindows(anyInt(), anyLong(), anyAssignerContext())) .thenReturn(Collections.singletonList(new TimeWindow(0, 0))); testHarness.processWatermark(20); testHarness.processElement(new StreamRecord<>(0, 5L)); verify(mockAssigner, times(1)).assignWindows(eq(0), eq(5L), anyAssignerContext()); assertThat(testHarness.getSideOutput(lateOutputTag), contains(isStreamRecord(0, 5L))); // we should also see side output if the WindowAssigner assigns no windows when(mockAssigner.assignWindows(anyInt(), anyLong(), anyAssignerContext())) .thenReturn(Collections.<TimeWindow>emptyList()); testHarness.processElement(new StreamRecord<>(0, 10L)); verify(mockAssigner, times(1)).assignWindows(eq(0), eq(5L), anyAssignerContext()); verify(mockAssigner, times(1)).assignWindows(eq(0), eq(10L), anyAssignerContext()); assertThat(testHarness.getSideOutput(lateOutputTag), contains(isStreamRecord(0, 5L), isStreamRecord(0, 10L))); }
windowAssigner.getWindowSerializer(getExecutionEnvironment().getConfig()), keySel, input.getKeyType().createSerializer(getExecutionEnvironment().getConfig()), windowAssigner.getWindowSerializer(getExecutionEnvironment().getConfig()), keySel, input.getKeyType().createSerializer(getExecutionEnvironment().getConfig()),
/** * Returns {@code true} if the watermark is after the end timestamp plus the allowed lateness * of the given window. */ protected boolean isWindowLate(W window) { return (windowAssigner.isEventTime() && (cleanupTime(window) <= internalTimerService.currentWatermark())); }
@PublicEvolving public WindowedStream(KeyedStream<T, K> input, WindowAssigner<? super T, W> windowAssigner) { this.input = input; this.windowAssigner = windowAssigner; this.trigger = windowAssigner.getDefaultTrigger(input.getExecutionEnvironment()); }
@Test public void testOnElementCalledPerWindow() throws Exception { WindowAssigner<Integer, TimeWindow> mockAssigner = mockTimeWindowAssigner(); Trigger<Integer, TimeWindow> mockTrigger = mockTrigger(); InternalWindowFunction<Iterable<Integer>, Void, Integer, TimeWindow> mockWindowFunction = mockWindowFunction(); OneInputStreamOperatorTestHarness<Integer, Void> testHarness = createWindowOperator(mockAssigner, mockTrigger, 0L, mockWindowFunction); testHarness.open(); when(mockAssigner.assignWindows(anyInt(), anyLong(), anyAssignerContext())) .thenReturn(Arrays.asList(new TimeWindow(2, 4), new TimeWindow(0, 2))); testHarness.processElement(new StreamRecord<>(42, 1L)); verify(mockTrigger).onElement(eq(42), eq(1L), eq(new TimeWindow(2, 4)), anyTriggerContext()); verify(mockTrigger).onElement(eq(42), eq(1L), eq(new TimeWindow(0, 2)), anyTriggerContext()); verify(mockTrigger, times(2)).onElement(anyInt(), anyLong(), anyTimeWindow(), anyTriggerContext()); }
WindowOperator<Integer, Integer, Iterable<Integer>, OUT, W> operator = new WindowOperator<>( assigner, assigner.getWindowSerializer(new ExecutionConfig()), keySelector, IntSerializer.INSTANCE,
@Override public void setIsEventTime(WindowAssigner<?, ?> mockAssigner) { when(mockAssigner.isEventTime()).thenReturn(true); }
@PublicEvolving public WindowedStream(KeyedStream<T, K> input, WindowAssigner<? super T, W> windowAssigner) { this.input = input; this.windowAssigner = windowAssigner; this.trigger = windowAssigner.getDefaultTrigger(input.getExecutionEnvironment()); }
/** * Verify that there is no late-data side output if the {@code WindowAssigner} does * not assign any windows. */ @Test public void testNoLateSideOutputForSkippedWindows() throws Exception { OutputTag<Integer> lateOutputTag = new OutputTag<Integer>("late"){}; WindowAssigner<Integer, TimeWindow> mockAssigner = mockTimeWindowAssigner(); Trigger<Integer, TimeWindow> mockTrigger = mockTrigger(); InternalWindowFunction<Iterable<Integer>, Void, Integer, TimeWindow> mockWindowFunction = mockWindowFunction(); OneInputStreamOperatorTestHarness<Integer, Void> testHarness = createWindowOperator(mockAssigner, mockTrigger, 0L, mockWindowFunction, lateOutputTag); testHarness.open(); when(mockAssigner.assignWindows(anyInt(), anyLong(), anyAssignerContext())) .thenReturn(Collections.<TimeWindow>emptyList()); testHarness.processWatermark(0); testHarness.processElement(new StreamRecord<>(0, 5L)); verify(mockAssigner, times(1)).assignWindows(eq(0), eq(5L), anyAssignerContext()); assertTrue(testHarness.getSideOutput(lateOutputTag) == null || testHarness.getSideOutput(lateOutputTag).isEmpty()); }
windowAssigner.getWindowSerializer(getExecutionEnvironment().getConfig()), keySel, input.getKeyType().createSerializer(getExecutionEnvironment().getConfig()), windowAssigner.getWindowSerializer(getExecutionEnvironment().getConfig()), keySel, input.getKeyType().createSerializer(getExecutionEnvironment().getConfig()),
@Override public void setIsEventTime(WindowAssigner<?, ?> mockAssigner) { when(mockAssigner.isEventTime()).thenReturn(false); }
@PublicEvolving public WindowedStream(KeyedStream<T, K> input, WindowAssigner<? super T, W> windowAssigner) { this.input = input; this.windowAssigner = windowAssigner; this.trigger = windowAssigner.getDefaultTrigger(input.getExecutionEnvironment()); }