/** * Decide if a record is currently late, based on current watermark and allowed lateness. * * @param element The element to check * @return The element for which should be considered when sideoutputs */ protected boolean isElementLate(StreamRecord<IN> element){ return (windowAssigner.isEventTime()) && (element.getTimestamp() + allowedLateness <= internalTimerService.currentWatermark()); }
/** * Returns {@code true} if the watermark is after the end timestamp plus the allowed lateness * of the given window. */ protected boolean isWindowLate(W window) { return (windowAssigner.isEventTime() && (cleanupTime(window) <= internalTimerService.currentWatermark())); }
@Override public void setIsEventTime(WindowAssigner<?, ?> mockAssigner) { when(mockAssigner.isEventTime()).thenReturn(true); }
@Override public void setIsEventTime(WindowAssigner<?, ?> mockAssigner) { when(mockAssigner.isEventTime()).thenReturn(false); }
/** * Returns the cleanup time for a window, which is * {@code window.maxTimestamp + allowedLateness}. In * case this leads to a value greater than {@link Long#MAX_VALUE} * then a cleanup time of {@link Long#MAX_VALUE} is * returned. * * @param window the window whose cleanup time we are computing. */ private long cleanupTime(W window) { if (windowAssigner.isEventTime()) { long cleanupTime = window.maxTimestamp() + allowedLateness; return cleanupTime >= window.maxTimestamp() ? cleanupTime : Long.MAX_VALUE; } else { return window.maxTimestamp(); } }
static <T> WindowAssigner<T, TimeWindow> mockTimeWindowAssigner() throws Exception { @SuppressWarnings("unchecked") WindowAssigner<T, TimeWindow> mockAssigner = mock(WindowAssigner.class); when(mockAssigner.getWindowSerializer(Mockito.<ExecutionConfig>any())).thenReturn(new TimeWindow.Serializer()); when(mockAssigner.isEventTime()).thenReturn(true); return mockAssigner; }
/** * Registers a timer to cleanup the content of the window. * @param window * the window whose state to discard */ protected void registerCleanupTimer(W window) { long cleanupTime = cleanupTime(window); if (cleanupTime == Long.MAX_VALUE) { // don't set a GC timer for "end of time" return; } if (windowAssigner.isEventTime()) { triggerContext.registerEventTimeTimer(cleanupTime); } else { triggerContext.registerProcessingTimeTimer(cleanupTime); } }
/** * Deletes the cleanup timer set for the contents of the provided window. * @param window * the window whose state to discard */ protected void deleteCleanupTimer(W window) { long cleanupTime = cleanupTime(window); if (cleanupTime == Long.MAX_VALUE) { // no need to clean up because we didn't set one return; } if (windowAssigner.isEventTime()) { triggerContext.deleteEventTimeTimer(cleanupTime); } else { triggerContext.deleteProcessingTimeTimer(cleanupTime); } }
static <T> WindowAssigner<T, GlobalWindow> mockGlobalWindowAssigner() throws Exception { @SuppressWarnings("unchecked") WindowAssigner<T, GlobalWindow> mockAssigner = mock(WindowAssigner.class); when(mockAssigner.getWindowSerializer(Mockito.<ExecutionConfig>any())).thenReturn(new GlobalWindow.Serializer()); when(mockAssigner.isEventTime()).thenReturn(true); when(mockAssigner.assignWindows(Mockito.<T>any(), anyLong(), anyAssignerContext())).thenReturn(Collections.singletonList(GlobalWindow.get())); return mockAssigner; }
@Test public void testProcessingTimeGarbageCollectionTimerIsAlwaysWindowMaxTimestamp() throws Exception { WindowAssigner<Integer, TimeWindow> mockAssigner = mockTimeWindowAssigner(); when(mockAssigner.isEventTime()).thenReturn(false); Trigger<Integer, TimeWindow> mockTrigger = mockTrigger(); InternalWindowFunction<Iterable<Integer>, Void, Integer, TimeWindow> mockWindowFunction = mockWindowFunction(); KeyedOneInputStreamOperatorTestHarness<Integer, Integer, Void> testHarness = createWindowOperator(mockAssigner, mockTrigger, 20L, mockWindowFunction); testHarness.open(); when(mockAssigner.assignWindows(anyInt(), anyLong(), anyAssignerContext())) .thenReturn(Arrays.asList(new TimeWindow(0, Long.MAX_VALUE - 10))); assertEquals(0, testHarness.getOutput().size()); assertEquals(0, testHarness.numKeyedStateEntries()); testHarness.processElement(new StreamRecord<>(0, 0L)); // just the window contents assertEquals(1, testHarness.numKeyedStateEntries()); // no GC timer assertEquals(0, testHarness.numEventTimeTimers()); assertEquals(1, testHarness.numProcessingTimeTimers()); verify(mockTrigger, never()).clear(anyTimeWindow(), anyTriggerContext()); testHarness.setProcessingTime(Long.MAX_VALUE - 10); verify(mockTrigger, times(1)).clear(anyTimeWindow(), anyTriggerContext()); assertEquals(0, testHarness.numEventTimeTimers()); assertEquals(0, testHarness.numProcessingTimeTimers()); }
if (!windowAssigner.isEventTime() && isCleanupTime(triggerContext.window, timer.getTimestamp())) { clearAllState(triggerContext.window, windowState, mergingWindows);
if (windowAssigner.isEventTime() && isCleanupTime(triggerContext.window, timer.getTimestamp())) { clearAllState(triggerContext.window, windowState, mergingWindows);
if (windowAssigner.isEventTime() && isCleanupTime(triggerContext.window, timer.getTimestamp())) { clearAllState(triggerContext.window, evictingWindowState, mergingWindows);
if (!windowAssigner.isEventTime() && isCleanupTime(triggerContext.window, timer.getTimestamp())) { clearAllState(triggerContext.window, evictingWindowState, mergingWindows);
/** * Decide if a record is currently late, based on current watermark and allowed lateness. * * @param element The element to check * @return The element for which should be considered when sideoutputs */ protected boolean isElementLate(StreamRecord<IN> element){ return (windowAssigner.isEventTime()) && (element.getTimestamp() + allowedLateness <= internalTimerService.currentWatermark()); }
/** * Returns {@code true} if the watermark is after the end timestamp plus the allowed lateness * of the given window. */ protected boolean isWindowLate(W window) { return (windowAssigner.isEventTime() && (cleanupTime(window) <= internalTimerService.currentWatermark())); }
/** * Decide if a record is currently late, based on current watermark and allowed lateness. * * @param element The element to check * @return The element for which should be considered when sideoutputs */ protected boolean isElementLate(StreamRecord<IN> element){ return (windowAssigner.isEventTime()) && (element.getTimestamp() + allowedLateness <= internalTimerService.currentWatermark()); }
/** * Returns {@code true} if the watermark is after the end timestamp plus the allowed lateness * of the given window. */ protected boolean isWindowLate(W window) { return (windowAssigner.isEventTime() && (cleanupTime(window) <= internalTimerService.currentWatermark())); }
/** * Decide if a record is currently late, based on current watermark and allowed lateness. * * @param element The element to check * @return The element for which should be considered when sideoutputs */ protected boolean isElementLate(StreamRecord<IN> element){ return (windowAssigner.isEventTime()) && (element.getTimestamp() + allowedLateness <= internalTimerService.currentWatermark()); }
/** * Returns {@code true} if the watermark is after the end timestamp plus the allowed lateness * of the given window. */ protected boolean isWindowLate(W window) { return (windowAssigner.isEventTime() && (cleanupTime(window) <= internalTimerService.currentWatermark())); }