/** * Return when {@code window} should be garbage collected. If the window's expiration time is on * or after the end of the global window, it will be truncated to the end of the global window. */ public static Instant garbageCollectionTime( BoundedWindow window, WindowingStrategy windowingStrategy) { return garbageCollectionTime(window, windowingStrategy.getAllowedLateness()); }
/** Is {@code window} expired w.r.t. the garbage collection watermark? */ private boolean canDropDueToExpiredWindow(BoundedWindow window) { Instant inputWM = timerInternals.currentInputWatermarkTime(); return LateDataUtils.garbageCollectionTime(window, windowingStrategy).isBefore(inputWM); } }
private boolean isLate(BoundedWindow window) { Instant gcTime = LateDataUtils.garbageCollectionTime(window, windowingStrategy); Instant inputWM = cleanupTimer.currentInputWatermarkTime(); return gcTime.isBefore(inputWM); }
@Override public boolean isForWindow( String timerId, BoundedWindow window, Instant timestamp, TimeDomain timeDomain) { boolean isEventTimer = timeDomain.equals(TimeDomain.EVENT_TIME); Instant gcTime = LateDataUtils.garbageCollectionTime(window, windowingStrategy); gcTime = gcTime.plus(GC_DELAY_MS); return isEventTimer && GC_TIMER_ID.equals(timerId) && gcTime.equals(timestamp); } }
@Override public void setForWindow(BoundedWindow window) { Instant gcTime = LateDataUtils.garbageCollectionTime(window, windowingStrategy); // make sure this fires after any window.maxTimestamp() timers gcTime = gcTime.plus(GC_DELAY_MS); timerInternals.setTimer( StateNamespaces.window(windowCoder, window), GC_TIMER_ID, gcTime, TimeDomain.EVENT_TIME); }
/** * Schedule a timer to garbage collect the window. * * <p>The timer: * * <ul> * <li>...must be fired strictly after the expiration of the window. * <li>...should be as close to the expiration as possible, to have a timely output of remaining * buffered data, and GC. * </ul> */ private void scheduleGarbageCollectionTimer(ReduceFn<?, ?, ?, W>.Context directContext) { Instant inputWM = timerInternals.currentInputWatermarkTime(); Instant gcTime = LateDataUtils.garbageCollectionTime(directContext.window(), windowingStrategy); WindowTracing.trace( "ReduceFnRunner.scheduleGarbageCollectionTimer: Scheduling at {} for " + "key:{}; window:{} where inputWatermark:{}; outputWatermark:{}", gcTime, key, directContext.window(), inputWM, timerInternals.currentOutputWatermarkTime()); checkState( !gcTime.isAfter(BoundedWindow.TIMESTAMP_MAX_VALUE), "Timer %s is beyond end-of-time", gcTime); directContext.timers().setTimer(gcTime, TimeDomain.EVENT_TIME); }
private void cancelEndOfWindowAndGarbageCollectionTimers( ReduceFn<?, ?, ?, W>.Context directContext) { WindowTracing.debug( "ReduceFnRunner.cancelEndOfWindowAndGarbageCollectionTimers: Deleting timers for " + "key:{}; window:{} where inputWatermark:{}; outputWatermark:{}", key, directContext.window(), timerInternals.currentInputWatermarkTime(), timerInternals.currentOutputWatermarkTime()); Instant eow = directContext.window().maxTimestamp(); directContext.timers().deleteTimer(eow, TimeDomain.EVENT_TIME); Instant gc = LateDataUtils.garbageCollectionTime(directContext.window(), windowingStrategy); if (gc.isAfter(eow)) { directContext.timers().deleteTimer(gc, TimeDomain.EVENT_TIME); } }
@Test public void garbageCollectionTimeAfterEndOfGlobalWindow() { FixedWindows windowFn = FixedWindows.of(Duration.standardMinutes(5)); WindowingStrategy<?, ?> strategy = WindowingStrategy.globalDefault().withWindowFn(windowFn); IntervalWindow window = windowFn.assignWindow(new Instant(BoundedWindow.TIMESTAMP_MAX_VALUE)); assertThat(window.maxTimestamp(), equalTo(GlobalWindow.INSTANCE.maxTimestamp())); assertThat( LateDataUtils.garbageCollectionTime(window, strategy), equalTo(GlobalWindow.INSTANCE.maxTimestamp())); }
@Test public void beforeEndOfGlobalWindowSame() { FixedWindows windowFn = FixedWindows.of(Duration.standardMinutes(5)); Duration allowedLateness = Duration.standardMinutes(2); WindowingStrategy<?, ?> strategy = WindowingStrategy.globalDefault() .withWindowFn(windowFn) .withAllowedLateness(allowedLateness); IntervalWindow window = windowFn.assignWindow(new Instant(10)); assertThat( LateDataUtils.garbageCollectionTime(window, strategy), equalTo(window.maxTimestamp().plus(allowedLateness))); }
@Test public void garbageCollectionTimeAfterEndOfGlobalWindowWithLateness() { FixedWindows windowFn = FixedWindows.of(Duration.standardMinutes(5)); Duration allowedLateness = Duration.millis(Long.MAX_VALUE); WindowingStrategy<?, ?> strategy = WindowingStrategy.globalDefault() .withWindowFn(windowFn) .withAllowedLateness(allowedLateness); IntervalWindow window = windowFn.assignWindow(new Instant(-100)); assertThat( window.maxTimestamp().plus(allowedLateness), Matchers.greaterThan(GlobalWindow.INSTANCE.maxTimestamp())); assertThat( LateDataUtils.garbageCollectionTime(window, strategy), equalTo(GlobalWindow.INSTANCE.maxTimestamp())); } }
Instant outputWM = timerInternals.currentOutputWatermarkTime(); Instant inputWM = timerInternals.currentInputWatermarkTime(); Instant gcHold = LateDataUtils.garbageCollectionTime(context.window(), windowingStrategy);
"Unexpected zero getAllowedLateness"); Instant cleanupTime = LateDataUtils.garbageCollectionTime(directContext.window(), windowingStrategy); WindowTracing.debug( "ReduceFnRunner.onTimer: Scheduling cleanup timer for key:{}; window:{} at {} with "
LateDataUtils.garbageCollectionTime(directContext.window(), windowingStrategy)), "new hold %s should be at garbage collection for window %s plus %s", newHold,