triggerRunner.prefetchIsClosed(directContext.state()); } else { triggerRunner.prefetchShouldFire(directContext.window(), directContext.state()); prefetchOnTrigger(directContext, timer.renamedContext); } else if (triggerRunner.shouldFire( directContext.window(), directContext.timers(), directContext.state())) { prefetchEmit(directContext, timer.renamedContext); ReduceFnRunner.class.getSimpleName(), key, directContext.window(), timerInternals.currentInputWatermarkTime(), timerInternals.currentOutputWatermarkTime()); + "inputWatermark:{}; outputWatermark:{}", key, directContext.window(), timerInternals.currentInputWatermarkTime(), timerInternals.currentOutputWatermarkTime()); if (windowActivation.windowIsActiveAndOpen() && triggerRunner.shouldFire( directContext.window(), directContext.timers(), directContext.state())) { emit(directContext, renamedContext); "Unexpected zero getAllowedLateness"); Instant cleanupTime = LateDataUtils.garbageCollectionTime(directContext.window(), windowingStrategy); WindowTracing.debug(
final boolean isEmpty = nonEmptyPanes.isEmpty(renamedContext.state()).read(); if (isEmpty && windowingStrategy.getClosingBehavior() == ClosingBehavior.FIRE_IF_NON_EMPTY checkState(!isFinished, "new hold at %s but finished %s", newHold, directContext.window()); if (newHold.isAfter(directContext.window().maxTimestamp())) { LateDataUtils.garbageCollectionTime(directContext.window(), windowingStrategy)), "new hold %s should be at garbage collection for window %s plus %s", newHold, directContext.window(), windowingStrategy.getAllowedLateness()); } else { newHold.isEqual(directContext.window().maxTimestamp()), "new hold %s should be at end of window %s", newHold, directContext.window()); checkState( !isEndOfWindow, "new hold at %s for %s but this is the watermark trigger", newHold, directContext.window()); final List<W> windows = Collections.singletonList(directContext.window()); ReduceFn<K, InputT, OutputT, W>.OnTriggerContext renamedTriggerContext = contextFactory.forTrigger( directContext.window(),
mergedWindow, contextFactory.base(mergedWindow, StateStyle.DIRECT).state());
"WatermarkHold.extractAndRelease: for key:{}; window:{}; inputWatermark:{}; " + "outputWatermark:{}", context.key(), context.window(), timerInternals.currentInputWatermarkTime(), timerInternals.currentOutputWatermarkTime()); final WatermarkHoldState elementHoldState = context.state().access(elementHoldTag); final WatermarkHoldState extraHoldState = context.state().access(EXTRA_HOLD_TAG); return new ReadableState<OldAndNewHolds>() { @Override
Instant outputWM = timerInternals.currentOutputWatermarkTime(); Instant inputWM = timerInternals.currentInputWatermarkTime(); Instant gcHold = LateDataUtils.garbageCollectionTime(context.window(), windowingStrategy); + "for key:{}; window: {}; inputWatermark: {}; outputWatermark: {}", getClass().getSimpleName(), context.key(), context.window(), inputWM, outputWM); && context.windowingStrategy().getClosingBehavior() == ClosingBehavior.FIRE_IF_NON_EMPTY) { WindowTracing.trace( "WatermarkHold.addGarbageCollectionHold: garbage collection hold at {} is unnecessary " + "outputWatermark:{}", gcHold, context.key(), context.window(), inputWM, outputWM); context.state().access(EXTRA_HOLD_TAG).add(gcHold); + "key:{}; window:{}; inputWatermark:{}; outputWatermark:{}", gcHold, context.key(), context.window(), inputWM,
Instant elementHold = shift(timestamp, context.window()); which = "too late to effect output watermark"; tooLate = true; } else if (context.window().maxTimestamp().isBefore(inputWM)) { which = "too late for end-of-window timer"; tooLate = true; "Element hold %s is beyond end-of-time", elementHold); context.state().access(elementHoldTag).add(elementHold); elementHold, which, context.key(), context.window(), inputWM, outputWM);
checkState( triggerRunner.shouldFire( directContext.window(), directContext.timers(), directContext.state())); triggerRunner.onFire(directContext.window(), directContext.timers(), directContext.state()); boolean isFinished = triggerRunner.isClosed(directContext.state()); nonEmptyPanes.clearPane(renamedContext.state()); directContext.window(), directContext.timers(), directContext.state()); paneInfoTracker.clear(directContext.state()); activeWindows.remove(directContext.window());
/** * Schedule a timer to garbage collect the window. * * <p>The timer: * * <ul> * <li>...must be fired strictly after the expiration of the window. * <li>...should be as close to the expiration as possible, to have a timely output of remaining * buffered data, and GC. * </ul> */ private void scheduleGarbageCollectionTimer(ReduceFn<?, ?, ?, W>.Context directContext) { Instant inputWM = timerInternals.currentInputWatermarkTime(); Instant gcTime = LateDataUtils.garbageCollectionTime(directContext.window(), windowingStrategy); WindowTracing.trace( "ReduceFnRunner.scheduleGarbageCollectionTimer: Scheduling at {} for " + "key:{}; window:{} where inputWatermark:{}; outputWatermark:{}", gcTime, key, directContext.window(), inputWM, timerInternals.currentOutputWatermarkTime()); checkState( !gcTime.isAfter(BoundedWindow.TIMESTAMP_MAX_VALUE), "Timer %s is beyond end-of-time", gcTime); directContext.timers().setTimer(gcTime, TimeDomain.EVENT_TIME); }
/** * Return a ({@link ReadableState} for) the pane info appropriate for {@code context}. The pane * info includes the timing for the pane, who's calculation is quite subtle. * * @param isFinal should be {@code true} only if the triggering machinery can guarantee no further * firings for the */ public ReadableState<PaneInfo> getNextPaneInfo( ReduceFn<?, ?, ?, ?>.Context context, final boolean isFinal) { final Object key = context.key(); final ReadableState<PaneInfo> previousPaneFuture = context.state().access(PaneInfoTracker.PANE_INFO_TAG); final Instant windowMaxTimestamp = context.window().maxTimestamp(); return new ReadableState<PaneInfo>() { @Override public ReadableState<PaneInfo> readLater() { previousPaneFuture.readLater(); return this; } @Override public PaneInfo read() { PaneInfo previousPane = previousPaneFuture.read(); return describePane(key, windowMaxTimestamp, previousPane, isFinal); } }; }
private void cancelEndOfWindowAndGarbageCollectionTimers( ReduceFn<?, ?, ?, W>.Context directContext) { WindowTracing.debug( "ReduceFnRunner.cancelEndOfWindowAndGarbageCollectionTimers: Deleting timers for " + "key:{}; window:{} where inputWatermark:{}; outputWatermark:{}", key, directContext.window(), timerInternals.currentInputWatermarkTime(), timerInternals.currentOutputWatermarkTime()); Instant eow = directContext.window().maxTimestamp(); directContext.timers().deleteTimer(eow, TimeDomain.EVENT_TIME); Instant gc = LateDataUtils.garbageCollectionTime(directContext.window(), windowingStrategy); if (gc.isAfter(eow)) { directContext.timers().deleteTimer(gc, TimeDomain.EVENT_TIME); } }
public void prefetchPaneInfo(ReduceFn<?, ?, ?, ?>.Context context) { context.state().access(PaneInfoTracker.PANE_INFO_TAG).readLater(); }
@Override public void clearState(Context c) throws Exception { c.state().access(bufferTag).clear(); }
public void storeCurrentPaneInfo(ReduceFn<?, ?, ?, ?>.Context context, PaneInfo currentPane) { context.state().access(PANE_INFO_TAG).write(currentPane); }
private Collection<W> windowsThatShouldFire(Set<W> windows) throws Exception { Collection<W> result = new ArrayList<>(); // Filter out timers that didn't trigger. for (W window : windows) { ReduceFn<K, InputT, OutputT, W>.Context directContext = contextFactory.base(window, StateStyle.DIRECT); if (triggerRunner.shouldFire( directContext.window(), directContext.timers(), directContext.state())) { result.add(window); } } return result; }
private Set<W> windowsThatAreOpen(Collection<W> windows) { Set<W> result = new HashSet<>(); for (W window : windows) { ReduceFn<K, InputT, OutputT, W>.Context directContext = contextFactory.base(window, StateStyle.DIRECT); if (!triggerRunner.isClosed(directContext.state())) { result.add(window); } } return result; }
@VisibleForTesting boolean isFinished(W window) { return triggerRunner.isClosed(contextFactory.base(window, StateStyle.DIRECT).state()); }
private void prefetchEmit( ReduceFn<K, InputT, OutputT, W>.Context directContext, ReduceFn<K, InputT, OutputT, W>.Context renamedContext) { triggerRunner.prefetchShouldFire(directContext.window(), directContext.state()); triggerRunner.prefetchOnFire(directContext.window(), directContext.state()); triggerRunner.prefetchIsClosed(directContext.state()); prefetchOnTrigger(directContext, renamedContext); }