/** * Returns the lowest frame timestamp greater than the given timestamp. If * there is no such {@code long} value, returns {@code Long.MAX_VALUE}. */ public long higherFrameTs(long timestamp) { long tsPlusFrame = timestamp + frameSize; return sumHadOverflow(timestamp, frameSize, tsPlusFrame) ? addClamped(floorFrameTs(timestamp), frameSize) : floorFrameTs(tsPlusFrame); }
@Override protected boolean tryProcess(int ordinal, @Nonnull Object item) { @SuppressWarnings("unchecked") final long frameTs = frameTimestampFns.get(ordinal).applyAsLong(item); assert frameTs == winPolicy.floorFrameTs(frameTs) : "getFrameTsFn returned an invalid frame timestamp"; // Ensure the event isn't late. We don't allow a "partially late" event: // one which belongs to some windows that are already emitted, even though // we still have the frame it belongs to. Such frames were already combined // into `slidingWindow` and we can't modify the value because that would // disturb the value that we'll deduct from `slidingWindow` later on. if (frameTs < nextWinToEmit) { logLateEvent(getLogger(), nextWinToEmit, item); lazyIncrement(lateEventsDropped); return true; } final K key = keyFns.get(ordinal).apply(item); A acc = tsToKeyToAcc .computeIfAbsent(frameTs, createMapPerTsFunction) .computeIfAbsent(key, createAccFunction); aggrOp.accumulateFn(ordinal).accept(acc, item); topTs = max(topTs, frameTs); return true; }
long newWm = watermarkThrottlingFrame != null ? watermarkThrottlingFrame.floorFrameTs(min) : Long.MIN_VALUE; if (newWm > lastEmittedWm) { traverser.append(new Watermark(newWm));
private Traverser<Object> windowTraverserAndEvictor(long wm) { long rangeStart; if (nextWinToEmit != Long.MIN_VALUE) { rangeStart = nextWinToEmit; } else { if (tsToKeyToAcc.isEmpty()) { // no item was observed, but initialize nextWinToEmit to the next window return Traversers.empty(); } // This is the first watermark we are acting upon. Find the lowest frame // timestamp that can be emitted: at most the top existing timestamp lower // than wm, but even lower than that if there are older frames on record. // The above guarantees that the sliding window can be correctly // initialized using the "add leading/deduct trailing" approach because we // start from a window that covers at most one existing frame -- the lowest // one on record. long bottomTs = tsToKeyToAcc .keySet().stream() .min(naturalOrder()) .orElseThrow(() -> new AssertionError("Failed to find the min key in a non-empty map")); rangeStart = min(bottomTs, winPolicy.floorFrameTs(wm)); } return traverseStream(range(rangeStart, wm, winPolicy.frameSize()).boxed()) .flatMap(winEnd -> traverseIterable(computeWindow(winEnd).entrySet()) .map(e -> mapToOutputFn.apply( winEnd - winPolicy.windowSize(), winEnd, e.getKey(), aggrOp.finishFn().apply(e.getValue()))) .onFirstNull(() -> completeWindow(winEnd))); }