@Override public long checkWmHistory() { if (idleMessagePending) { idleMessagePending = false; return IDLE_MESSAGE.timestamp(); } return NO_NEW_WM; }
@Override public boolean tryProcessWatermark(@Nonnull Watermark watermark) { assert lastEmittedWm <= lastReceivedWm : "lastEmittedWm=" + lastEmittedWm + ", lastReceivedWm=" + lastReceivedWm; // Ignore a watermark that is going back. This is possible after restoring from a snapshot // taken in at-least-once mode. if (watermark.timestamp() <= lastReceivedWm) { return true; } lastReceivedWm = watermark.timestamp(); if (watermarkCounts.isEmpty()) { lastEmittedWm = watermark.timestamp(); return tryEmit(watermark); } return true; }
@Override public boolean tryProcessWatermark(@Nonnull Watermark wm) { currentWatermark = wm.timestamp(); assert totalWindows.get() == deadlineToKeys.values().stream().mapToInt(Set::size).sum() : "unexpected totalWindows. Expected=" + deadlineToKeys.values().stream().mapToInt(Set::size).sum() + ", actual=" + totalWindows.get(); return closedWindowFlatmapper.tryProcess(wm); }
@Override public void write(ObjectDataOutput out, Watermark object) throws IOException { out.writeLong(object.timestamp()); }
@Override public long observeWm(int queueIndex, long wmValue) { assert queueIndex == 0 : "queueIndex=" + queueIndex; if (queueWm.get() >= wmValue) { throw new JetException("Watermarks not monotonically increasing on queue: " + "last one=" + queueWm + ", new one=" + wmValue); } if (wmValue != IDLE_MESSAGE.timestamp()) { queueWm.lazySet(wmValue); } return wmValue; }
private Traverser<Object> traverseClosedWindows(Watermark wm) { SortedMap<Long, Set<K>> windowsToClose = deadlineToKeys.headMap(wm.timestamp()); Stream<Object> closedWindows = windowsToClose .values().stream() .flatMap(Set::stream) .map(key -> closeWindows(keyToWindows.get(key), key, wm.timestamp())) .flatMap(List::stream); Traverser<Object> result = traverseStream(closedWindows) .onFirstNull(() -> { lazyAdd(totalWindows, -windowsToClose.values().stream().mapToInt(Set::size).sum()); windowsToClose.clear(); }); if (wm != COMPLETING_WM) { result = result.append(wm); } return result; }
? IDLE_MESSAGE.timestamp() : NO_NEW_WM;
@Override public long observeWm(int queueIndex, long wmValue) { if (queueWms[queueIndex] >= wmValue) { throw new JetException("Watermarks not monotonically increasing on queue: " + "last one=" + queueWms[queueIndex] + ", new one=" + wmValue); } if (wmValue == IDLE_MESSAGE.timestamp()) { isIdle[queueIndex] = true; return checkObservedWms(); } else { isIdle[queueIndex] = false; allInputsAreIdle = false; queueWms[queueIndex] = wmValue; if (wmValue > topObservedWm.get()) { topObservedWm.lazySet(wmValue); } return checkObservedWms(); } }
"or AbstractProcessor.tryEmit() returned false"; if (item instanceof Watermark) { lastForwardedWm.lazySet(((Watermark) item).timestamp());
long newWmValue = ((Watermark) inbox.queue().removeLast()).timestamp(); long wm = watermarkCoalescer.observeWm(currInstream.ordinal(), newWmValue); if (wm != NO_NEW_WM) { assert pendingWatermark == null || pendingWatermark.timestamp() < wm : "trying to assign lower WM. Old=" + pendingWatermark.timestamp() + ", new=" + wm; pendingWatermark = new Watermark(wm);
long wmTimestamp = ((Watermark) itemDetector.item).timestamp(); boolean forwarded = maybeEmitWm(watermarkCoalescer.observeWm(queueIndex, wmTimestamp), dest); if (logger.isFinestEnabled()) {