@Override public void track(Event<T> event) { if (started && !event.isWatermark()) { if (currentCount.incrementAndGet() >= count) { evictionPolicy.setContext(new DefaultEvictionContext(System.currentTimeMillis())); handler.onTrigger(); } } }
@Override public boolean hasNext() { while (!stopped && windowEvent == null && inner.hasNext()) { Event<T> cur = inner.next(); EvictionPolicy.Action action = evictionPolicy.evict(cur); if (action == EXPIRE) { inner.remove(); } else if (action == STOP) { stopped = true; } else if (action == PROCESS) { windowEvent = cur.get(); } } return windowEvent != null; }
/** * feed the event to the eviction and trigger policies for bookkeeping and optionally firing the trigger. */ private void track(Event<T> windowEvent) { evictionPolicy.track(windowEvent); triggerPolicy.track(windowEvent); }
prevWindowEvents.addAll(windowEvents); LOG.debug("invoking windowLifecycleListener onActivation, [{}] events in window.", events.size()); windowLifecycleListener.onActivation(events, newEvents, expired, evictionPolicy.getContext().getReferenceTime()); } else { LOG.debug("No events in the window, skipping onActivation");
public void restoreState(Map<String, Optional<?>> state) { Optional.ofNullable(state.get(EVICTION_STATE_KEY)) .flatMap(x -> x) .ifPresent(v -> ((EvictionPolicy) evictionPolicy).restoreState(v)); Optional.ofNullable(state.get(TRIGGER_STATE_KEY)) .flatMap(x -> x) .ifPresent(v -> ((TriggerPolicy) triggerPolicy).restoreState(v)); }
private Iterator<T> scanEventsStateful() { LOG.debug("Scan events, eviction policy {}", evictionPolicy); evictionPolicy.reset(); Iterator<T> it = new Iterator<T>() { private Iterator<Event<T>> inner = queue.iterator();
public Map<String, Optional<?>> getState() { return ImmutableMap.of( EVICTION_STATE_KEY, Optional.ofNullable(evictionPolicy.getState()), TRIGGER_STATE_KEY, Optional.ofNullable(triggerPolicy.getState()) ); } }
windowLifecycleListener.onActivation(wrapper, null, null, evictionPolicy.getContext().getReferenceTime());
@Override public void run() { // do not process current timestamp since tuples might arrive while the trigger is executing long now = System.currentTimeMillis() - 1; try { /* * set the current timestamp as the reference time for the eviction policy * to evict the events */ evictionPolicy.setContext(new DefaultEvictionContext(now, null, null, duration)); handler.onTrigger(); } catch (Throwable th) { LOG.error("handler.onTrigger failed ", th); /* * propagate it so that task gets canceled and the exception * can be retrieved from executorFuture.get() */ throw th; } } };
while (it.hasNext()) { Event<T> windowEvent = it.next(); Action action = evictionPolicy.evict(windowEvent); if (action == EXPIRE) { eventsToExpire.add(windowEvent.get());
/** * feed the event to the eviction and trigger policies * for bookkeeping and optionally firing the trigger. */ private void track(Event<T> windowEvent) { evictionPolicy.track(windowEvent); triggerPolicy.track(windowEvent); }
/** * Triggers all the pending windows up to the waterMarkEvent timestamp based on the sliding interval count. * * @param waterMarkEvent the watermark event */ private void handleWaterMarkEvent(Event<T> waterMarkEvent) { long watermarkTs = waterMarkEvent.getTimestamp(); List<Long> eventTs = windowManager.getSlidingCountTimestamps(lastProcessedTs, watermarkTs, count); for (long ts : eventTs) { evictionPolicy.setContext(new DefaultEvictionContext(ts, null, Long.valueOf(count))); handler.onTrigger(); lastProcessedTs = ts; } }
while (it.hasNext()) { Event<T> windowEvent = it.next(); Action action = evictionPolicy.evict(windowEvent); if (action == EXPIRE) { eventsToExpire.add(windowEvent.get());
/** * Invokes the trigger all pending windows up to the watermark timestamp. The end ts of the window is set in the eviction policy context * so that the events falling within that window can be processed. */ private void handleWaterMarkEvent(Event<T> event) { long watermarkTs = event.getTimestamp(); long windowEndTs = nextWindowEndTs; LOG.debug("Window end ts {} Watermark ts {}", windowEndTs, watermarkTs); while (windowEndTs <= watermarkTs) { long currentCount = windowManager.getEventCount(windowEndTs); evictionPolicy.setContext(new DefaultEvictionContext(windowEndTs, currentCount)); if (handler.onTrigger()) { windowEndTs += slidingIntervalMs; } else { /* * No events were found in the previous window interval. * Scan through the events in the queue to find the next * window intervals based on event ts. */ long ts = getNextAlignedWindowTs(windowEndTs, watermarkTs); LOG.debug("Next aligned window end ts {}", ts); if (ts == Long.MAX_VALUE) { LOG.debug("No events to process between {} and watermark ts {}", windowEndTs, watermarkTs); break; } windowEndTs = ts; } } nextWindowEndTs = windowEndTs; }
evictionPolicy.setContext(new DefaultEvictionContext(now + 60)); windowManager.onTrigger(); listener.clear(); evictionPolicy.setContext(new DefaultEvictionContext(now + 120)); windowManager.onTrigger(); assertTrue(listener.onActivationEvents.isEmpty()); listener.clear(); evictionPolicy.setContext(new DefaultEvictionContext(now + 180)); windowManager.onTrigger(); assertTrue(listener.onActivationExpiredEvents.isEmpty());
evictionPolicy.setContext(new DefaultEvictionContext(now + 100)); windowManager.onTrigger(); evictionPolicy.setContext(new DefaultEvictionContext(now + 200)); windowManager.onTrigger(); assertTrue(listener.onExpiryEvents.isEmpty());
@Override public void track(Event<T> event) { if (started && !event.isWatermark()) { if (currentCount.incrementAndGet() >= count) { evictionPolicy.setContext(new DefaultEvictionContext(System.currentTimeMillis())); handler.onTrigger(); } } }
@Override public void run() { // do not process current timestamp since tuples might arrive while the trigger is executing long now = System.currentTimeMillis() - 1; try { /* * set the current timestamp as the reference time for the eviction policy * to evict the events */ if (evictionPolicy != null) { evictionPolicy.setContext(new DefaultEvictionContext(now, null, null, duration)); } handler.onTrigger(); } catch (Throwable th) { LOG.error("handler.onTrigger failed ", th); /* * propagate it so that task gets canceled and the exception * can be retrieved from executorFuture.get() */ throw th; } } };
/** * Triggers all the pending windows up to the waterMarkEvent timestamp * based on the sliding interval count. * * @param waterMarkEvent the watermark event */ private void handleWaterMarkEvent(Event<T> waterMarkEvent) { long watermarkTs = waterMarkEvent.getTimestamp(); List<Long> eventTs = windowManager.getSlidingCountTimestamps(lastProcessedTs, watermarkTs, count); for (long ts : eventTs) { evictionPolicy.setContext(new DefaultEvictionContext(ts, null, Long.valueOf(count))); handler.onTrigger(); lastProcessedTs = ts; } }
while (windowEndTs <= watermarkTs) { long currentCount = windowManager.getEventCount(windowEndTs); evictionPolicy.setContext(new DefaultEvictionContext(windowEndTs, currentCount)); if (handler.onTrigger()) { windowEndTs += slidingIntervalMs;