public AbstractTridentWindowManager(WindowConfig windowConfig, String windowTaskId, WindowsStore windowStore, Aggregator aggregator, BatchOutputCollector delegateCollector) { this.windowTaskId = windowTaskId; this.windowStore = windowStore; this.aggregator = aggregator; this.delegateCollector = delegateCollector; windowTriggerCountId = WindowTridentProcessor.TRIGGER_COUNT_PREFIX + windowTaskId; windowManager = new WindowManager<>(new TridentWindowLifeCycleListener()); WindowStrategy<T> windowStrategy = windowConfig.getWindowStrategy(); EvictionPolicy<T, ?> evictionPolicy = windowStrategy.getEvictionPolicy(); windowManager.setEvictionPolicy(evictionPolicy); triggerPolicy = windowStrategy.getTriggerPolicy(windowManager, evictionPolicy); windowManager.setTriggerPolicy(triggerPolicy); }
/** * Add an event into the window, with {@link System#currentTimeMillis()} as the tracking ts. * * @param event the event to add */ public void add(T event) { add(event, System.currentTimeMillis()); }
public void shutdown() { try { LOG.info("window manager [{}] is being shutdown", windowManager); windowManager.shutdown(); } finally { LOG.info("window store [{}] is being shutdown", windowStore); windowStore.shutdown(); } }
/** * Tracks a window event * * @param windowEvent the window event to track */ public void add(Event<T> windowEvent) { // watermark events are not added to the queue. if (!windowEvent.isWatermark()) { queue.add(windowEvent); } else { LOG.debug("Got watermark event with ts {}", windowEvent.getTimestamp()); } track(windowEvent); compactWindow(); }
@Test public void testEventTimeLag() throws Exception { EvictionPolicy<Integer, ?> evictionPolicy = new WatermarkTimeEvictionPolicy<>(20, 5); windowManager.setEvictionPolicy(evictionPolicy); TriggerPolicy<Integer, ?> triggerPolicy = new WatermarkTimeTriggerPolicy<Integer>(10, windowManager, evictionPolicy, windowManager); triggerPolicy.start(); windowManager.setTriggerPolicy(triggerPolicy); windowManager.add(1, 603); windowManager.add(2, 605); windowManager.add(3, 607); windowManager.add(4, 618); windowManager.add(5, 626); windowManager.add(6, 632); windowManager.add(7, 629); windowManager.add(8, 636); // send a watermark event, which should trigger three windows. windowManager.add(new WaterMarkEvent<Integer>(631)); // System.out.println(listener.allOnActivationEvents); assertEquals(3, listener.allOnActivationEvents.size()); assertEquals(seq(1, 3), listener.allOnActivationEvents.get(0)); assertEquals(seq(1, 4), listener.allOnActivationEvents.get(1)); // out of order events should be processed upto the lag assertEquals(Arrays.asList(4, 5, 7), listener.allOnActivationEvents.get(2)); }
@Test public void testTimeBasedWindowExpiry() throws Exception { EvictionPolicy<Integer, ?> evictionPolicy = new TimeEvictionPolicy<Integer>(new Duration(100, TimeUnit.MILLISECONDS).value); windowManager.setEvictionPolicy(evictionPolicy); windowManager.setTriggerPolicy(triggerPolicy); long now = System.currentTimeMillis(); windowManager.add(i); windowManager.onTrigger(); windowManager.onTrigger(); listener.clear(); evictionPolicy.setContext(new DefaultEvictionContext(now + 180)); windowManager.onTrigger(); assertTrue(listener.onActivationExpiredEvents.isEmpty()); assertTrue(listener.onActivationEvents.isEmpty());
@Before public void setUp() { listener = new Listener(); windowManager = new WindowManager<>(listener); }
/** * expires events that fall out of the window every EXPIRE_EVENTS_THRESHOLD so that the window does not grow too big. */ protected void compactWindow() { if (eventsSinceLastExpiry.incrementAndGet() >= EXPIRE_EVENTS_THRESHOLD) { scanEvents(false); } }
/** * Triggers all the pending windows up to the waterMarkEvent timestamp based on the sliding interval count. * * @param waterMarkEvent the watermark event */ private void handleWaterMarkEvent(Event<T> waterMarkEvent) { long watermarkTs = waterMarkEvent.getTimestamp(); List<Long> eventTs = windowManager.getSlidingCountTimestamps(lastProcessedTs, watermarkTs, count); for (long ts : eventTs) { evictionPolicy.setContext(new DefaultEvictionContext(ts, null, Long.valueOf(count))); handler.onTrigger(); lastProcessedTs = ts; } }
/** * Computes the next window by scanning the events in the window and finds the next aligned window between the startTs and endTs. Return * the end ts of the next aligned window, i.e. the ts when the window should fire. * * @param startTs the start timestamp (excluding) * @param endTs the end timestamp (including) * @return the aligned window end ts for the next window or Long.MAX_VALUE if there are no more events to be processed. */ private long getNextAlignedWindowTs(long startTs, long endTs) { long nextTs = windowManager.getEarliestEventTs(startTs, endTs); if (nextTs == Long.MAX_VALUE || (nextTs % slidingIntervalMs == 0)) { return nextTs; } return nextTs + (slidingIntervalMs - (nextTs % slidingIntervalMs)); }
/** * Invokes the trigger all pending windows up to the watermark timestamp. The end ts of the window is set in the eviction policy context * so that the events falling within that window can be processed. */ private void handleWaterMarkEvent(Event<T> event) { long watermarkTs = event.getTimestamp(); long windowEndTs = nextWindowEndTs; LOG.debug("Window end ts {} Watermark ts {}", windowEndTs, watermarkTs); while (windowEndTs <= watermarkTs) { long currentCount = windowManager.getEventCount(windowEndTs); evictionPolicy.setContext(new DefaultEvictionContext(windowEndTs, currentCount)); if (handler.onTrigger()) { windowEndTs += slidingIntervalMs; } else { /* * No events were found in the previous window interval. * Scan through the events in the queue to find the next * window intervals based on event ts. */ long ts = getNextAlignedWindowTs(windowEndTs, watermarkTs); LOG.debug("Next aligned window end ts {}", ts); if (ts == Long.MAX_VALUE) { LOG.debug("No events to process between {} and watermark ts {}", windowEndTs, watermarkTs); break; } windowEndTs = ts; } } nextWindowEndTs = windowEndTs; }
windowManager.setEvictionPolicy(evictionPolicy); TriggerPolicy<Integer, ?> triggerPolicy = new WatermarkTimeTriggerPolicy<Integer>(10, windowManager, evictionPolicy, windowManager); triggerPolicy.start(); windowManager.setTriggerPolicy(triggerPolicy); windowManager.add(1, 603); windowManager.add(2, 605); windowManager.add(3, 607); windowManager.add(4, 618); windowManager.add(5, 626); windowManager.add(6, 629); windowManager.add(7, 636); windowManager.add(8, 637); windowManager.add(9, 638); windowManager.add(10, 639); windowManager.add(new WaterMarkEvent<Integer>(631));
@Test public void testTimeBasedWindow() throws Exception { EvictionPolicy<Integer, ?> evictionPolicy = new TimeEvictionPolicy<Integer>(new Duration(1, TimeUnit.SECONDS).value); windowManager.setEvictionPolicy(evictionPolicy); new TimeTriggerPolicy<Integer>(new Duration(1, TimeUnit.DAYS).value, windowManager, evictionPolicy); triggerPolicy.start(); windowManager.setTriggerPolicy(triggerPolicy); long now = System.currentTimeMillis(); windowManager.add(i, now - 1000); windowManager.add(i, now); windowManager.add(i, now - 1000); windowManager.onTrigger(); windowManager.add(i, now); windowManager.onTrigger(); assertTrue(listener.onExpiryEvents.isEmpty()); assertEquals(activationsEvents, listener.onActivationEvents);
/** * Tracks a window event * * @param windowEvent the window event to track */ public void add(Event<T> windowEvent) { // watermark events are not added to the queue. if (!windowEvent.isWatermark()) { queue.add(windowEvent); } else { LOG.debug("Got watermark event with ts {}", windowEvent.getTimestamp()); } track(windowEvent); compactWindow(); }
windowEvents = scanEvents(true); expired = new ArrayList<>(expiredEvents); expiredEvents.clear();
/** * Triggers all the pending windows up to the waterMarkEvent timestamp * based on the sliding interval count. * * @param waterMarkEvent the watermark event */ private void handleWaterMarkEvent(Event<T> waterMarkEvent) { long watermarkTs = waterMarkEvent.getTimestamp(); List<Long> eventTs = windowManager.getSlidingCountTimestamps(lastProcessedTs, watermarkTs, count); for (long ts : eventTs) { evictionPolicy.setContext(new DefaultEvictionContext(ts, null, Long.valueOf(count))); handler.onTrigger(); lastProcessedTs = ts; } }
/** * Computes the next window by scanning the events in the window and * finds the next aligned window between the startTs and endTs. Return the end ts * of the next aligned window, i.e. the ts when the window should fire. * * @param startTs the start timestamp (excluding) * @param endTs the end timestamp (including) * @return the aligned window end ts for the next window or Long.MAX_VALUE if there * are no more events to be processed. */ private long getNextAlignedWindowTs(long startTs, long endTs) { long nextTs = windowManager.getEarliestEventTs(startTs, endTs); if (nextTs == Long.MAX_VALUE || (nextTs % slidingIntervalMs == 0)) { return nextTs; } return nextTs + (slidingIntervalMs - (nextTs % slidingIntervalMs)); }
LOG.debug("Window end ts {} Watermark ts {}", windowEndTs, watermarkTs); while (windowEndTs <= watermarkTs) { long currentCount = windowManager.getEventCount(windowEndTs); evictionPolicy.setContext(new DefaultEvictionContext(windowEndTs, currentCount)); if (handler.onTrigger()) {
@Test public void testCountBasedTumblingWithSameEventTs() throws Exception { EvictionPolicy<Integer, ?> evictionPolicy = new WatermarkCountEvictionPolicy<>(2); windowManager.setEvictionPolicy(evictionPolicy); TriggerPolicy<Integer, ?> triggerPolicy = new WatermarkCountTriggerPolicy<Integer>(2, windowManager, evictionPolicy, windowManager); triggerPolicy.start(); windowManager.setTriggerPolicy(triggerPolicy); windowManager.add(1, 10); windowManager.add(2, 10); windowManager.add(3, 11); windowManager.add(4, 12); windowManager.add(5, 12); windowManager.add(6, 12); windowManager.add(7, 12); windowManager.add(8, 13); windowManager.add(9, 14); windowManager.add(10, 15); windowManager.add(new WaterMarkEvent<Integer>(20)); assertEquals(5, listener.allOnActivationEvents.size()); assertEquals(seq(1, 2), listener.allOnActivationEvents.get(0)); assertEquals(seq(3, 4), listener.allOnActivationEvents.get(1)); assertEquals(seq(5, 6), listener.allOnActivationEvents.get(2)); assertEquals(seq(7, 8), listener.allOnActivationEvents.get(3)); assertEquals(seq(9, 10), listener.allOnActivationEvents.get(4)); }
: new WindowManager<>(lifecycleListener, queue); triggerPolicy = getTriggerPolicy(slidingIntervalCount, slidingIntervalDuration, manager, evictionPolicy); manager.setEvictionPolicy(evictionPolicy); manager.setTriggerPolicy(triggerPolicy); return manager;