/** * Scans the event queue and returns number of events having timestamp less than or equal to the reference time. * * @param referenceTime the reference timestamp in millis * @return the count of events with timestamp less than or equal to referenceTime */ public int getEventCount(long referenceTime) { int count = 0; for (Event<T> event : queue) { if (event.getTimestamp() <= referenceTime) { ++count; } } return count; }
/** * Scans the event queue and returns the next earliest event ts between the startTs and endTs * * @param startTs the start ts (exclusive) * @param endTs the end ts (inclusive) * @return the earliest event ts between startTs and endTs */ public long getEarliestEventTs(long startTs, long endTs) { long minTs = Long.MAX_VALUE; for (Event<T> event : queue) { if (event.getTimestamp() > startTs && event.getTimestamp() <= endTs) { minTs = Math.min(minTs, event.getTimestamp()); } } return minTs; }
/** * Scans the event queue and returns the list of event ts falling between startTs (exclusive) and endTs (inclusive) at each sliding * interval counts. * * @param startTs the start timestamp (exclusive) * @param endTs the end timestamp (inclusive) * @param slidingCount the sliding interval count * @return the list of event ts */ public List<Long> getSlidingCountTimestamps(long startTs, long endTs, int slidingCount) { List<Long> timestamps = new ArrayList<>(); if (endTs > startTs) { int count = 0; long ts = Long.MIN_VALUE; for (Event<T> event : queue) { if (event.getTimestamp() > startTs && event.getTimestamp() <= endTs) { ts = Math.max(ts, event.getTimestamp()); if (++count % slidingCount == 0) { timestamps.add(ts); } } } } return timestamps; }
/** * {@inheritDoc} */ @Override public Action evict(Event<T> event) { long now = evictionContext == null ? System.currentTimeMillis() : evictionContext.getReferenceTime(); long diff = now - event.getTimestamp(); if (diff >= (windowLength + delta)) { return Action.EXPIRE; } else if (diff < 0) { // do not process events beyond current ts return Action.KEEP; } return Action.PROCESS; }
/** * Tracks a window event * * @param windowEvent the window event to track */ public void add(Event<T> windowEvent) { // watermark events are not added to the queue. if (!windowEvent.isWatermark()) { queue.add(windowEvent); } else { LOG.debug("Got watermark event with ts {}", windowEvent.getTimestamp()); } track(windowEvent); compactWindow(); }
/** * {@inheritDoc} * <p/> * Keeps events with future ts in the queue for processing in the next window. If the ts difference is more than the lag, stops scanning * the queue for the current window. */ @Override public Action evict(Event<T> event) { if (evictionContext == null) { //It is possible to get asked about eviction before we have a context, due to WindowManager.compactWindow. //In this case we should hold on to all the events. When the first watermark is received, the context will be set, //and the events will be reevaluated for eviction return Action.STOP; } long referenceTime = evictionContext.getReferenceTime(); long diff = referenceTime - event.getTimestamp(); if (diff < -lag) { return Action.STOP; } else if (diff < 0) { return Action.KEEP; } else { return super.evict(event); } }
/** * Triggers all the pending windows up to the waterMarkEvent timestamp based on the sliding interval count. * * @param waterMarkEvent the watermark event */ private void handleWaterMarkEvent(Event<T> waterMarkEvent) { long watermarkTs = waterMarkEvent.getTimestamp(); List<Long> eventTs = windowManager.getSlidingCountTimestamps(lastProcessedTs, watermarkTs, count); for (long ts : eventTs) { evictionPolicy.setContext(new DefaultEvictionContext(ts, null, Long.valueOf(count))); handler.onTrigger(); lastProcessedTs = ts; } }
public Action evict(Event<T> event) { if (getContext() == null) { //It is possible to get asked about eviction before we have a context, due to WindowManager.compactWindow. //In this case we should hold on to all the events. When the first watermark is received, the context will be set, //and the events will be reevaluated for eviction return Action.STOP; } Action action; if (event.getTimestamp() <= getContext().getReferenceTime() && processed < currentCount.get()) { action = doEvict(event); if (action == Action.PROCESS) { ++processed; } } else { action = Action.KEEP; } return action; }
/** * Invokes the trigger all pending windows up to the watermark timestamp. The end ts of the window is set in the eviction policy context * so that the events falling within that window can be processed. */ private void handleWaterMarkEvent(Event<T> event) { long watermarkTs = event.getTimestamp(); long windowEndTs = nextWindowEndTs; LOG.debug("Window end ts {} Watermark ts {}", windowEndTs, watermarkTs); while (windowEndTs <= watermarkTs) { long currentCount = windowManager.getEventCount(windowEndTs); evictionPolicy.setContext(new DefaultEvictionContext(windowEndTs, currentCount)); if (handler.onTrigger()) { windowEndTs += slidingIntervalMs; } else { /* * No events were found in the previous window interval. * Scan through the events in the queue to find the next * window intervals based on event ts. */ long ts = getNextAlignedWindowTs(windowEndTs, watermarkTs); LOG.debug("Next aligned window end ts {}", ts); if (ts == Long.MAX_VALUE) { LOG.debug("No events to process between {} and watermark ts {}", windowEndTs, watermarkTs); break; } windowEndTs = ts; } } nextWindowEndTs = windowEndTs; }
@Test public void testTrackSingleStream() throws Exception { waterMarkEventGenerator.track(streamId("s1"), 100); waterMarkEventGenerator.track(streamId("s1"), 110); waterMarkEventGenerator.run(); assertTrue(eventList.get(0).isWatermark()); assertEquals(105, eventList.get(0).getTimestamp()); }
@Test public void testTrackSingleStreamOutOfOrder() throws Exception { waterMarkEventGenerator.track(streamId("s1"), 100); waterMarkEventGenerator.track(streamId("s1"), 110); waterMarkEventGenerator.track(streamId("s1"), 104); waterMarkEventGenerator.run(); assertTrue(eventList.get(0).isWatermark()); assertEquals(105, eventList.get(0).getTimestamp()); }
@Test public void testTrackTwoStreams() throws Exception { Set<GlobalStreamId> streams = new HashSet<>(); streams.add(streamId("s1")); streams.add(streamId("s2")); waterMarkEventGenerator = new WaterMarkEventGenerator<>(windowManager, 100000, 5, streams); waterMarkEventGenerator.track(streamId("s1"), 100); waterMarkEventGenerator.track(streamId("s1"), 110); waterMarkEventGenerator.run(); assertTrue(eventList.isEmpty()); waterMarkEventGenerator.track(streamId("s2"), 95); waterMarkEventGenerator.track(streamId("s2"), 98); waterMarkEventGenerator.run(); assertTrue(eventList.get(0).isWatermark()); assertEquals(93, eventList.get(0).getTimestamp()); }
@Test public void testLateEvent() throws Exception { assertTrue(waterMarkEventGenerator.track(streamId("s1"), 100)); assertTrue(waterMarkEventGenerator.track(streamId("s1"), 110)); waterMarkEventGenerator.run(); assertTrue(eventList.get(0).isWatermark()); assertEquals(105, eventList.get(0).getTimestamp()); eventList.clear(); assertTrue(waterMarkEventGenerator.track(streamId("s1"), 105)); assertTrue(waterMarkEventGenerator.track(streamId("s1"), 106)); assertTrue(waterMarkEventGenerator.track(streamId("s1"), 115)); assertFalse(waterMarkEventGenerator.track(streamId("s1"), 104)); waterMarkEventGenerator.run(); assertTrue(eventList.get(0).isWatermark()); assertEquals(110, eventList.get(0).getTimestamp()); } }
/** * Scans the event queue and returns number of events having * timestamp less than or equal to the reference time. * * @param referenceTime the reference timestamp in millis * @return the count of events with timestamp less than or equal to referenceTime */ public int getEventCount(long referenceTime) { int count = 0; for (Event<T> event : queue) { if (event.getTimestamp() <= referenceTime) { ++count; } } return count; }
/** * Scans the event queue and returns the next earliest event ts * between the startTs and endTs * * @param startTs the start ts (exclusive) * @param endTs the end ts (inclusive) * @return the earliest event ts between startTs and endTs */ public long getEarliestEventTs(long startTs, long endTs) { long minTs = Long.MAX_VALUE; for (Event<T> event : queue) { if (event.getTimestamp() > startTs && event.getTimestamp() <= endTs) { minTs = Math.min(minTs, event.getTimestamp()); } } return minTs; }
/** * {@inheritDoc} */ @Override public Action evict(Event<T> event) { long now = referenceTime == null ? System.currentTimeMillis() : referenceTime; long diff = now - event.getTimestamp(); if (diff >= (windowLength + delta)) { return Action.EXPIRE; } else if (diff < 0) { // do not process events beyond current ts return Action.KEEP; } return Action.PROCESS; }
@Override public Action evict(Event<T> event) { if(context == null) { //It is possible to get asked about eviction before we have a context, due to WindowManager.compactWindow. //In this case we should hold on to all the events. When the first watermark is received, the context will be set, //and the events will be reevaluated for eviction return Action.STOP; } Action action; if (event.getTimestamp() <= referenceTime && processed < currentCount.get()) { action = super.evict(event); if (action == Action.PROCESS) { ++processed; } } else { action = Action.KEEP; } return action; }
/** * Tracks a window event * * @param windowEvent the window event to track */ public void add(Event<T> windowEvent) { // watermark events are not added to the queue. if (!windowEvent.isWatermark()) { queue.add(windowEvent); } else { LOG.debug("Got watermark event with ts {}", windowEvent.getTimestamp()); } track(windowEvent); compactWindow(); }
/** * {@inheritDoc} * <p/> * Keeps events with future ts in the queue for processing in the next * window. If the ts difference is more than the lag, stops scanning * the queue for the current window. */ @Override public Action evict(Event<T> event) { if(evictionContext == null) { //It is possible to get asked about eviction before we have a context, due to WindowManager.compactWindow. //In this case we should hold on to all the events. When the first watermark is received, the context will be set, //and the events will be reevaluated for eviction return Action.STOP; } long referenceTime = evictionContext.getReferenceTime(); long diff = referenceTime - event.getTimestamp(); if (diff < -lag) { return Action.STOP; } else if (diff < 0) { return Action.KEEP; } else { return super.evict(event); } }
/** * Triggers all the pending windows up to the waterMarkEvent timestamp * based on the sliding interval count. * * @param waterMarkEvent the watermark event */ private void handleWaterMarkEvent(Event<T> waterMarkEvent) { long watermarkTs = waterMarkEvent.getTimestamp(); List<Long> eventTs = windowManager.getSlidingCountTimestamps(lastProcessedTs, watermarkTs, count); for (long ts : eventTs) { evictionPolicy.setContext(new DefaultEvictionContext(ts, null, Long.valueOf(count))); handler.onTrigger(); lastProcessedTs = ts; } }