@Override public boolean hasNext() { while (!stopped && windowEvent == null && inner.hasNext()) { Event<T> cur = inner.next(); EvictionPolicy.Action action = evictionPolicy.evict(cur); if (action == EXPIRE) { inner.remove(); } else if (action == STOP) { stopped = true; } else if (action == PROCESS) { windowEvent = cur.get(); } } return windowEvent != null; }
/** * Tracks a window event * * @param windowEvent the window event to track */ public void add(Event<T> windowEvent) { // watermark events are not added to the queue. if (!windowEvent.isWatermark()) { queue.add(windowEvent); } else { LOG.debug("Got watermark event with ts {}", windowEvent.getTimestamp()); } track(windowEvent); compactWindow(); }
holder.EventTitle.setHint(event.getEventTitle()); holder.EventDesc.setText(event.getEventDesc()); holder.EventStartTime.setText(event.getEventStartTime()); holder.EventEndTime.setText(event.getEventEndTime()); holder.EventID.setText(event.getEventID()); return event.get(selectedPosition);
/** * Scans the event queue and returns number of events having timestamp less than or equal to the reference time. * * @param referenceTime the reference timestamp in millis * @return the count of events with timestamp less than or equal to referenceTime */ public int getEventCount(long referenceTime) { int count = 0; for (Event<T> event : queue) { if (event.getTimestamp() <= referenceTime) { ++count; } } return count; }
@Override public void track(Event<T> event) { if (!event.isWatermark()) { currentCount.incrementAndGet(); } }
/** * Scans the event queue and returns the next earliest event ts between the startTs and endTs * * @param startTs the start ts (exclusive) * @param endTs the end ts (inclusive) * @return the earliest event ts between startTs and endTs */ public long getEarliestEventTs(long startTs, long endTs) { long minTs = Long.MAX_VALUE; for (Event<T> event : queue) { if (event.getTimestamp() > startTs && event.getTimestamp() <= endTs) { minTs = Math.min(minTs, event.getTimestamp()); } } return minTs; }
@Override public void track(Event<T> event) { if (started && event.isWatermark()) { handleWaterMarkEvent(event); } }
@Test public void testTrackSingleStream() throws Exception { waterMarkEventGenerator.track(streamId("s1"), 100); waterMarkEventGenerator.track(streamId("s1"), 110); waterMarkEventGenerator.run(); assertTrue(eventList.get(0).isWatermark()); assertEquals(105, eventList.get(0).getTimestamp()); }
/** * Scans the event queue and returns the list of event ts falling between startTs (exclusive) and endTs (inclusive) at each sliding * interval counts. * * @param startTs the start timestamp (exclusive) * @param endTs the end timestamp (inclusive) * @param slidingCount the sliding interval count * @return the list of event ts */ public List<Long> getSlidingCountTimestamps(long startTs, long endTs, int slidingCount) { List<Long> timestamps = new ArrayList<>(); if (endTs > startTs) { int count = 0; long ts = Long.MIN_VALUE; for (Event<T> event : queue) { if (event.getTimestamp() > startTs && event.getTimestamp() <= endTs) { ts = Math.max(ts, event.getTimestamp()); if (++count % slidingCount == 0) { timestamps.add(ts); } } } } return timestamps; }
Action action = evictionPolicy.evict(windowEvent); if (action == EXPIRE) { eventsToExpire.add(windowEvent.get()); it.remove(); } else if (!fullScan || action == STOP) {
@Override public void track(Event<T> event) { if (started && event.isWatermark()) { handleWaterMarkEvent(event); } }
@Test public void testTrackSingleStreamOutOfOrder() throws Exception { waterMarkEventGenerator.track(streamId("s1"), 100); waterMarkEventGenerator.track(streamId("s1"), 110); waterMarkEventGenerator.track(streamId("s1"), 104); waterMarkEventGenerator.run(); assertTrue(eventList.get(0).isWatermark()); assertEquals(105, eventList.get(0).getTimestamp()); }
/** * {@inheritDoc} */ @Override public Action evict(Event<T> event) { long now = evictionContext == null ? System.currentTimeMillis() : evictionContext.getReferenceTime(); long diff = now - event.getTimestamp(); if (diff >= (windowLength + delta)) { return Action.EXPIRE; } else if (diff < 0) { // do not process events beyond current ts return Action.KEEP; } return Action.PROCESS; }
@Override public void track(Event<T> event) { if (started && !event.isWatermark()) { if (currentCount.incrementAndGet() >= count) { evictionPolicy.setContext(new DefaultEvictionContext(System.currentTimeMillis())); handler.onTrigger(); } } }
@Test public void testTrackTwoStreams() throws Exception { Set<GlobalStreamId> streams = new HashSet<>(); streams.add(streamId("s1")); streams.add(streamId("s2")); waterMarkEventGenerator = new WaterMarkEventGenerator<>(windowManager, 100000, 5, streams); waterMarkEventGenerator.track(streamId("s1"), 100); waterMarkEventGenerator.track(streamId("s1"), 110); waterMarkEventGenerator.run(); assertTrue(eventList.isEmpty()); waterMarkEventGenerator.track(streamId("s2"), 95); waterMarkEventGenerator.track(streamId("s2"), 98); waterMarkEventGenerator.run(); assertTrue(eventList.get(0).isWatermark()); assertEquals(93, eventList.get(0).getTimestamp()); }
/** * {@inheritDoc} * <p/> * Keeps events with future ts in the queue for processing in the next window. If the ts difference is more than the lag, stops scanning * the queue for the current window. */ @Override public Action evict(Event<T> event) { if (evictionContext == null) { //It is possible to get asked about eviction before we have a context, due to WindowManager.compactWindow. //In this case we should hold on to all the events. When the first watermark is received, the context will be set, //and the events will be reevaluated for eviction return Action.STOP; } long referenceTime = evictionContext.getReferenceTime(); long diff = referenceTime - event.getTimestamp(); if (diff < -lag) { return Action.STOP; } else if (diff < 0) { return Action.KEEP; } else { return super.evict(event); } }
@Override public Action evict(Event<Integer> event) { eventsScanned.add(event.get()); return super.evict(event); }
@Override public void track(Event<T> event) { if (!event.isWatermark()) { currentCount.incrementAndGet(); } }
@Test public void testLateEvent() throws Exception { assertTrue(waterMarkEventGenerator.track(streamId("s1"), 100)); assertTrue(waterMarkEventGenerator.track(streamId("s1"), 110)); waterMarkEventGenerator.run(); assertTrue(eventList.get(0).isWatermark()); assertEquals(105, eventList.get(0).getTimestamp()); eventList.clear(); assertTrue(waterMarkEventGenerator.track(streamId("s1"), 105)); assertTrue(waterMarkEventGenerator.track(streamId("s1"), 106)); assertTrue(waterMarkEventGenerator.track(streamId("s1"), 115)); assertFalse(waterMarkEventGenerator.track(streamId("s1"), 104)); waterMarkEventGenerator.run(); assertTrue(eventList.get(0).isWatermark()); assertEquals(110, eventList.get(0).getTimestamp()); } }