/** * Add an event into the window, with {@link System#currentTimeMillis()} as the tracking ts. * * @param event the event to add */ public void add(T event) { add(event, System.currentTimeMillis()); }
public void addTuplesBatch(Object batchId, List<TridentTuple> tuples) { LOG.debug("Adding tuples to window-manager for batch: [{}]", batchId); for (TridentTuple tridentTuple : tuples) { windowManager.add(tridentTuple); } }
/** * Add an event into the window, with the given ts as the tracking ts. * * @param event the event to track * @param ts the timestamp */ public void add(T event, long ts) { add(new EventImpl<T>(event, ts)); }
private void addToWindowManager(int tupleIndex, String effectiveBatchId, TridentTuple tridentTuple) { TridentTuple actualTuple = null; if (maxCachedTuplesSize == null || currentCachedTuplesSize.get() < maxCachedTuplesSize) { actualTuple = tridentTuple; } currentCachedTuplesSize.incrementAndGet(); windowManager.add(new TridentBatchTuple(effectiveBatchId, System.currentTimeMillis(), tupleIndex, actualTuple)); }
@Override public void run() { try { long waterMarkTs = computeWaterMarkTs(); if (waterMarkTs > lastWaterMarkTs) { this.windowManager.add(new WaterMarkEvent<>(waterMarkTs)); lastWaterMarkTs = waterMarkTs; } } catch (Throwable th) { LOG.error("Failed while processing watermark event ", th); throw th; } }
String batchId = secondLastPart(key); LOG.debug("Received tuple with batch [{}] and tuple index [{}]", batchId, tupleIndexValue); windowManager.add(new TridentBatchTuple(batchId, System.currentTimeMillis(), tupleIndexValue)); } else if (key.startsWith(windowTriggerTaskId)) { triggerKeys.add(key);
@Override public void execute(Tuple input) { if (isTupleTs()) { long ts = timestampExtractor.extractTimestamp(input); if (waterMarkEventGenerator.track(input.getSourceGlobalStreamId(), ts)) { windowManager.add(input, ts); } else { if (lateTupleStream != null) { windowedOutputCollector.emit(lateTupleStream, input, new Values(input)); } else { LOG.info("Received a late tuple {} with ts {}. This will not be processed.", input, ts); } windowedOutputCollector.ack(input); } } else { windowManager.add(input); } }
@Test public void testEventTimeLag() throws Exception { EvictionPolicy<Integer, ?> evictionPolicy = new WatermarkTimeEvictionPolicy<>(20, 5); windowManager.setEvictionPolicy(evictionPolicy); TriggerPolicy<Integer, ?> triggerPolicy = new WatermarkTimeTriggerPolicy<Integer>(10, windowManager, evictionPolicy, windowManager); triggerPolicy.start(); windowManager.setTriggerPolicy(triggerPolicy); windowManager.add(1, 603); windowManager.add(2, 605); windowManager.add(3, 607); windowManager.add(4, 618); windowManager.add(5, 626); windowManager.add(6, 632); windowManager.add(7, 629); windowManager.add(8, 636); // send a watermark event, which should trigger three windows. windowManager.add(new WaterMarkEvent<Integer>(631)); // System.out.println(listener.allOnActivationEvents); assertEquals(3, listener.allOnActivationEvents.size()); assertEquals(seq(1, 3), listener.allOnActivationEvents.get(0)); assertEquals(seq(1, 4), listener.allOnActivationEvents.get(1)); // out of order events should be processed upto the lag assertEquals(Arrays.asList(4, 5, 7), listener.allOnActivationEvents.get(2)); }
windowManager.setTriggerPolicy(triggerPolicy); windowManager.add(1, 603); windowManager.add(2, 605); windowManager.add(3, 607); windowManager.add(4, 618); windowManager.add(5, 626); windowManager.add(6, 629); windowManager.add(7, 636); windowManager.add(8, 637); windowManager.add(9, 638); windowManager.add(10, 639); windowManager.add(new WaterMarkEvent<Integer>(631));
windowManager.setTriggerPolicy(triggerPolicy); windowManager.add(1, 603); windowManager.add(2, 605); windowManager.add(3, 607); windowManager.add(4, 618); windowManager.add(5, 625); windowManager.add(6, 626); windowManager.add(7, 629); windowManager.add(8, 636); windowManager.add(new WaterMarkEvent<Integer>(631)); windowManager.add(9, 665); windowManager.add(10, 666); windowManager.add(11, 667); windowManager.add(12, 669); windowManager.add(12, 679); windowManager.add(new WaterMarkEvent<Integer>(674));
@Test public void testCountBasedTumblingWithSameEventTs() throws Exception { EvictionPolicy<Integer, ?> evictionPolicy = new WatermarkCountEvictionPolicy<>(2); windowManager.setEvictionPolicy(evictionPolicy); TriggerPolicy<Integer, ?> triggerPolicy = new WatermarkCountTriggerPolicy<Integer>(2, windowManager, evictionPolicy, windowManager); triggerPolicy.start(); windowManager.setTriggerPolicy(triggerPolicy); windowManager.add(1, 10); windowManager.add(2, 10); windowManager.add(3, 11); windowManager.add(4, 12); windowManager.add(5, 12); windowManager.add(6, 12); windowManager.add(7, 12); windowManager.add(8, 13); windowManager.add(9, 14); windowManager.add(10, 15); windowManager.add(new WaterMarkEvent<Integer>(20)); assertEquals(5, listener.allOnActivationEvents.size()); assertEquals(seq(1, 2), listener.allOnActivationEvents.get(0)); assertEquals(seq(3, 4), listener.allOnActivationEvents.get(1)); assertEquals(seq(5, 6), listener.allOnActivationEvents.get(2)); assertEquals(seq(7, 8), listener.allOnActivationEvents.get(3)); assertEquals(seq(9, 10), listener.allOnActivationEvents.get(4)); }
@Test public void testCountBasedSlidingWithSameEventTs() throws Exception { EvictionPolicy<Integer, ?> evictionPolicy = new WatermarkCountEvictionPolicy<>(5); windowManager.setEvictionPolicy(evictionPolicy); TriggerPolicy<Integer, ?> triggerPolicy = new WatermarkCountTriggerPolicy<Integer>(2, windowManager, evictionPolicy, windowManager); triggerPolicy.start(); windowManager.setTriggerPolicy(triggerPolicy); windowManager.add(1, 10); windowManager.add(2, 10); windowManager.add(3, 11); windowManager.add(4, 12); windowManager.add(5, 12); windowManager.add(6, 12); windowManager.add(7, 12); windowManager.add(8, 13); windowManager.add(9, 14); windowManager.add(10, 15); windowManager.add(new WaterMarkEvent<Integer>(20)); assertEquals(5, listener.allOnActivationEvents.size()); assertEquals(seq(1, 2), listener.allOnActivationEvents.get(0)); assertEquals(seq(1, 4), listener.allOnActivationEvents.get(1)); assertEquals(seq(2, 6), listener.allOnActivationEvents.get(2)); assertEquals(seq(4, 8), listener.allOnActivationEvents.get(3)); assertEquals(seq(6, 10), listener.allOnActivationEvents.get(4)); }
windowManager.setTriggerPolicy(triggerPolicy); windowManager.add(1, 603); windowManager.add(2, 605); windowManager.add(3, 607); windowManager.add(4, 618); windowManager.add(5, 626); windowManager.add(6, 636); windowManager.add(new WaterMarkEvent<Integer>(631)); windowManager.add(7, 665); windowManager.add(8, 666); windowManager.add(9, 667); windowManager.add(10, 679); windowManager.add(new WaterMarkEvent<Integer>(674));
executor.getWindowManager().add(new WaterMarkEvent<>(activationTs));
executor.getWindowManager().add(new WaterMarkEvent<>(activationTs)); executor.prePrepare(0);
private void testEvictBeforeWatermarkForWatermarkEvictionPolicy(EvictionPolicy watermarkEvictionPolicy, int windowLength) throws Exception { /** * The watermark eviction policy must not evict tuples until the first watermark has been received. * The policies can't make a meaningful decision prior to the first watermark, so the safe decision * is to postpone eviction. */ int threshold = WindowManager.EXPIRE_EVENTS_THRESHOLD; windowManager.setEvictionPolicy(watermarkEvictionPolicy); WatermarkCountTriggerPolicy triggerPolicy = new WatermarkCountTriggerPolicy(windowLength, windowManager, watermarkEvictionPolicy, windowManager); triggerPolicy.start(); windowManager.setTriggerPolicy(triggerPolicy); for (int i : seq(1, threshold)) { windowManager.add(i, i); } assertThat("The watermark eviction policies should never evict events before the first watermark is received", listener.onExpiryEvents, is(empty())); windowManager.add(new WaterMarkEvent<>(threshold)); // The events should be put in a window when the first watermark is received assertEquals(seq(1, threshold), listener.onActivationEvents); //Now add some more events and a new watermark, and check that the previous events are expired for (int i : seq(threshold + 1, threshold * 2)) { windowManager.add(i, i); } windowManager.add(new WaterMarkEvent<>(threshold + windowLength + 1)); //All the events should be expired when the next watermark is received assertThat("All the events should be expired after the second watermark", listener.onExpiryEvents, equalTo(seq(1, threshold))); }
@Test public void testTumblingWindow() throws Exception { EvictionPolicy<Integer, ?> evictionPolicy = new CountEvictionPolicy<Integer>(3); windowManager.setEvictionPolicy(evictionPolicy); TriggerPolicy<Integer, ?> triggerPolicy = new CountTriggerPolicy<Integer>(3, windowManager, evictionPolicy); triggerPolicy.start(); windowManager.setTriggerPolicy(triggerPolicy); windowManager.add(1); windowManager.add(2); // nothing expired yet assertTrue(listener.onExpiryEvents.isEmpty()); windowManager.add(3); assertTrue(listener.onExpiryEvents.isEmpty()); assertEquals(seq(1, 3), listener.onActivationEvents); assertTrue(listener.onActivationExpiredEvents.isEmpty()); assertEquals(seq(1, 3), listener.onActivationNewEvents); listener.clear(); windowManager.add(4); windowManager.add(5); windowManager.add(6); assertEquals(seq(1, 3), listener.onExpiryEvents); assertEquals(seq(4, 6), listener.onActivationEvents); assertEquals(seq(1, 3), listener.onActivationExpiredEvents); assertEquals(seq(4, 6), listener.onActivationNewEvents); }
@Test public void testExpireThreshold() throws Exception { int threshold = WindowManager.EXPIRE_EVENTS_THRESHOLD; int windowLength = 5; windowManager.setEvictionPolicy(new CountEvictionPolicy<Integer>(5)); TriggerPolicy<Integer, ?> triggerPolicy = new TimeTriggerPolicy<Integer>(new Duration(1, TimeUnit.HOURS).value, windowManager); triggerPolicy.start(); windowManager.setTriggerPolicy(triggerPolicy); for (int i : seq(1, 5)) { windowManager.add(i); } // nothing expired yet assertTrue(listener.onExpiryEvents.isEmpty()); for (int i : seq(6, 10)) { windowManager.add(i); } for (int i : seq(11, threshold)) { windowManager.add(i); } // window should be compacted and events should be expired. assertEquals(seq(1, threshold - windowLength), listener.onExpiryEvents); }
windowManager.setEvictionPolicy(evictionPolicy); windowManager.setTriggerPolicy(triggerPolicy); windowManager.add(1); windowManager.add(2); assertEquals(seq(1, 2), listener.onActivationNewEvents); assertTrue(listener.onActivationExpiredEvents.isEmpty()); windowManager.add(3); windowManager.add(4); assertEquals(seq(3, 4), listener.onActivationNewEvents); assertTrue(listener.onActivationExpiredEvents.isEmpty()); windowManager.add(5); windowManager.add(6); assertEquals(seq(1), listener.onActivationExpiredEvents); listener.clear(); windowManager.add(7); windowManager.add(8);
windowManager.add(i);