@Test public void testExpireThresholdWithWatermarkTimeEvictionPolicy() throws Exception { int windowLength = WindowManager.EXPIRE_EVENTS_THRESHOLD; EvictionPolicy watermarkTimeEvictionPolicy = new WatermarkTimeEvictionPolicy(windowLength); testEvictBeforeWatermarkForWatermarkEvictionPolicy(watermarkTimeEvictionPolicy, windowLength); }
@Override public Action evict(Event<Integer> event) { eventsScanned.add(event.get()); return super.evict(event); }
private EvictionPolicy<Tuple, ?> getEvictionPolicy(Count windowLengthCount, Duration windowLengthDuration) { if (windowLengthCount != null) { if (isTupleTs()) { return new WatermarkCountEvictionPolicy<>(windowLengthCount.value); } else { return new CountEvictionPolicy<>(windowLengthCount.value); } } else { if (isTupleTs()) { return new WatermarkTimeEvictionPolicy<>(windowLengthDuration.value, maxLagMs); } else { return new TimeEvictionPolicy<>(windowLengthDuration.value); } } }
@Test public void testEventTimeLag() throws Exception { EvictionPolicy<Integer, ?> evictionPolicy = new WatermarkTimeEvictionPolicy<>(20, 5); windowManager.setEvictionPolicy(evictionPolicy); TriggerPolicy<Integer, ?> triggerPolicy = new WatermarkTimeTriggerPolicy<Integer>(10, windowManager, evictionPolicy, windowManager); triggerPolicy.start(); windowManager.setTriggerPolicy(triggerPolicy); windowManager.add(1, 603); windowManager.add(2, 605); windowManager.add(3, 607); windowManager.add(4, 618); windowManager.add(5, 626); windowManager.add(6, 632); windowManager.add(7, 629); windowManager.add(8, 636); // send a watermark event, which should trigger three windows. windowManager.add(new WaterMarkEvent<Integer>(631)); // System.out.println(listener.allOnActivationEvents); assertEquals(3, listener.allOnActivationEvents.size()); assertEquals(seq(1, 3), listener.allOnActivationEvents.get(0)); assertEquals(seq(1, 4), listener.allOnActivationEvents.get(1)); // out of order events should be processed upto the lag assertEquals(Arrays.asList(4, 5, 7), listener.allOnActivationEvents.get(2)); }
@Test public void testEventTimeBasedWindow() throws Exception { EvictionPolicy<Integer, ?> evictionPolicy = new WatermarkTimeEvictionPolicy<>(20); windowManager.setEvictionPolicy(evictionPolicy); TriggerPolicy<Integer, ?> triggerPolicy = new WatermarkTimeTriggerPolicy<Integer>(10, windowManager, evictionPolicy, windowManager);
@Test public void testCountBasedTriggerWithEventTs() throws Exception { EvictionPolicy<Integer, ?> evictionPolicy = new WatermarkTimeEvictionPolicy<Integer>(20); windowManager.setEvictionPolicy(evictionPolicy); TriggerPolicy<Integer, ?> triggerPolicy = new WatermarkCountTriggerPolicy<Integer>(3, windowManager, evictionPolicy, windowManager);
private EvictionPolicy<Tuple> getEvictionPolicy(Count windowLengthCount, Duration windowLengthDuration) { if (windowLengthCount != null) { if (isTupleTs()) { return new WatermarkCountEvictionPolicy<>(windowLengthCount.value); } else { return new CountEvictionPolicy<>(windowLengthCount.value); } } else { if (isTupleTs()) { return new WatermarkTimeEvictionPolicy<>(windowLengthDuration.value, maxLagMs); } else { return new TimeEvictionPolicy<>(windowLengthDuration.value); } } }