private TriggerPolicy<Tuple, ?> getTriggerPolicy(Count slidingIntervalCount, Duration slidingIntervalDuration, WindowManager<Tuple> manager, EvictionPolicy<Tuple, ?> evictionPolicy) { if (slidingIntervalCount != null) { if (isTupleTs()) { return new WatermarkCountTriggerPolicy<>(slidingIntervalCount.value, manager, evictionPolicy, manager); } else { return new CountTriggerPolicy<>(slidingIntervalCount.value, manager, evictionPolicy); } } else { if (isTupleTs()) { return new WatermarkTimeTriggerPolicy<>(slidingIntervalDuration.value, manager, evictionPolicy, manager); } else { return new TimeTriggerPolicy<>(slidingIntervalDuration.value, manager, evictionPolicy); } } }
@Override public void track(Event<T> event) { if (started && event.isWatermark()) { handleWaterMarkEvent(event); } }
private void testEvictBeforeWatermarkForWatermarkEvictionPolicy(EvictionPolicy watermarkEvictionPolicy, int windowLength) throws Exception { /** * The watermark eviction policy must not evict tuples until the first watermark has been received. * The policies can't make a meaningful decision prior to the first watermark, so the safe decision * is to postpone eviction. */ int threshold = WindowManager.EXPIRE_EVENTS_THRESHOLD; windowManager.setEvictionPolicy(watermarkEvictionPolicy); WatermarkCountTriggerPolicy triggerPolicy = new WatermarkCountTriggerPolicy(windowLength, windowManager, watermarkEvictionPolicy, windowManager); triggerPolicy.start(); windowManager.setTriggerPolicy(triggerPolicy); for (int i : seq(1, threshold)) { windowManager.add(i, i); } assertThat("The watermark eviction policies should never evict events before the first watermark is received", listener.onExpiryEvents, is(empty())); windowManager.add(new WaterMarkEvent<>(threshold)); // The events should be put in a window when the first watermark is received assertEquals(seq(1, threshold), listener.onActivationEvents); //Now add some more events and a new watermark, and check that the previous events are expired for (int i : seq(threshold + 1, threshold * 2)) { windowManager.add(i, i); } windowManager.add(new WaterMarkEvent<>(threshold + windowLength + 1)); //All the events should be expired when the next watermark is received assertThat("All the events should be expired after the second watermark", listener.onExpiryEvents, equalTo(seq(1, threshold))); }
@Test public void testCountBasedTumblingWithSameEventTs() throws Exception { EvictionPolicy<Integer, ?> evictionPolicy = new WatermarkCountEvictionPolicy<>(2); windowManager.setEvictionPolicy(evictionPolicy); TriggerPolicy<Integer, ?> triggerPolicy = new WatermarkCountTriggerPolicy<Integer>(2, windowManager, evictionPolicy, windowManager); triggerPolicy.start(); windowManager.setTriggerPolicy(triggerPolicy); windowManager.add(1, 10); windowManager.add(2, 10); windowManager.add(3, 11); windowManager.add(4, 12); windowManager.add(5, 12); windowManager.add(6, 12); windowManager.add(7, 12); windowManager.add(8, 13); windowManager.add(9, 14); windowManager.add(10, 15); windowManager.add(new WaterMarkEvent<Integer>(20)); assertEquals(5, listener.allOnActivationEvents.size()); assertEquals(seq(1, 2), listener.allOnActivationEvents.get(0)); assertEquals(seq(3, 4), listener.allOnActivationEvents.get(1)); assertEquals(seq(5, 6), listener.allOnActivationEvents.get(2)); assertEquals(seq(7, 8), listener.allOnActivationEvents.get(3)); assertEquals(seq(9, 10), listener.allOnActivationEvents.get(4)); }
@Override public void track(Event<T> event) { if (started && event.isWatermark()) { handleWaterMarkEvent(event); } }
@Test public void testCountBasedSlidingWithSameEventTs() throws Exception { EvictionPolicy<Integer, ?> evictionPolicy = new WatermarkCountEvictionPolicy<>(5); windowManager.setEvictionPolicy(evictionPolicy); TriggerPolicy<Integer, ?> triggerPolicy = new WatermarkCountTriggerPolicy<Integer>(2, windowManager, evictionPolicy, windowManager); triggerPolicy.start(); windowManager.setTriggerPolicy(triggerPolicy); windowManager.add(1, 10); windowManager.add(2, 10); windowManager.add(3, 11); windowManager.add(4, 12); windowManager.add(5, 12); windowManager.add(6, 12); windowManager.add(7, 12); windowManager.add(8, 13); windowManager.add(9, 14); windowManager.add(10, 15); windowManager.add(new WaterMarkEvent<Integer>(20)); assertEquals(5, listener.allOnActivationEvents.size()); assertEquals(seq(1, 2), listener.allOnActivationEvents.get(0)); assertEquals(seq(1, 4), listener.allOnActivationEvents.get(1)); assertEquals(seq(2, 6), listener.allOnActivationEvents.get(2)); assertEquals(seq(4, 8), listener.allOnActivationEvents.get(3)); assertEquals(seq(6, 10), listener.allOnActivationEvents.get(4)); }
EvictionPolicy<Integer, ?> evictionPolicy = new WatermarkTimeEvictionPolicy<Integer>(20); windowManager.setEvictionPolicy(evictionPolicy); TriggerPolicy<Integer, ?> triggerPolicy = new WatermarkCountTriggerPolicy<Integer>(3, windowManager, evictionPolicy, windowManager); triggerPolicy.start(); windowManager.setTriggerPolicy(triggerPolicy);
private TriggerPolicy<Tuple> getTriggerPolicy(Count slidingIntervalCount, Duration slidingIntervalDuration, WindowManager<Tuple> manager, EvictionPolicy<Tuple> evictionPolicy) { if (slidingIntervalCount != null) { if (isTupleTs()) { return new WatermarkCountTriggerPolicy<>(slidingIntervalCount.value, manager, evictionPolicy, manager); } else { return new CountTriggerPolicy<>(slidingIntervalCount.value, manager, evictionPolicy); } } else { if (isTupleTs()) { return new WatermarkTimeTriggerPolicy<>(slidingIntervalDuration.value, manager, evictionPolicy, manager); } else { return new TimeTriggerPolicy<>(slidingIntervalDuration.value, manager, evictionPolicy); } } }