/** * Windows this {@code KeyedStream} into sliding count windows. * * @param size The size of the windows in number of elements. * @param slide The slide interval in number of elements. */ public WindowedStream<T, KEY, GlobalWindow> countWindow(long size, long slide) { return window(GlobalWindows.create()) .evictor(CountEvictor.of(size)) .trigger(CountTrigger.of(slide)); }
/** * Creates a {@code CountEvictor} that keeps the given number of elements. * Eviction is done before the window function. * * @param maxCount The number of elements to keep in the pane. */ public static <W extends Window> CountEvictor<W> of(long maxCount) { return new CountEvictor<>(maxCount); }
@Override public void evictAfter(Iterable<TimestampedValue<Object>> elements, int size, W window, EvictorContext ctx) { if (doEvictAfter) { evict(elements, size, ctx); } }
/** * Windows this {@code DataStream} into sliding count windows. * * <p>Note: This operation is inherently non-parallel since all elements have to pass through * the same operator instance. * * @param size The size of the windows in number of elements. * @param slide The slide interval in number of elements. */ public AllWindowedStream<T, GlobalWindow> countWindowAll(long size, long slide) { return windowAll(GlobalWindows.create()) .evictor(CountEvictor.of(size)) .trigger(CountTrigger.of(slide)); }
@Override public void evictBefore(Iterable<TimestampedValue<Object>> elements, int size, W window, EvictorContext ctx) { if (!doEvictAfter) { evict(elements, size, ctx); } }
/** * Creates a {@code CountEvictor} that keeps the given number of elements in the pane * Eviction is done before/after the window function based on the value of doEvictAfter. * * @param maxCount The number of elements to keep in the pane. * @param doEvictAfter Whether to do eviction after the window function. */ public static <W extends Window> CountEvictor<W> of(long maxCount, boolean doEvictAfter) { return new CountEvictor<>(maxCount, doEvictAfter); } }
windowFunction, trigger, CountEvictor.<W>of(100), allowedLatenss, lateOutputTag);
@Override public void evictBefore(Iterable<TimestampedValue<Object>> elements, int size, W window, EvictorContext ctx) { if (!doEvictAfter) { evict(elements, size, ctx); } }
/** * Creates a {@code CountEvictor} that keeps the given number of elements in the pane * Eviction is done before/after the window function based on the value of doEvictAfter. * * @param maxCount The number of elements to keep in the pane. * @param doEvictAfter Whether to do eviction after the window function. */ public static <W extends Window> CountEvictor<W> of(long maxCount, boolean doEvictAfter) { return new CountEvictor<>(maxCount, doEvictAfter); } }
@Test @SuppressWarnings("rawtypes") public void testMergingWindowsWithEvictor() throws Exception { StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment(); env.setStreamTimeCharacteristic(TimeCharacteristic.IngestionTime); DataStream<Tuple2<String, Integer>> source = env.fromElements(Tuple2.of("hello", 1), Tuple2.of("hello", 2)); DataStream<Tuple3<String, String, Integer>> window1 = source .windowAll(EventTimeSessionWindows.withGap(Time.seconds(5))) .evictor(CountEvictor.of(5)) .process(new TestProcessAllWindowFunction()); OneInputTransformation<Tuple2<String, Integer>, Tuple3<String, String, Integer>> transform = (OneInputTransformation<Tuple2<String, Integer>, Tuple3<String, String, Integer>>) window1.getTransformation(); OneInputStreamOperator<Tuple2<String, Integer>, Tuple3<String, String, Integer>> operator = transform.getOperator(); Assert.assertTrue(operator instanceof WindowOperator); WindowOperator<String, Tuple2<String, Integer>, ?, ?, ?> winOperator = (WindowOperator<String, Tuple2<String, Integer>, ?, ?, ?>) operator; Assert.assertTrue(winOperator.getTrigger() instanceof EventTimeTrigger); Assert.assertTrue(winOperator.getWindowAssigner() instanceof EventTimeSessionWindows); Assert.assertTrue(winOperator.getStateDescriptor() instanceof ListStateDescriptor); processElementAndEnsureOutput(winOperator, winOperator.getKeySelector(), BasicTypeInfo.STRING_TYPE_INFO, new Tuple2<>("hello", 1)); }
@Override public void evictBefore(Iterable<TimestampedValue<Object>> elements, int size, W window, EvictorContext ctx) { if (!doEvictAfter) { evict(elements, size, ctx); } }
/** * Creates a {@code CountEvictor} that keeps the given number of elements. * Eviction is done before the window function. * * @param maxCount The number of elements to keep in the pane. */ public static <W extends Window> CountEvictor<W> of(long maxCount) { return new CountEvictor<>(maxCount); }
new InternalIterableWindowFunction<>(new RichSumReducer<GlobalWindow>(closeCalled)), CountTrigger.of(windowSlide), CountEvictor.of(windowSize), 0,
@Override public void evictAfter(Iterable<TimestampedValue<Object>> elements, int size, W window, EvictorContext ctx) { if (doEvictAfter) { evict(elements, size, ctx); } }
/** * Creates a {@code CountEvictor} that keeps the given number of elements in the pane * Eviction is done before/after the window function based on the value of doEvictAfter. * * @param maxCount The number of elements to keep in the pane. * @param doEvictAfter Whether to do eviction after the window function. */ public static <W extends Window> CountEvictor<W> of(long maxCount, boolean doEvictAfter) { return new CountEvictor<>(maxCount, doEvictAfter); } }
@Test public void testAggregateWithEvictor() throws Exception { StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment(); env.setStreamTimeCharacteristic(TimeCharacteristic.IngestionTime); DataStream<Tuple2<String, Integer>> source = env.fromElements(Tuple2.of("hello", 1), Tuple2.of("hello", 2)); DataStream<Tuple2<String, Integer>> window1 = source .windowAll(SlidingEventTimeWindows.of(Time.of(1, TimeUnit.SECONDS), Time.of(100, TimeUnit.MILLISECONDS))) .evictor(CountEvictor.of(100)) .aggregate(new DummyAggregationFunction()); OneInputTransformation<Tuple2<String, Integer>, Tuple2<String, Integer>> transform = (OneInputTransformation<Tuple2<String, Integer>, Tuple2<String, Integer>>) window1.getTransformation(); OneInputStreamOperator<Tuple2<String, Integer>, Tuple2<String, Integer>> operator = transform.getOperator(); Assert.assertTrue(operator instanceof WindowOperator); WindowOperator<String, Tuple2<String, Integer>, ?, ?, ?> winOperator = (WindowOperator<String, Tuple2<String, Integer>, ?, ?, ?>) operator; Assert.assertTrue(winOperator.getTrigger() instanceof EventTimeTrigger); Assert.assertTrue(winOperator.getWindowAssigner() instanceof SlidingEventTimeWindows); Assert.assertTrue(winOperator.getStateDescriptor() instanceof ListStateDescriptor); processElementAndEnsureOutput( winOperator, winOperator.getKeySelector(), BasicTypeInfo.STRING_TYPE_INFO, new Tuple2<>("hello", 1)); }
@Override public void evictAfter(Iterable<TimestampedValue<Object>> elements, int size, W window, EvictorContext ctx) { if (doEvictAfter) { evict(elements, size, ctx); } }
/** * Creates a {@code CountEvictor} that keeps the given number of elements. * Eviction is done before the window function. * * @param maxCount The number of elements to keep in the pane. */ public static <W extends Window> CountEvictor<W> of(long maxCount) { return new CountEvictor<>(maxCount); }
new InternalIterableWindowFunction<>(new RichSumReducer<TimeWindow>(closeCalled)), EventTimeTrigger.create(), CountEvictor.of(windowSize), 0,
@Override public void evictBefore(Iterable<TimestampedValue<Object>> elements, int size, W window, EvictorContext ctx) { if (!doEvictAfter) { evict(elements, size, ctx); } }