@Override public <U> TStream<U> batch(BiFunction<List<T>, K, U> batcher) { batcher = Functions.synchronizedBiFunction(batcher); Window<T, K, List<T>> window = Windows.window( alwaysInsert(), Policies.scheduleEvictOnFirstInsert(time, unit), Policies.evictAllAndScheduleEvictWithProcess(time, unit), (partition, tuple) -> {}, getKeyFunction(), () -> new ArrayList<T>()); Aggregate<T,U,K> op = new Aggregate<T,U,K>(window, batcher); return feeder().pipe(op); }
Window<T, K, InsertionTimeList<T>> window = Windows.window( alwaysInsert(), scheduleEvictIfEmpty(time, unit), evictOlderWithProcess(time, unit), processOnInsert(), getKeyFunction(), insertionTimeList());
/** * Return a window that maintains the last {@code count} tuples inserted * with processing triggered on every insert. This provides * a continuous processing, where processing is invoked every * time the window changes. Since insertion drives eviction * there is no need to process on eviction, thus once the window * has reached {@code count} tuples, each insertion results in an * eviction followed by processing of {@code count} tuples * including the tuple just inserted, which is the definition of * the window. * * @param <T> Tuple type. * @param <K> Key type. * * @param count Number of tuple to maintain per partition * @param keyFunction Tuple partitioning key function * @return window that maintains the last {@code count} tuples on a stream */ public static <T, K> Window<T, K, LinkedList<T>> lastNProcessOnInsert(final int count, Function<T, K> keyFunction) { Window<T, K, LinkedList<T>> window = Windows.window( alwaysInsert(), countContentsPolicy(count), evictOldest(), processOnInsert(), keyFunction, () -> new LinkedList<T>()); return window; }
/** * Pressure reliever that maintains up to {@code count} most recent tuples per key. * * @param count Number of tuples to maintain where downstream processing cannot keep up. * @param keyFunction Key function for tuples. */ public PressureReliever(int count, Function<T, K> keyFunction) { window = Windows.window( Policies.alwaysInsert(), Policies.countContentsPolicy(count), Policies.evictOldest(), new FirstSubmitter(), keyFunction, () -> new LinkedList<T>()); // No processing of the window takes place window.registerPartitionProcessor((tuples, k) -> { }); }
@Override public <U> TStream<U> batch(BiFunction<List<T>, K, U> batcher) { batcher = Functions.synchronizedBiFunction(batcher); Window<T, K, List<T>> window = Windows.window( alwaysInsert(), Policies.doNothing(), Policies.evictAll(), Policies.processWhenFullAndEvict(size), getKeyFunction(), () -> new ArrayList<T>(size)); Aggregate<T,U,K> op = new Aggregate<T,U,K>(window, batcher); return feeder().pipe(op); }
TimeUnit unit = ((TWindowTimeImpl<U, K>)(twindow)).getUnit(); window = Windows.window( alwaysInsert(), scheduleEvictIfEmpty(time, unit), evictOlderWithProcess(time, unit), Policies.doNothing(), twindow.getKeyFunction(), insertionTimeList());
Policies.alwaysInsert(), // insertion policy Policies.scheduleEvictIfEmpty(1000, TimeUnit.MILLISECONDS), Policies.evictOlderWithProcess(1000, TimeUnit.MILLISECONDS), (partition, tuple) -> { if(initialized.get(0).booleanValue() == false){
Policies.processOnInsert(), tuple -> tuple, () -> new LinkedList<Integer>());
@Override public <U> TStream<U> batch(BiFunction<List<T>, K, U> batcher) { batcher = Functions.synchronizedBiFunction(batcher); Window<T, K, List<T>> window = Windows.window( alwaysInsert(), Policies.doNothing(), Policies.evictAll(), Policies.processWhenFullAndEvict(size), getKeyFunction(), () -> new ArrayList<T>(size)); Aggregate<T,U,K> op = new Aggregate<T,U,K>(window, batcher); return feeder().pipe(op); }
TimeUnit unit = ((TWindowTimeImpl<U, K>)(twindow)).getUnit(); window = Windows.window( alwaysInsert(), scheduleEvictIfEmpty(time, unit), evictOlderWithProcess(time, unit), Policies.doNothing(), twindow.getKeyFunction(), insertionTimeList());
/** * Pressure reliever that maintains up to {@code count} most recent tuples per key. * * @param count Number of tuples to maintain where downstream processing cannot keep up. * @param keyFunction Key function for tuples. */ public PressureReliever(int count, Function<T, K> keyFunction) { window = Windows.window( Policies.alwaysInsert(), Policies.countContentsPolicy(count), Policies.evictOldest(), new FirstSubmitter(), keyFunction, () -> new LinkedList<T>()); // No processing of the window takes place window.registerPartitionProcessor((tuples, k) -> { }); }
@Override public <U> TStream<U> batch(BiFunction<List<T>, K, U> batcher) { batcher = Functions.synchronizedBiFunction(batcher); Window<T, K, List<T>> window = Windows.window( alwaysInsert(), Policies.scheduleEvictOnFirstInsert(time, unit), Policies.evictAllAndScheduleEvictWithProcess(time, unit), (partition, tuple) -> {}, getKeyFunction(), () -> new ArrayList<T>()); Aggregate<T,U,K> op = new Aggregate<T,U,K>(window, batcher); return feeder().pipe(op); }
@SuppressWarnings("serial") @Test public void countBatchWindowTest(){ List<Integer> numBatches = new LinkedList<>(); Window<Integer, Integer, List<Integer>> window = Windows.window( alwaysInsert(), Policies.doNothing(), Policies.evictAll(), Policies.processWhenFullAndEvict(113), tuple -> 0, () -> new ArrayList<Integer>()); window.registerPartitionProcessor(new BiConsumer<List<Integer>, Integer>() { @Override public void accept(List<Integer> t, Integer u) { numBatches.add(1); } }); for(int i = 0; i < 1000; i++){ window.insert(i); } assertTrue(numBatches.size() == 8); }
Window<T, K, InsertionTimeList<T>> window = Windows.window( alwaysInsert(), scheduleEvictIfEmpty(time, unit), evictOlderWithProcess(time, unit), processOnInsert(), getKeyFunction(), insertionTimeList());
/** * Return a window that maintains the last {@code count} tuples inserted * with processing triggered on every insert. This provides * a continuous processing, where processing is invoked every * time the window changes. Since insertion drives eviction * there is no need to process on eviction, thus once the window * has reached {@code count} tuples, each insertion results in an * eviction followed by processing of {@code count} tuples * including the tuple just inserted, which is the definition of * the window. * * @param <T> Tuple type. * @param <K> Key type. * * @param count Number of tuple to maintain per partition * @param keyFunction Tuple partitioning key function * @return window that maintains the last {@code count} tuples on a stream */ public static <T, K> Window<T, K, LinkedList<T>> lastNProcessOnInsert(final int count, Function<T, K> keyFunction) { Window<T, K, LinkedList<T>> window = Windows.window( alwaysInsert(), countContentsPolicy(count), evictOldest(), processOnInsert(), keyFunction, () -> new LinkedList<T>()); return window; }
Window<Integer, Integer, List<Integer>> window = Windows.window( alwaysInsert(), Policies.scheduleEvictOnFirstInsert(1, TimeUnit.SECONDS), Policies.evictAllAndScheduleEvictWithProcess(1, TimeUnit.SECONDS), (partiton, tuple) -> {}, tuple -> 0,
Window<Integer, Integer, List<Integer>> window = Windows.window( alwaysInsert(), Policies.scheduleEvictOnFirstInsert(1, TimeUnit.SECONDS), Policies.evictAllAndScheduleEvictWithProcess(1, TimeUnit.SECONDS), (partiton, tuple) -> {}, tuple -> 0,
Window<Integer, Integer, List<Integer>> window = Windows.window( alwaysInsert(), Policies.scheduleEvictOnFirstInsert(1, TimeUnit.SECONDS), Policies.evictAllAndScheduleEvictWithProcess(1, TimeUnit.SECONDS), (partiton, tuple) -> {}, tuple -> tuple,