/** * Pressure reliever that maintains up to {@code count} most recent tuples per key. * * @param count Number of tuples to maintain where downstream processing cannot keep up. * @param keyFunction Key function for tuples. */ public PressureReliever(int count, Function<T, K> keyFunction) { window = Windows.window( Policies.alwaysInsert(), Policies.countContentsPolicy(count), Policies.evictOldest(), new FirstSubmitter(), keyFunction, () -> new LinkedList<T>()); // No processing of the window takes place window.registerPartitionProcessor((tuples, k) -> { }); }
@Override public <U> TStream<U> aggregate(BiFunction<List<T>,K, U> processor) { processor = Functions.synchronizedBiFunction(processor); Window<T, K, LinkedList<T>> window = Windows.lastNProcessOnInsert(size, getKeyFunction()); Aggregate<T,U,K> op = new Aggregate<T,U,K>(window, processor); return feeder().pipe(op); }
Window<U, K, ?> window; if(twindow instanceof TWindowImpl){ window = Windows.lastNProcessOnInsert(((TWindowImpl<U, K>)twindow).getSize(), twindow.getKeyFunction()); long time = ((TWindowTimeImpl<U, K>)(twindow)).getTime(); TimeUnit unit = ((TWindowTimeImpl<U, K>)(twindow)).getUnit(); window = Windows.window( alwaysInsert(), scheduleEvictIfEmpty(time, unit),
Window<U, K, ?> window; if(twindow instanceof TWindowImpl){ window = Windows.lastNProcessOnInsert(((TWindowImpl<U, K>)twindow).getSize(), twindow.getKeyFunction()); long time = ((TWindowTimeImpl<U, K>)(twindow)).getTime(); TimeUnit unit = ((TWindowTimeImpl<U, K>)(twindow)).getUnit(); window = Windows.window( alwaysInsert(), scheduleEvictIfEmpty(time, unit),
/** * Pressure reliever that maintains up to {@code count} most recent tuples per key. * * @param count Number of tuples to maintain where downstream processing cannot keep up. * @param keyFunction Key function for tuples. */ public PressureReliever(int count, Function<T, K> keyFunction) { window = Windows.window( Policies.alwaysInsert(), Policies.countContentsPolicy(count), Policies.evictOldest(), new FirstSubmitter(), keyFunction, () -> new LinkedList<T>()); // No processing of the window takes place window.registerPartitionProcessor((tuples, k) -> { }); }
@Override public <U> TStream<U> aggregate(BiFunction<List<T>,K, U> processor) { processor = Functions.synchronizedBiFunction(processor); Window<T, K, LinkedList<T>> window = Windows.lastNProcessOnInsert(size, getKeyFunction()); Aggregate<T,U,K> op = new Aggregate<T,U,K>(window, processor); return feeder().pipe(op); }
/** * Return a window that maintains the last {@code count} tuples inserted * with processing triggered on every insert. This provides * a continuous processing, where processing is invoked every * time the window changes. Since insertion drives eviction * there is no need to process on eviction, thus once the window * has reached {@code count} tuples, each insertion results in an * eviction followed by processing of {@code count} tuples * including the tuple just inserted, which is the definition of * the window. * * @param <T> Tuple type. * @param <K> Key type. * * @param count Number of tuple to maintain per partition * @param keyFunction Tuple partitioning key function * @return window that maintains the last {@code count} tuples on a stream */ public static <T, K> Window<T, K, LinkedList<T>> lastNProcessOnInsert(final int count, Function<T, K> keyFunction) { Window<T, K, LinkedList<T>> window = Windows.window( alwaysInsert(), countContentsPolicy(count), evictOldest(), processOnInsert(), keyFunction, () -> new LinkedList<T>()); return window; }
@Test public void noWaitConcurrentWindowAccessTest() throws InterruptedException { Window<Integer, Integer, ? extends List<Integer>> window = Windows.lastNProcessOnInsert(100, tuple -> 0); window.registerPartitionProcessor((tuples, key) -> {}); long finishTime = System.currentTimeMillis() + 3000;
/** * Return a window that maintains the last {@code count} tuples inserted * with processing triggered on every insert. This provides * a continuous processing, where processing is invoked every * time the window changes. Since insertion drives eviction * there is no need to process on eviction, thus once the window * has reached {@code count} tuples, each insertion results in an * eviction followed by processing of {@code count} tuples * including the tuple just inserted, which is the definition of * the window. * * @param <T> Tuple type. * @param <K> Key type. * * @param count Number of tuple to maintain per partition * @param keyFunction Tuple partitioning key function * @return window that maintains the last {@code count} tuples on a stream */ public static <T, K> Window<T, K, LinkedList<T>> lastNProcessOnInsert(final int count, Function<T, K> keyFunction) { Window<T, K, LinkedList<T>> window = Windows.window( alwaysInsert(), countContentsPolicy(count), evictOldest(), processOnInsert(), keyFunction, () -> new LinkedList<T>()); return window; }
@Override public <J, U, K> TStream<J> joinLast(Function<T, K> keyer, TStream<U> lastStream, Function<U, K> lastStreamKeyer, BiFunction<T, U, J> joiner) { BiFunction<List<U>,K, Object> processor = Functions.synchronizedBiFunction((list, key) -> null); Window<U, K, LinkedList<U>> window = Windows.lastNProcessOnInsert(1, lastStreamKeyer); Aggregate<U,Object,K> op = new Aggregate<U,Object,K>(window, processor); lastStream.pipe(op); return this.map((tuple) -> { Partition<U, K, ? extends List<U>> part = window.getPartitions().get(keyer.apply(tuple)); if(part == null) return null; J ret; synchronized (part) { U last = part.getContents().get(0); ret = joiner.apply(tuple, last); } return ret; }); }
@Override public <U> TStream<U> batch(BiFunction<List<T>, K, U> batcher) { batcher = Functions.synchronizedBiFunction(batcher); Window<T, K, List<T>> window = Windows.window( alwaysInsert(), Policies.scheduleEvictOnFirstInsert(time, unit), Policies.evictAllAndScheduleEvictWithProcess(time, unit), (partition, tuple) -> {}, getKeyFunction(), () -> new ArrayList<T>()); Aggregate<T,U,K> op = new Aggregate<T,U,K>(window, batcher); return feeder().pipe(op); }
@Override public <J, U, K> TStream<J> joinLast(Function<T, K> keyer, TStream<U> lastStream, Function<U, K> lastStreamKeyer, BiFunction<T, U, J> joiner) { BiFunction<List<U>,K, Object> processor = Functions.synchronizedBiFunction((list, key) -> null); Window<U, K, LinkedList<U>> window = Windows.lastNProcessOnInsert(1, lastStreamKeyer); Aggregate<U,Object,K> op = new Aggregate<U,Object,K>(window, processor); lastStream.pipe(op); return this.map((tuple) -> { Partition<U, K, ? extends List<U>> part = window.getPartitions().get(keyer.apply(tuple)); if(part == null) return null; J ret; synchronized (part) { U last = part.getContents().get(0); ret = joiner.apply(tuple, last); } return ret; }); }
@Override public <U> TStream<U> batch(BiFunction<List<T>, K, U> batcher) { batcher = Functions.synchronizedBiFunction(batcher); Window<T, K, List<T>> window = Windows.window( alwaysInsert(), Policies.scheduleEvictOnFirstInsert(time, unit), Policies.evictAllAndScheduleEvictWithProcess(time, unit), (partition, tuple) -> {}, getKeyFunction(), () -> new ArrayList<T>()); Aggregate<T,U,K> op = new Aggregate<T,U,K>(window, batcher); return feeder().pipe(op); }
Window<Integer, Integer, ? extends List<Integer>> window = Windows.lastNProcessOnInsert(10, tuple->tuple%10);
@Override public <U> TStream<U> batch(BiFunction<List<T>, K, U> batcher) { batcher = Functions.synchronizedBiFunction(batcher); Window<T, K, List<T>> window = Windows.window( alwaysInsert(), Policies.doNothing(), Policies.evictAll(), Policies.processWhenFullAndEvict(size), getKeyFunction(), () -> new ArrayList<T>(size)); Aggregate<T,U,K> op = new Aggregate<T,U,K>(window, batcher); return feeder().pipe(op); }
@Test public void concurrentWindowAccessTest() throws InterruptedException { Window<Integer, Integer, ? extends List<Integer>> window = Windows.lastNProcessOnInsert(10, tuple -> 0);
@Override public <U> TStream<U> batch(BiFunction<List<T>, K, U> batcher) { batcher = Functions.synchronizedBiFunction(batcher); Window<T, K, List<T>> window = Windows.window( alwaysInsert(), Policies.doNothing(), Policies.evictAll(), Policies.processWhenFullAndEvict(size), getKeyFunction(), () -> new ArrayList<T>(size)); Aggregate<T,U,K> op = new Aggregate<T,U,K>(window, batcher); return feeder().pipe(op); }
final int COUNT = 100; Window<Integer, Integer, ? extends List<Integer>> window = Windows.lastNProcessOnInsert(10, unpartitioned());
Windows.window( alwaysInsert(), Policies.scheduleEvictOnFirstInsert(1, TimeUnit.SECONDS),
Windows.window( alwaysInsert(), Policies.scheduleEvictOnFirstInsert(1, TimeUnit.SECONDS),