public static <T> TimeSliding<T> of(Duration duration, Duration step) { return new TimeSliding<>(duration.toMillis(), step.toMillis()); }
@Override protected Dataset<Triple<TimeInterval, Integer, String>> getOutput(Dataset<Pair<String, Integer>> input) { input = AssignEventTime.of(input).using(e -> e.getSecond()).output(); Dataset<Pair<Integer, String>> reduced = ReduceStateByKey.of(input) .keyBy(e -> e.getFirst().charAt(0) - '0') .valueBy(e -> e.getFirst().substring(2)) .stateFactory((StateFactory<String, String, AccState<String>>) AccState::new) .mergeStatesBy(AccState::combine) .windowBy(TimeSliding.of(Duration.ofMillis(10), Duration.ofMillis(5))) .output(); return FlatMap.of(reduced) .using((UnaryFunctor<Pair<Integer, String>, Triple<TimeInterval, Integer, String>>) (elem, context) -> context.collect(Triple.of((TimeInterval) context.getWindow(), elem.getFirst(), elem.getSecond()))) .output(); }
@Test public void testTimeSlidingLabelAssignment() { TimeSliding<Long> windowing = TimeSliding .of(Duration.ofHours(1), Duration.ofMinutes(20)); UnaryFunction<Long, Long> eventTimeAssigner = e -> e * 1000L; long[] data = { 3590, 3600, 3610, 3800, 7190, 7200, 7210 }; for (long event : data) { Iterable<TimeInterval> labels = windowing .assignWindowsToElement(new Elem<>( GlobalWindowing.Window.get(), event, eventTimeAssigner.apply(event))); // verify window count assertEquals(3, Iterables.size(labels)); // verify that each window contains the original event for (TimeInterval l : labels) { long stamp = event * 1000L; assertTrue(stamp >= l.getStartMillis()); assertTrue(stamp <= l.getEndMillis()); } } }
.valueBy(e -> 1) .combineBy(Sums.ofInts()) .windowBy(TimeSliding.of(params.getLongStats(), params.getShortStats())) .output(); .using((Triple<Byte, Pair<String, Double>, Double> e, Collector<String> c) -> { Date now = new Date(); Date stamp = new Date(TimeSliding.getLabel(c).getEndMillis()); c.collect(now + ": " + stamp + ", " + e.getSecond().getFirst() + ", " + e.getSecond().getSecond()); })
@Override protected Dataset<Triple<TimeInterval, Integer, String>> getOutput(Dataset<Pair<String, Integer>> input) { input = AssignEventTime.of(input).using(e -> e.getSecond()).output(); Dataset<Pair<Integer, String>> reduced = ReduceStateByKey.of(input) .keyBy(e -> e.getFirst().charAt(0) - '0') .valueBy(e -> e.getFirst().substring(2)) .stateFactory((StateFactory<String, String, AccState<String>>) AccState::new) .mergeStatesBy(AccState::combine) .windowBy(TimeSliding.of(Duration.ofMillis(10), Duration.ofMillis(5))) .output(); return FlatMap.of(reduced) .using((UnaryFunctor<Pair<Integer, String>, Triple<TimeInterval, Integer, String>>) (elem, context) -> context.collect(Triple.of((TimeInterval) context.getWindow(), elem.getFirst(), elem.getSecond()))) .output(); }
@Test public void testWindowAssignment() { TimeSliding<?> windowing = TimeSliding.of(Duration.ofMillis(10), Duration.ofMillis(5)); windowing.assignWindowsToElement(new TimestampedElement<>(16)); Sets.newHashSet(windows)); windows = windowing.assignWindowsToElement(new TimestampedElement<>(10)); Sets.newHashSet(windows)); windows = windowing.assignWindowsToElement(new TimestampedElement<>(9));
.valueBy(e -> 1L) .reduceBy(Sums.ofLongs()) .windowBy(TimeSliding.of(Duration.ofMillis(10), Duration.ofMillis(5))) .output();
public static <T> TimeSliding<T> of(Duration duration, Duration step) { return new TimeSliding<>(duration.toMillis(), step.toMillis()); }
.valueBy(e -> 1L) .combineBy(Sums.ofLongs()) .windowBy(TimeSliding.of(Duration.ofMillis(10), Duration.ofMillis(5))) .output();