@Override protected Dataset<Integer> getOutput(Dataset<Pair<Integer, Long>> input) { input = AssignEventTime.of(input).using(Pair::getSecond).output(); return Distinct.of(input) .mapped(Pair::getFirst) .windowBy(Time.of(Duration.ofSeconds(1))) .output(); }
@Override protected Dataset<Integer> getOutput(Dataset<Pair<Integer, Long>> input) { input = AssignEventTime.of(input).using(Pair::getSecond).output(); return Distinct.of(input) .mapped(Pair::getFirst) .windowBy(Time.of(Duration.ofSeconds(1))) .output(); }
@Override protected Dataset<Integer> getOutput(Dataset<Pair<Integer, Long>> input) { input = AssignEventTime.of(input).using(Pair::getSecond).output(); return Distinct.of(input) .mapped(Pair::getFirst) .windowBy(Time.of(Duration.ofSeconds(1))) .output(); }
Distinct.of(f.createInput(source, e -> e.time)) .mapped(e -> e) .windowBy(Time.of(Duration.ofMillis(5))) .output();
@Override protected Dataset<Triple<Instant, Type, Long>> getOutput( Dataset<Triple<Instant, Type, String>> input) { input = AssignEventTime.of(input).using(t -> t.getFirst().toEpochMilli()).output(); Dataset<ComparablePair<Type, String>> distinct = Distinct.of(input) .mapped(t -> new ComparablePair<>(t.getSecond(), t.getThird())) .windowBy(Time.of(Duration.ofHours(1))) .output(); Dataset<Pair<Type, Long>> reduced = ReduceByKey.of(distinct) .keyBy(ComparablePair::getFirst) .valueBy(p -> 1L) .combineBy(Sums.ofLongs()) .windowBy(Time.of(Duration.ofHours(1))) .output(); // extract window end timestamp return FlatMap .of(reduced) .using((Pair<Type, Long> p, Collector<Triple<Instant, Type, Long>> ctx) -> { long windowEnd = ((TimeInterval) ctx.getWindow()).getEndMillis(); ctx.collect(Triple.of(Instant.ofEpochMilli(windowEnd), p.getFirst(), p.getSecond())); }) .output(); }
@Override protected Dataset<Triple<Instant, Type, Long>> getOutput( Dataset<Triple<Instant, Type, String>> input) { input = AssignEventTime.of(input).using(t -> t.getFirst().toEpochMilli()).output(); Dataset<ComparablePair<Type, String>> distinct = Distinct.of(input) .mapped(t -> new ComparablePair<>(t.getSecond(), t.getThird())) .windowBy(Time.of(Duration.ofHours(1))) .output(); Dataset<Pair<Type, Long>> reduced = ReduceByKey.of(distinct) .keyBy(ComparablePair::getFirst) .valueBy(p -> 1L) .combineBy(Sums.ofLongs()) .windowBy(Time.of(Duration.ofHours(1))) .output(); // extract window end timestamp return FlatMap .of(reduced) .using((Pair<Type, Long> p, Collector<Triple<Instant, Type, Long>> ctx) -> { long windowEnd = ((TimeInterval) ctx.getWindow()).getEndMillis(); ctx.collect(Triple.of(Instant.ofEpochMilli(windowEnd), p.getFirst(), p.getSecond())); }) .output(); }
@Test public void testWindow_applyIf() { Flow flow = Flow.create("TEST"); Dataset<String> dataset = Util.createMockDataset(flow, 3); Distinct.of(dataset) .applyIf(true, b -> b.windowBy(Time.of(Duration.ofHours(1)))) .output(); Distinct distinct = (Distinct) flow.operators().iterator().next(); assertTrue(distinct.getWindowing() instanceof Time); }
@Override protected Dataset<Integer> getOutput(Dataset<Pair<Integer, Long>> input) { input = AssignEventTime.of(input).using(Pair::getSecond).output(); return Distinct.of(input) .mapped(Pair::getFirst) .windowBy(Time.of(Duration.ofSeconds(1))) .output(); }