/** * Attaches a stage that performs the given aggregate operation over all * the items that belong to a given window. Once the window is complete, it * emits a {@code TimestampedItem} with the result of the aggregate * operation and the timestamp denoting the window's ending time. * * @see com.hazelcast.jet.aggregate.AggregateOperations AggregateOperations * @param aggrOp the aggregate operation to perform * @param <R> the type of the result */ @Nonnull default <R> StreamStage<TimestampedItem<R>> aggregate( @Nonnull AggregateOperation1<? super T, ?, R> aggrOp ) { return aggregate(aggrOp, TimestampedItem::fromWindowResult); }
/** * Attaches a stage that passes through just the items that are distinct * within their window (no two items emitted for a window are equal). There * is no guarantee which one of the items with the same key will pass * through. The stage emits results in the form of {@link TimestampedItem * TimestampedItem(windowEnd, distinctItem)}. * * @return the newly attached stage */ @Nonnull default StreamStage<TimestampedItem<T>> distinct() { return groupingKey(wholeItem()).distinct(); }
/** * Attaches a stage that performs the given aggregate operation over all * the items that belong to the same window. It receives the items from * both this stage and {@code stage1}. Once a given window is complete, it * emits a {@link TimestampedItem} with the result of the aggregate * operation and the timestamp denoting the window's ending time. * <p> * This variant requires you to provide a two-input aggregate operation * (refer to its {@linkplain AggregateOperation2 Javadoc} for a simple * example). If you can express your logic in terms of two single-input * aggregate operations, one for each input stream, then you should use * {@link #aggregate2(AggregateOperation1, StreamStage, AggregateOperation1) * stage0.aggregate2(aggrOp0, stage1, aggrOp1)} because it offers a simpler * API and you can use the already defined single-input operations. Use * this variant only when you have the need to implement an aggregate * operation that combines the input streams into the same accumulator. * * @see com.hazelcast.jet.aggregate.AggregateOperations AggregateOperations * @param aggrOp the aggregate operation to perform * @param <T1> type of items in {@code stage1} * @param <R> type of the result */ @Nonnull default <T1, R> StreamStage<TimestampedItem<R>> aggregate2( @Nonnull StreamStage<T1> stage1, @Nonnull AggregateOperation2<? super T, ? super T1, ?, R> aggrOp ) { return aggregate2(stage1, aggrOp, TimestampedItem::fromWindowResult); }
AggregateOperation1<? super T, ?, ? extends R0> aggrOp ) { return new WindowAggregateBuilder<>(streamStage(), aggrOp, windowDefinition());
@Nonnull AggregateOperation3<? super T, ? super T1, ? super T2, ?, R> aggrOp ) { return aggregate3(stage1, stage2, aggrOp, TimestampedItem::fromWindowResult);
return new WindowAggregateBuilder1<>(streamStage(), windowDefinition());
AggregateOperation3<T, T1, T2, ?, Tuple3<R0, R1, R2>> aggrOp = aggregateOperation3(aggrOp0, aggrOp1, aggrOp2, Tuple3::tuple3); return aggregate3(stage1, stage2, aggrOp, (start, end, t3) -> new TimestampedItem<>(end, t3));
/** * Builds and returns the Pipeline which represents the actual computation. */ private static Pipeline buildPipeline(String modelPath) { Pipeline pipeline = Pipeline.create(); pipeline.drawFrom(WebcamSource.webcam(500)) .mapUsingContext(classifierContext(modelPath), (ctx, img) -> { Entry<String, Double> classification = classifyWithModel(ctx, img); return tuple3(img, classification.getKey(), classification.getValue()); } ) .window(tumbling(1000)) .aggregate(maxBy(comparingDouble(Tuple3::f2))) .drainTo(buildGUISink()); return pipeline; }
/** * Attaches a stage that passes through just the items that are distinct * within their window (no two items emitted for a window are equal). There * is no guarantee among the items with the same key which one it will pass * through. To create the item to emit, the stage calls the supplied {@code * mapToOutputFn}. * * @param mapToOutputFn function that returns the items to emit * @return the newly attached stage */ @Nonnull default <R> StreamStage<R> distinct(@Nonnull WindowResultFunction<? super T, ? extends R> mapToOutputFn) { return groupingKey(wholeItem()).distinct(mapToOutputFn); }
/** * Attaches a stage that performs the given co-aggregate operation over the * items from both this stage and {@code stage1} you supply. It performs * the aggregation separately for each input stage: {@code aggrOp0} on this * stage and {@code aggrOp1} on {@code stage1}. Once it has received all * the items belonging to a window, it emits a {@code * TimestampedItem(Tuple2(result0, result1))}. * <p> * The aggregating stage emits a single item for each completed window. * * @see com.hazelcast.jet.aggregate.AggregateOperations AggregateOperations * @param aggrOp0 aggregate operation to perform on this stage * @param stage1 the other stage * @param aggrOp1 aggregate operation to perform on the other stage * @param <T1> type of the items in the other stage * @param <R0> type of the aggregated result for this stage * @param <R1> type of the aggregated result for the other stage */ @Nonnull default <T1, R0, R1> StreamStage<TimestampedItem<Tuple2<R0, R1>>> aggregate2( @Nonnull AggregateOperation1<? super T, ?, ? extends R0> aggrOp0, @Nonnull StreamStage<T1> stage1, @Nonnull AggregateOperation1<? super T1, ?, ? extends R1> aggrOp1 ) { AggregateOperation2<T, T1, ?, Tuple2<R0, R1>> aggrOp = aggregateOperation2(aggrOp0, aggrOp1, Tuple2::tuple2); return aggregate2(stage1, aggrOp, (start, end, t2) -> new TimestampedItem<>(end, t2)); }
@Nonnull WindowResult3Function<? super R0, ? super R1, ? super R2, ? extends OUT> mapToOutputFn ) { return aggregate3(stage1, stage2, aggregateOperation3(aggrOp0, aggrOp1, aggrOp2, Tuple3::tuple3), (start, end, t3) -> mapToOutputFn.apply(start, end, t3.f0(), t3.f1(), t3.f2()));
@SuppressWarnings("Convert2MethodRef") // https://bugs.openjdk.java.net/browse/JDK-8154236 private static Pipeline aggregate() { Pipeline p = Pipeline.create(); p.drawFrom(Sources.<PageVisit, Integer, PageVisit>mapJournal(PAGE_VISIT, mapPutEvents(), mapEventNewValue(), START_FROM_OLDEST)) .addTimestamps(pv -> pv.timestamp(), 100) .window(sliding(10, 1)) .aggregate(counting()) .drainTo(Sinks.logger()); return p; }
@SuppressWarnings("Convert2MethodRef") // https://bugs.openjdk.java.net/browse/JDK-8154236 private static Pipeline groupAndAggregate() { Pipeline p = Pipeline.create(); p.drawFrom(Sources.<PageVisit, Integer, PageVisit>mapJournal(PAGE_VISIT, mapPutEvents(), mapEventNewValue(), START_FROM_OLDEST)) .addTimestamps(pv -> pv.timestamp(), 100) .window(sliding(10, 1)) .groupingKey(pv -> pv.userId()) .aggregate(toList()) .drainTo(Sinks.logger()); return p; }
/** * Attaches a stage that performs the given co-aggregate operation over the * items from both this stage and {@code stage1} you supply. It performs * the aggregation separately for each input stage: {@code aggrOp0} on this * stage and {@code aggrOp1} on {@code stage1}. Once it has received all * the items belonging to a window, it calls the supplied {@code * mapToOutputFn} with the aggregation results to create the item to emit. * <p> * The aggregating stage emits a single item for each completed window. * * @see com.hazelcast.jet.aggregate.AggregateOperations AggregateOperations * @param aggrOp0 aggregate operation to perform on this stage * @param stage1 the other stage * @param aggrOp1 aggregate operation to perform on the other stage * @param mapToOutputFn function to apply to the aggregated results * @param <T1> type of the items in the other stage * @param <R0> type of the aggregated result for this stage * @param <R1> type of the aggregated result for the other stage * @param <OUT> the output item type */ @Nonnull default <T1, R0, R1, OUT> StreamStage<OUT> aggregate2( @Nonnull AggregateOperation1<? super T, ?, ? extends R0> aggrOp0, @Nonnull StreamStage<T1> stage1, @Nonnull AggregateOperation1<? super T1, ?, ? extends R1> aggrOp1, @Nonnull WindowResult2Function<? super R0, ? super R1, ? extends OUT> mapToOutputFn ) { return aggregate2(stage1, aggregateOperation2(aggrOp0, aggrOp1, Tuple2::tuple2), (start, end, r) -> mapToOutputFn.apply(start, end, r.f0(), r.f1())); }
/** * This code is the main point of the sample: use the source builder to * create an HTTP source connector, then create a Jet pipeline that * performs windowed aggregation over its data. */ private static Pipeline buildPipeline() { StreamSource<TimestampedItem<Long>> usedMemorySource = SourceBuilder .timestampedStream("used-memory", x -> new PollHttp()) .fillBufferFn(PollHttp::fillBuffer) .destroyFn(PollHttp::close) .build(); Pipeline p = Pipeline.create(); p.drawFrom(usedMemorySource) .window(sliding(100, 20)) .aggregate(linearTrend(TimestampedItem::timestamp, TimestampedItem::item)) .map(tsItem -> entry(tsItem.timestamp(), tsItem.item())) .drainTo(Sinks.map(MAP_NAME)); return p; }