public StreamExecutionEnvironment getExecutionEnvironment() { return input.getExecutionEnvironment(); }
public StreamExecutionEnvironment getExecutionEnvironment() { return input.getExecutionEnvironment(); }
@PublicEvolving public WindowedStream(KeyedStream<T, K> input, WindowAssigner<? super T, W> windowAssigner) { this.input = input; this.windowAssigner = windowAssigner; this.trigger = windowAssigner.getDefaultTrigger(input.getExecutionEnvironment()); }
/** * Send late arriving data to the side output identified by the given {@link OutputTag}. Data * is considered late after the watermark has passed the end of the window plus the allowed * lateness set using {@link #allowedLateness(Time)}. * * <p>You can get the stream of late data using * {@link SingleOutputStreamOperator#getSideOutput(OutputTag)} on the * {@link SingleOutputStreamOperator} resulting from the windowed operation * with the same {@link OutputTag}. */ @PublicEvolving public AllWindowedStream<T, W> sideOutputLateData(OutputTag<T> outputTag) { Preconditions.checkNotNull(outputTag, "Side output tag must not be null."); this.lateDataOutputTag = input.getExecutionEnvironment().clean(outputTag); return this; }
/** * Applies the given window function to each window. The window function is called for each * evaluation of the window for each key individually. The output of the window function is * interpreted as a regular non-windowed stream. * * <p>Note that this function requires that all data in the windows is buffered until the window * is evaluated, as the function provides no means of incremental aggregation. * * @param function The window function. * @param resultType Type information for the result type of the window function * @return The data stream that is the result of applying the window function to the window. */ public <R> SingleOutputStreamOperator<R> apply(WindowFunction<T, R, K, W> function, TypeInformation<R> resultType) { function = input.getExecutionEnvironment().clean(function); return apply(new InternalIterableWindowFunction<>(function), resultType, function); }
/** * Applies a reduce function to the window. The window function is called for each evaluation * of the window for each key individually. The output of the reduce function is interpreted * as a regular non-windowed stream. * * <p>This window will try and incrementally aggregate data as much as the window policies * permit. For example, tumbling time windows can aggregate the data, meaning that only one * element per key is stored. Sliding time windows will aggregate on the granularity of the * slide interval, so a few elements are stored per key (one per slide interval). * Custom windows may not be able to incrementally aggregate, or may need to store extra values * in an aggregation tree. * * @param function The reduce function. * @return The data stream that is the result of applying the reduce function to the window. */ @SuppressWarnings("unchecked") public SingleOutputStreamOperator<T> reduce(ReduceFunction<T> function) { if (function instanceof RichFunction) { throw new UnsupportedOperationException("ReduceFunction of reduce can not be a RichFunction. " + "Please use reduce(ReduceFunction, WindowFunction) instead."); } //clean the closure function = input.getExecutionEnvironment().clean(function); return reduce(function, new PassThroughWindowFunction<K, W, T>()); }
/** * Send late arriving data to the side output identified by the given {@link OutputTag}. Data * is considered late after the watermark has passed the end of the window plus the allowed * lateness set using {@link #allowedLateness(Time)}. * * <p>You can get the stream of late data using * {@link SingleOutputStreamOperator#getSideOutput(OutputTag)} on the * {@link SingleOutputStreamOperator} resulting from the windowed operation * with the same {@link OutputTag}. */ @PublicEvolving public WindowedStream<T, K, W> sideOutputLateData(OutputTag<T> outputTag) { Preconditions.checkNotNull(outputTag, "Side output tag must not be null."); this.lateDataOutputTag = input.getExecutionEnvironment().clean(outputTag); return this; }
/** * Applies a reduce function to the window. The window function is called for each evaluation * of the window for each key individually. The output of the reduce function is interpreted * as a regular non-windowed stream. * * <p>This window will try and incrementally aggregate data as much as the window policies permit. * For example, tumbling time windows can aggregate the data, meaning that only one element per * key is stored. Sliding time windows will aggregate on the granularity of the slide interval, * so a few elements are stored per key (one per slide interval). * Custom windows may not be able to incrementally aggregate, or may need to store extra values * in an aggregation tree. * * @param function The reduce function. * @return The data stream that is the result of applying the reduce function to the window. */ @SuppressWarnings("unchecked") public SingleOutputStreamOperator<T> reduce(ReduceFunction<T> function) { if (function instanceof RichFunction) { throw new UnsupportedOperationException("ReduceFunction of reduce can not be a RichFunction. " + "Please use reduce(ReduceFunction, WindowFunction) instead."); } //clean the closure function = input.getExecutionEnvironment().clean(function); String callLocation = Utils.getCallLocationName(); String udfName = "AllWindowedStream." + callLocation; return reduce(function, new PassThroughAllWindowFunction<W, T>()); }
/** * Applies the given window function to each window. The window function is called for each * evaluation of the window. The output of the window function is * interpreted as a regular non-windowed stream. * * <p>Not that this function requires that all data in the windows is buffered until the window * is evaluated, as the function provides no means of incremental aggregation. * * @param function The window function. * @return The data stream that is the result of applying the window function to the window. */ public <R> SingleOutputStreamOperator<R> apply(AllWindowFunction<T, R, W> function, TypeInformation<R> resultType) { String callLocation = Utils.getCallLocationName(); function = input.getExecutionEnvironment().clean(function); return apply(new InternalIterableAllWindowFunction<>(function), resultType, callLocation); }
/** * Applies the given window function to each window. The window function is called for each * evaluation of the window for each key individually. The output of the window function is * interpreted as a regular non-windowed stream. * * <p>Note that this function requires that all data in the windows is buffered until the window * is evaluated, as the function provides no means of incremental aggregation. * * @param function The window function. * @param resultType Type information for the result type of the window function * @return The data stream that is the result of applying the window function to the window. */ @Internal public <R> SingleOutputStreamOperator<R> process(ProcessWindowFunction<T, R, K, W> function, TypeInformation<R> resultType) { function = input.getExecutionEnvironment().clean(function); return apply(new InternalIterableProcessWindowFunction<>(function), resultType, function); }
/** * Applies the given window function to each window. The window function is called for each * evaluation of the window. The output of the window function is * interpreted as a regular non-windowed stream. * * <p>Not that this function requires that all data in the windows is buffered until the window * is evaluated, as the function provides no means of incremental aggregation. * * @param function The process window function. * @return The data stream that is the result of applying the window function to the window. */ @PublicEvolving public <R> SingleOutputStreamOperator<R> process(ProcessAllWindowFunction<T, R, W> function, TypeInformation<R> resultType) { String callLocation = Utils.getCallLocationName(); function = input.getExecutionEnvironment().clean(function); return apply(new InternalIterableProcessAllWindowFunction<>(function), resultType, callLocation); }
/** * Applies the given window function to each window. The window function is called for each * evaluation of the window. The output of the window function is * interpreted as a regular non-windowed stream. * * <p>Not that this function requires that all data in the windows is buffered until the window * is evaluated, as the function provides no means of incremental aggregation. * * @param function The window function. * @return The data stream that is the result of applying the window function to the window. */ public <R> SingleOutputStreamOperator<R> apply(AllWindowFunction<T, R, W> function) { String callLocation = Utils.getCallLocationName(); function = input.getExecutionEnvironment().clean(function); TypeInformation<R> resultType = getAllWindowFunctionReturnType(function, getInputType()); return apply(new InternalIterableAllWindowFunction<>(function), resultType, callLocation); }
/** * Applies the given window function to each window. The window function is called for each * evaluation of the window. The output of the window function is * interpreted as a regular non-windowed stream. * * <p>Not that this function requires that all data in the windows is buffered until the window * is evaluated, as the function provides no means of incremental aggregation. * * @param function The process window function. * @return The data stream that is the result of applying the window function to the window. */ @PublicEvolving public <R> SingleOutputStreamOperator<R> process(ProcessAllWindowFunction<T, R, W> function) { String callLocation = Utils.getCallLocationName(); function = input.getExecutionEnvironment().clean(function); TypeInformation<R> resultType = getProcessAllWindowFunctionReturnType(function, getInputType()); return apply(new InternalIterableProcessAllWindowFunction<>(function), resultType, callLocation); }
/** * Specifies the time boundaries over which the join operation works, so that * <pre>leftElement.timestamp + lowerBound <= rightElement.timestamp <= leftElement.timestamp + upperBound</pre> * By default both the lower and the upper bound are inclusive. This can be configured * with {@link IntervalJoined#lowerBoundExclusive()} and * {@link IntervalJoined#upperBoundExclusive()} * * @param lowerBound The lower bound. Needs to be smaller than or equal to the upperBound * @param upperBound The upper bound. Needs to be bigger than or equal to the lowerBound */ @PublicEvolving public IntervalJoined<T1, T2, KEY> between(Time lowerBound, Time upperBound) { TimeCharacteristic timeCharacteristic = streamOne.getExecutionEnvironment().getStreamTimeCharacteristic(); if (timeCharacteristic != TimeCharacteristic.EventTime) { throw new UnsupportedTimeCharacteristicException("Time-bounded stream joins are only supported in event time"); } checkNotNull(lowerBound, "A lower bound needs to be provided for a time-bounded join"); checkNotNull(upperBound, "An upper bound needs to be provided for a time-bounded join"); return new IntervalJoined<>( streamOne, streamTwo, lowerBound.toMilliseconds(), upperBound.toMilliseconds(), true, true ); } }
function = input.getExecutionEnvironment().clean(function); foldFunction = input.getExecutionEnvironment().clean(foldFunction);
Preconditions.checkNotNull(outputType); final ProcessJoinFunction<IN1, IN2, OUT> cleanedUdf = left.getExecutionEnvironment().clean(processJoinFunction);
function = input.getExecutionEnvironment().clean(function); reduceFunction = input.getExecutionEnvironment().clean(reduceFunction);
function = input.getExecutionEnvironment().clean(function); reduceFunction = input.getExecutionEnvironment().clean(reduceFunction);
function = input.getExecutionEnvironment().clean(function); reduceFunction = input.getExecutionEnvironment().clean(reduceFunction);
function = input.getExecutionEnvironment().clean(function); reduceFunction = input.getExecutionEnvironment().clean(reduceFunction);