/** * Assumes as inputs a {@link BroadcastStream} and a {@link KeyedStream} and applies the given * {@link KeyedBroadcastProcessFunction} on them, thereby creating a transformed output stream. * * @param function The {@link KeyedBroadcastProcessFunction} that is called for each element in the stream. * @param <KS> The type of the keys in the keyed stream. * @param <OUT> The type of the output elements. * @return The transformed {@link DataStream}. */ @PublicEvolving public <KS, OUT> SingleOutputStreamOperator<OUT> process(final KeyedBroadcastProcessFunction<KS, IN1, IN2, OUT> function) { TypeInformation<OUT> outTypeInfo = TypeExtractor.getBinaryOperatorReturnType( function, KeyedBroadcastProcessFunction.class, 1, 2, 3, TypeExtractor.NO_INDEX, getType1(), getType2(), Utils.getCallLocationName(), true); return process(function, outTypeInfo); }
/** * Assumes as inputs a {@link BroadcastStream} and a non-keyed {@link DataStream} and applies the given * {@link BroadcastProcessFunction} on them, thereby creating a transformed output stream. * * @param function The {@link BroadcastProcessFunction} that is called for each element in the stream. * @param <OUT> The type of the output elements. * @return The transformed {@link DataStream}. */ @PublicEvolving public <OUT> SingleOutputStreamOperator<OUT> process(final BroadcastProcessFunction<IN1, IN2, OUT> function) { TypeInformation<OUT> outTypeInfo = TypeExtractor.getBinaryOperatorReturnType( function, BroadcastProcessFunction.class, 0, 1, 2, TypeExtractor.NO_INDEX, getType1(), getType2(), Utils.getCallLocationName(), true); return process(function, outTypeInfo); }
final DataStream<String> output = srcOne.connect(broadcast).process( new TestKeyedBroadcastProcessFunction(100000L, expected));
final DataStream<String> output = srcOne.connect(broadcast).process( new TestBroadcastProcessFunction());
bcStream.process( new KeyedBroadcastProcessFunction<String, Long, String, String>() { @Override
bcStream.process( new BroadcastProcessFunction<Long, String, String>() { @Override
/** * Assumes as inputs a {@link BroadcastStream} and a non-keyed {@link DataStream} and applies the given * {@link BroadcastProcessFunction} on them, thereby creating a transformed output stream. * * @param function The {@link BroadcastProcessFunction} that is called for each element in the stream. * @param <OUT> The type of the output elements. * @return The transformed {@link DataStream}. */ @PublicEvolving public <OUT> SingleOutputStreamOperator<OUT> process(final BroadcastProcessFunction<IN1, IN2, OUT> function) { TypeInformation<OUT> outTypeInfo = TypeExtractor.getBinaryOperatorReturnType( function, BroadcastProcessFunction.class, 0, 1, 2, TypeExtractor.NO_INDEX, getType1(), getType2(), Utils.getCallLocationName(), true); return process(function, outTypeInfo); }
/** * Assumes as inputs a {@link BroadcastStream} and a {@link KeyedStream} and applies the given * {@link KeyedBroadcastProcessFunction} on them, thereby creating a transformed output stream. * * @param function The {@link KeyedBroadcastProcessFunction} that is called for each element in the stream. * @param <KS> The type of the keys in the keyed stream. * @param <OUT> The type of the output elements. * @return The transformed {@link DataStream}. */ @PublicEvolving public <KS, OUT> SingleOutputStreamOperator<OUT> process(final KeyedBroadcastProcessFunction<KS, IN1, IN2, OUT> function) { TypeInformation<OUT> outTypeInfo = TypeExtractor.getBinaryOperatorReturnType( function, KeyedBroadcastProcessFunction.class, 1, 2, 3, TypeExtractor.NO_INDEX, getType1(), getType2(), Utils.getCallLocationName(), true); return process(function, outTypeInfo); }
/** * Assumes as inputs a {@link BroadcastStream} and a non-keyed {@link DataStream} and applies the given * {@link BroadcastProcessFunction} on them, thereby creating a transformed output stream. * * @param function The {@link BroadcastProcessFunction} that is called for each element in the stream. * @param <OUT> The type of the output elements. * @return The transformed {@link DataStream}. */ @PublicEvolving public <OUT> SingleOutputStreamOperator<OUT> process(final BroadcastProcessFunction<IN1, IN2, OUT> function) { TypeInformation<OUT> outTypeInfo = TypeExtractor.getBinaryOperatorReturnType( function, BroadcastProcessFunction.class, 0, 1, 2, TypeExtractor.NO_INDEX, getType1(), getType2(), Utils.getCallLocationName(), true); return process(function, outTypeInfo); }
/** * Assumes as inputs a {@link BroadcastStream} and a {@link KeyedStream} and applies the given * {@link KeyedBroadcastProcessFunction} on them, thereby creating a transformed output stream. * * @param function The {@link KeyedBroadcastProcessFunction} that is called for each element in the stream. * @param <KS> The type of the keys in the keyed stream. * @param <OUT> The type of the output elements. * @return The transformed {@link DataStream}. */ @PublicEvolving public <KS, OUT> SingleOutputStreamOperator<OUT> process(final KeyedBroadcastProcessFunction<KS, IN1, IN2, OUT> function) { TypeInformation<OUT> outTypeInfo = TypeExtractor.getBinaryOperatorReturnType( function, KeyedBroadcastProcessFunction.class, 1, 2, 3, TypeExtractor.NO_INDEX, getType1(), getType2(), Utils.getCallLocationName(), true); return process(function, outTypeInfo); }
.process(new MatchFunction());
public DataStream<String> constructTestPipeline(DataStream<String> source) { OutputTag<Integer> filtered = new OutputTag<>("filter", BasicTypeInfo.INT_TYPE_INFO); OutputTag<Integer> process = new OutputTag<>("process", BasicTypeInfo.INT_TYPE_INFO); SingleOutputStreamOperator<String> input = source.process(new ProcessFunction<String, String>() { private static final long serialVersionUID = 1L; @Override public void processElement(String s, Context ctx, Collector<String> out) throws Exception { if (s.startsWith("filter ")) { ctx.output(filtered, Integer.parseInt(s.substring(7))); } else if (s.startsWith("process ")) { ctx.output(process, Integer.parseInt(s.substring(8))); } else { throw new RuntimeException("oOoO"); } } }); BroadcastStream<Integer> broadcast = input.getSideOutput(filtered).broadcast(bcstate); return input.getSideOutput(process).keyBy(i -> i).connect(broadcast).process(new BroadcastProcessor(bcstate)) .uid("stateful"); }
public static void main(String[] args) throws Exception { ParameterTool params = ParameterTool.fromArgs(args); final String input = params.get("input", ExerciseBase.pathToRideData); final int maxEventDelay = 60; // events are out of order by at most 60 seconds final int servingSpeedFactor = 1800; // 30 minutes worth of events are served every second // set up streaming execution environment StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment(); env.setStreamTimeCharacteristic(TimeCharacteristic.EventTime); env.setParallelism(ExerciseBase.parallelism); // setup a stream of taxi rides DataStream<TaxiRide> rides = env.addSource(rideSourceOrTest(new TaxiRideSource(input, maxEventDelay, servingSpeedFactor))); // add a socket source for the query stream BroadcastStream<String> queryStream = env .addSource(stringSourceOrTest(new SocketTextStreamFunction("localhost", 9999, "\n", -1))) .broadcast(queryDescriptor); // connect the two streams and process queries DataStream<Tuple2<String, String>> results = rides .keyBy((TaxiRide ride) -> ride.taxiId) .connect(queryStream) .process(new QueryProcessor()); printOrTest(results); env.execute("Taxi Query"); }
public static void main(String[] args) throws Exception { ParameterTool params = ParameterTool.fromArgs(args); final String input = params.get("input", ExerciseBase.pathToRideData); final int maxEventDelay = 60; // events are out of order by at most 60 seconds final int servingSpeedFactor = 600; // 10 minutes worth of events are served every second // In this simple case we need a broadcast state descriptor, but aren't going to // use it to store anything. final MapStateDescriptor<Long, Long> dummyBroadcastState = new MapStateDescriptor<>( "dummy", BasicTypeInfo.LONG_TYPE_INFO, BasicTypeInfo.LONG_TYPE_INFO ); // set up streaming execution environment StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment(); env.setStreamTimeCharacteristic(TimeCharacteristic.EventTime); env.setParallelism(ExerciseBase.parallelism); DataStream<TaxiRide> rides = env.addSource(new TaxiRideSource(input, maxEventDelay, servingSpeedFactor)); // add a socket source BroadcastStream<String> queryStream = env.socketTextStream("localhost", 9999) // EXERCISE QUESTION: Is this needed? // .assignTimestampsAndWatermarks(new QueryStreamAssigner()) .broadcast(dummyBroadcastState); DataStream<TaxiRide> reports = rides .keyBy((TaxiRide ride) -> ride.taxiId) .connect(queryStream) .process(new QueryFunction()); printOrTest(reports); env.execute("Ongoing Rides"); }
public static void main(String[] args) throws Exception { ParameterTool params = ParameterTool.fromArgs(args); final String input = params.get("input", ExerciseBase.pathToRideData); final int maxEventDelay = 60; // events are out of order by at most 60 seconds final int servingSpeedFactor = 600; // 10 minutes worth of events are served every second // In this simple case we need a broadcast state descriptor, but aren't going to // use it to store anything. final MapStateDescriptor<Long, Long> dummyBroadcastState = new MapStateDescriptor<>( "dummy", BasicTypeInfo.LONG_TYPE_INFO, BasicTypeInfo.LONG_TYPE_INFO ); // set up streaming execution environment StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment(); env.setStreamTimeCharacteristic(TimeCharacteristic.EventTime); env.setParallelism(ExerciseBase.parallelism); DataStream<TaxiRide> rides = env.addSource(new TaxiRideSource(input, maxEventDelay, servingSpeedFactor)); // add a socket source BroadcastStream<String> queryStream = env.socketTextStream("localhost", 9999) .assignTimestampsAndWatermarks(new QueryStreamAssigner()) .broadcast(dummyBroadcastState); DataStream<TaxiRide> reports = rides .keyBy((TaxiRide ride) -> ride.taxiId) .connect(queryStream) .process(new QueryFunction()); printOrTest(reports); env.execute("Ongoing Rides"); }
public static void main(String[] args) throws Exception { ParameterTool params = ParameterTool.fromArgs(args); final String input = params.get("input", ExerciseBase.pathToRideData); final int maxEventDelay = 60; // events are out of order by at most 60 seconds final int servingSpeedFactor = 1800; // 30 minutes worth of events are served every second // set up streaming execution environment StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment(); env.setStreamTimeCharacteristic(TimeCharacteristic.EventTime); env.setParallelism(ExerciseBase.parallelism); // setup a stream of taxi rides DataStream<TaxiRide> rides = env.addSource(rideSourceOrTest(new TaxiRideSource(input, maxEventDelay, servingSpeedFactor))); // add a socket source for the query stream BroadcastStream<String> queryStream = env .addSource(stringSourceOrTest(new SocketTextStreamFunction("localhost", 9999, "\n", -1))) .assignTimestampsAndWatermarks(new QueryStreamAssigner()) .broadcast(queryDescriptor); // connect the two streams and process queries DataStream<Tuple2<String, String>> results = rides .keyBy((TaxiRide ride) -> ride.taxiId) .connect(queryStream) .process(new QueryProcessor()); printOrTest(results); env.execute("Taxi Query"); }
.keyBy((TaxiRide ride) -> ride.taxiId) .connect(queryStream) .process(new QueryFunction());
.keyBy((TaxiRide ride) -> ride.taxiId) .connect(queryStream) .process(new QueryFunction());
.keyBy((TaxiRide ride) -> ride.taxiId) .connect(queryStream) .process(new QueryFunction());