.filter(dummyFilter).slotSharingGroup("isolated") .filter(dummyFilter).slotSharingGroup("default").disableChaining() .filter(dummyFilter).slotSharingGroup("group 1") .filter(dummyFilter).slotSharingGroup("isolated-2") .filter(dummyFilter).slotSharingGroup("default").disableChaining() .filter(dummyFilter).slotSharingGroup("group 2")
static DataStream<Tuple2<Long, String>> addKafkaSource( StreamExecutionEnvironment env, String brokers, String topic) { // configure Kafka consumer Properties props = new Properties(); props.setProperty("bootstrap.servers", brokers); // host:port props.setProperty("group.id", "flink-stream-test" + UUID.randomUUID()); // unique group props.setProperty("auto.offset.reset", "earliest");// Read topic from start // create a Kafka consumer FlinkKafkaConsumer010<SearchEventsParser.Query> consumer = new FlinkKafkaConsumer010<>(topic, new ParseSchema(), props); return env.addSource(consumer) .filter(q -> q != null && q.query != null && !q.query.isEmpty()) .map(q -> Tuple2.of(q.timestamp, q.query)) .returns(new TypeHint<Tuple2<Long, String>>() {}); }
public static void main(String[] args) throws Exception { final StreamExecutionEnvironment env = StreamExecutionEnvironment .getExecutionEnvironment().setParallelism(1); env.addSource(new LongSource()) .setParallelism(2) .filter(new RichFilterFunction<Integer>() { private Counter filterOutNumber; @Override public void open(Configuration parameters) throws Exception { filterOutNumber = getRuntimeContext() .getMetricGroup() .addGroup("minwenjun") .counter("filterOutNumber"); } @Override public boolean filter(Integer integer) throws Exception { if (integer % 2 == 0) { filterOutNumber.inc(); } return !(integer % 2 == 0); } }).print(); env.execute(); } }
public static void main(String[] args) throws Exception { ParameterTool params = ParameterTool.fromArgs(args); final String ridesFile = params.get("rides", pathToRideData); final String faresFile = params.get("fares", pathToFareData); final int delay = 60; // at most 60 seconds of delay final int servingSpeedFactor = 1800; // 30 minutes worth of events are served every second // set up streaming execution environment StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment(); env.setStreamTimeCharacteristic(TimeCharacteristic.EventTime); env.setParallelism(ExerciseBase.parallelism); DataStream<TaxiRide> rides = env .addSource(rideSourceOrTest(new TaxiRideSource(ridesFile, delay, servingSpeedFactor))) .filter((TaxiRide ride) -> ride.isStart) .keyBy("rideId"); DataStream<TaxiFare> fares = env .addSource(fareSourceOrTest(new TaxiFareSource(faresFile, delay, servingSpeedFactor))) .keyBy("rideId"); DataStream<Tuple2<TaxiRide, TaxiFare>> enrichedRides = rides .connect(fares) .flatMap(new EnrichmentFunction()); printOrTest(enrichedRides); env.execute("Join Rides with Fares (java RichCoFlatMap)"); }
public static void main(String[] args) throws Exception { ParameterTool params = ParameterTool.fromArgs(args); final String ridesFile = params.get("rides", ExerciseBase.pathToRideData); final String faresFile = params.get("fares", ExerciseBase.pathToFareData); final int servingSpeedFactor = 600; // 10 minutes worth of events are served every second // set up streaming execution environment StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment(); env.setStreamTimeCharacteristic(TimeCharacteristic.EventTime); env.setParallelism(ExerciseBase.parallelism); DataStream<TaxiRide> rides = env .addSource(rideSourceOrTest(new CheckpointedTaxiRideSource(ridesFile, servingSpeedFactor))) .filter((TaxiRide ride) -> (ride.isStart && (ride.rideId % 1000 != 0))) .keyBy(ride -> ride.rideId); DataStream<TaxiFare> fares = env .addSource(fareSourceOrTest(new CheckpointedTaxiFareSource(faresFile, servingSpeedFactor))) .keyBy(fare -> fare.rideId); SingleOutputStreamOperator processed = rides .connect(fares) .process(new EnrichmentFunction()); printOrTest(processed.getSideOutput(unmatchedFares)); env.execute("ExpiringStateExercise (java)"); }
public static void main(String[] args) throws Exception { ParameterTool params = ParameterTool.fromArgs(args); final String ridesFile = params.get("rides", pathToRideData); final String faresFile = params.get("fares", pathToFareData); final int delay = 60; // at most 60 seconds of delay final int servingSpeedFactor = 1800; // 30 minutes worth of events are served every second // set up streaming execution environment StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment(); env.setStreamTimeCharacteristic(TimeCharacteristic.EventTime); env.setParallelism(ExerciseBase.parallelism); DataStream<TaxiRide> rides = env .addSource(rideSourceOrTest(new TaxiRideSource(ridesFile, delay, servingSpeedFactor))) .filter((TaxiRide ride) -> ride.isStart) .keyBy("rideId"); DataStream<TaxiFare> fares = env .addSource(fareSourceOrTest(new TaxiFareSource(faresFile, delay, servingSpeedFactor))) .keyBy("rideId"); DataStream<Tuple2<TaxiRide, TaxiFare>> enrichedRides = rides .connect(fares) .flatMap(new EnrichmentFunction()); printOrTest(enrichedRides); env.execute("Join Rides with Fares (java RichCoFlatMap)"); }
public static void main(String[] args) throws Exception { ParameterTool params = ParameterTool.fromArgs(args); final String ridesFile = params.get("rides", ExerciseBase.pathToRideData); final String faresFile = params.get("fares", ExerciseBase.pathToFareData); final int servingSpeedFactor = 600; // 10 minutes worth of events are served every second // set up streaming execution environment StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment(); env.setStreamTimeCharacteristic(TimeCharacteristic.EventTime); env.setParallelism(ExerciseBase.parallelism); DataStream<TaxiRide> rides = env .addSource(rideSourceOrTest(new CheckpointedTaxiRideSource(ridesFile, servingSpeedFactor))) .filter((TaxiRide ride) -> (ride.isStart && (ride.rideId % 1000 != 0))) .keyBy("rideId"); DataStream<TaxiFare> fares = env .addSource(fareSourceOrTest(new CheckpointedTaxiFareSource(faresFile, servingSpeedFactor))) .keyBy("rideId"); SingleOutputStreamOperator processed = rides .connect(fares) .process(new EnrichmentFunction()); printOrTest(processed.getSideOutput(unmatchedFares)); env.execute("ExpiringStateSolution (java)"); }