/** * Reinterprets the given {@link DataStream} as a {@link KeyedStream}, which extracts keys with the given * {@link KeySelector}. * * <p>IMPORTANT: For every partition of the base stream, the keys of events in the base stream must be * partitioned exactly in the same way as if it was created through a {@link DataStream#keyBy(KeySelector)}. * * @param stream The data stream to reinterpret. For every partition, this stream must be partitioned exactly * in the same way as if it was created through a {@link DataStream#keyBy(KeySelector)}. * @param keySelector Function that defines how keys are extracted from the data stream. * @param <T> Type of events in the data stream. * @param <K> Type of the extracted keys. * @return The reinterpretation of the {@link DataStream} as a {@link KeyedStream}. */ public static <T, K> KeyedStream<T, K> reinterpretAsKeyedStream( DataStream<T> stream, KeySelector<T, K> keySelector) { return reinterpretAsKeyedStream( stream, keySelector, TypeExtractor.getKeySelectorTypes(keySelector, stream.getType())); }
@Test public void testCollect() throws Exception { final StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment(); env.setParallelism(1); final long n = 10; DataStream<Long> stream = env.generateSequence(1, n); long i = 1; for (Iterator<Long> it = DataStreamUtils.collect(stream); it.hasNext(); ) { long x = it.next(); assertEquals("received wrong element", i, x); i++; } assertEquals("received wrong number of elements", n + 1, i); } }
@Test public void testProcessingTimeWithWindow() throws Exception { StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment(); env.setParallelism(1); DataStream<Integer> input = env.fromElements(1, 2); Pattern<Integer, ?> pattern = Pattern.<Integer>begin("start").followedByAny("end").within(Time.days(1)); DataStream<Integer> result = CEP.pattern(input, pattern).select(new PatternSelectFunction<Integer, Integer>() { @Override public Integer select(Map<String, List<Integer>> pattern) throws Exception { return pattern.get("start").get(0) + pattern.get("end").get(0); } }); List<Integer> resultList = new ArrayList<>(); DataStreamUtils.collect(result).forEachRemaining(resultList::add); assertEquals(Arrays.asList(3), resultList); }
DataStreamUtils.reinterpretAsKeyedStream( env.addSource(new FromPartitionFileSource(partitionFiles)), (KeySelector<Tuple2<Integer, Integer>, Integer>) value -> value.f0,
@Test public void testSimplePatternWithSingleState() throws Exception { StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment(); DataStream<Tuple2<Integer, Integer>> input = env.fromElements( new Tuple2<>(0, 1), new Tuple2<>(0, 2)); Pattern<Tuple2<Integer, Integer>, ?> pattern = Pattern.<Tuple2<Integer, Integer>>begin("start") .where(new SimpleCondition<Tuple2<Integer, Integer>>() { @Override public boolean filter(Tuple2<Integer, Integer> rec) throws Exception { return rec.f1 == 1; } }); PatternStream<Tuple2<Integer, Integer>> pStream = CEP.pattern(input, pattern); DataStream<Tuple2<Integer, Integer>> result = pStream.select(new PatternSelectFunction<Tuple2<Integer, Integer>, Tuple2<Integer, Integer>>() { @Override public Tuple2<Integer, Integer> select(Map<String, List<Tuple2<Integer, Integer>>> pattern) throws Exception { return pattern.get("start").get(0); } }); List<Tuple2<Integer, Integer>> resultList = new ArrayList<>(); DataStreamUtils.collect(result).forEachRemaining(resultList::add); assertEquals(Arrays.asList(new Tuple2<>(0, 1)), resultList); }
/** * Reinterprets the given {@link DataStream} as a {@link KeyedStream}, which extracts keys with the given * {@link KeySelector}. * * <p>IMPORTANT: For every partition of the base stream, the keys of events in the base stream must be * partitioned exactly in the same way as if it was created through a {@link DataStream#keyBy(KeySelector)}. * * @param stream The data stream to reinterpret. For every partition, this stream must be partitioned exactly * in the same way as if it was created through a {@link DataStream#keyBy(KeySelector)}. * @param keySelector Function that defines how keys are extracted from the data stream. * @param <T> Type of events in the data stream. * @param <K> Type of the extracted keys. * @return The reinterpretation of the {@link DataStream} as a {@link KeyedStream}. */ public static <T, K> KeyedStream<T, K> reinterpretAsKeyedStream( DataStream<T> stream, KeySelector<T, K> keySelector) { return reinterpretAsKeyedStream( stream, keySelector, TypeExtractor.getKeySelectorTypes(keySelector, stream.getType())); }
@Test public void testSimpleAfterMatchSkip() throws Exception { StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment(); DataStream<Tuple2<Integer, String>> input = env.fromElements( new Tuple2<>(1, "a"), new Tuple2<>(2, "a"), new Tuple2<>(3, "a"), new Tuple2<>(4, "a")); Pattern<Tuple2<Integer, String>, ?> pattern = Pattern.<Tuple2<Integer, String>>begin("start", AfterMatchSkipStrategy.skipPastLastEvent()) .where(new SimpleCondition<Tuple2<Integer, String>>() { @Override public boolean filter(Tuple2<Integer, String> rec) throws Exception { return rec.f1.equals("a"); } }).times(2); PatternStream<Tuple2<Integer, String>> pStream = CEP.pattern(input, pattern); DataStream<Tuple2<Integer, String>> result = pStream.select(new PatternSelectFunction<Tuple2<Integer, String>, Tuple2<Integer, String>>() { @Override public Tuple2<Integer, String> select(Map<String, List<Tuple2<Integer, String>>> pattern) throws Exception { return pattern.get("start").get(0); } }); List<Tuple2<Integer, String>> resultList = new ArrayList<>(); DataStreamUtils.collect(result).forEachRemaining(resultList::add); resultList.sort(Comparator.comparing(tuple2 -> tuple2.toString())); List<Tuple2<Integer, String>> expected = Arrays.asList(Tuple2.of(1, "a"), Tuple2.of(3, "a")); assertEquals(expected, resultList); }
/** * Reinterprets the given {@link DataStream} as a {@link KeyedStream}, which extracts keys with the given * {@link KeySelector}. * * <p>IMPORTANT: For every partition of the base stream, the keys of events in the base stream must be * partitioned exactly in the same way as if it was created through a {@link DataStream#keyBy(KeySelector)}. * * @param stream The data stream to reinterpret. For every partition, this stream must be partitioned exactly * in the same way as if it was created through a {@link DataStream#keyBy(KeySelector)}. * @param keySelector Function that defines how keys are extracted from the data stream. * @param <T> Type of events in the data stream. * @param <K> Type of the extracted keys. * @return The reinterpretation of the {@link DataStream} as a {@link KeyedStream}. */ public static <T, K> KeyedStream<T, K> reinterpretAsKeyedStream( DataStream<T> stream, KeySelector<T, K> keySelector) { return reinterpretAsKeyedStream( stream, keySelector, TypeExtractor.getKeySelectorTypes(keySelector, stream.getType())); }
DataStreamUtils.collect(result).forEachRemaining(resultList::add);
DataStreamUtils.collect(result).forEachRemaining(resultList::add);
DataStreamUtils.collect(result).forEachRemaining(resultList::add);
DataStreamUtils.collect(result).forEachRemaining(resultList::add);
DataStreamUtils.collect(result).forEachRemaining(resultList::add);
DataStreamUtils.collect(result).forEachRemaining(resultList::add);
DataStreamUtils.collect(result).forEachRemaining(resultList::add);
DataStreamUtils.collect(result).forEachRemaining(resultList::add);
DataStreamUtils.collect(result).forEachRemaining(resultList::add);