public static <T> OneInputStreamOperatorTestHarness<Event, T> getCepTestHarness( CepOperator<Event, Integer, T> cepOperator) throws Exception { KeySelector<Event, Integer> keySelector = new TestKeySelector(); return new KeyedOneInputStreamOperatorTestHarness<>( cepOperator, keySelector, BasicTypeInfo.INT_TYPE_INFO); }
private static <OUT> OneInputStreamOperatorTestHarness<Tuple2<String, Integer>, OUT> createTestHarness(OneInputStreamOperator<Tuple2<String, Integer>, OUT> operator) throws Exception { return new KeyedOneInputStreamOperatorTestHarness<>(operator, new TupleKeySelector(), BasicTypeInfo.STRING_TYPE_INFO); }
new KeyedOneInputStreamOperatorTestHarness<>(operator, keySelector, keyType);
@Test public void testProcessingTimeTimers() throws Exception { LegacyKeyedProcessOperator<Integer, Integer, Integer> operator = new LegacyKeyedProcessOperator<>(new TriggeringFlatMapFunction(TimeDomain.PROCESSING_TIME)); OneInputStreamOperatorTestHarness<Integer, Integer> testHarness = new KeyedOneInputStreamOperatorTestHarness<>(operator, new IdentityKeySelector<Integer>(), BasicTypeInfo.INT_TYPE_INFO); testHarness.setup(); testHarness.open(); testHarness.processElement(new StreamRecord<>(17)); testHarness.setProcessingTime(5); ConcurrentLinkedQueue<Object> expectedOutput = new ConcurrentLinkedQueue<>(); expectedOutput.add(new StreamRecord<>(17)); expectedOutput.add(new StreamRecord<>(1777)); TestHarnessUtil.assertOutputEquals("Output was not correct.", expectedOutput, testHarness.getOutput()); testHarness.close(); }
@Test public void testStateDoesNotInterfere() throws Exception { TestOperator testOperator = new TestOperator(); KeyedOneInputStreamOperatorTestHarness<Integer, Tuple2<Integer, String>, String> testHarness = new KeyedOneInputStreamOperatorTestHarness<>(testOperator, new TestKeySelector(), BasicTypeInfo.INT_TYPE_INFO); testHarness.open(); testHarness.processElement(new Tuple2<>(0, "SET_STATE:HELLO"), 0); testHarness.processElement(new Tuple2<>(1, "SET_STATE:CIAO"), 0); testHarness.processElement(new Tuple2<>(1, "EMIT_STATE"), 0); testHarness.processElement(new Tuple2<>(0, "EMIT_STATE"), 0); assertThat( extractResult(testHarness), contains("ON_ELEMENT:1:CIAO", "ON_ELEMENT:0:HELLO")); }
@Test public void testProcessingTimeTimers() throws Exception { final int expectedKey = 17; KeyedProcessOperator<Integer, Integer, Integer> operator = new KeyedProcessOperator<>(new TriggeringFlatMapFunction(TimeDomain.PROCESSING_TIME, expectedKey)); OneInputStreamOperatorTestHarness<Integer, Integer> testHarness = new KeyedOneInputStreamOperatorTestHarness<>(operator, new IdentityKeySelector<Integer>(), BasicTypeInfo.INT_TYPE_INFO); testHarness.setup(); testHarness.open(); testHarness.processElement(new StreamRecord<>(expectedKey)); testHarness.setProcessingTime(5); ConcurrentLinkedQueue<Object> expectedOutput = new ConcurrentLinkedQueue<>(); expectedOutput.add(new StreamRecord<>(expectedKey)); expectedOutput.add(new StreamRecord<>(1777)); TestHarnessUtil.assertOutputEquals("Output was not correct.", expectedOutput, testHarness.getOutput()); testHarness.close(); }
@Test public void testEventTimeTimers() throws Exception { LegacyKeyedProcessOperator<Integer, Integer, Integer> operator = new LegacyKeyedProcessOperator<>(new TriggeringFlatMapFunction(TimeDomain.EVENT_TIME)); OneInputStreamOperatorTestHarness<Integer, Integer> testHarness = new KeyedOneInputStreamOperatorTestHarness<>(operator, new IdentityKeySelector<Integer>(), BasicTypeInfo.INT_TYPE_INFO); testHarness.setup(); testHarness.open(); testHarness.processWatermark(new Watermark(0)); testHarness.processElement(new StreamRecord<>(17, 42L)); testHarness.processWatermark(new Watermark(5)); ConcurrentLinkedQueue<Object> expectedOutput = new ConcurrentLinkedQueue<>(); expectedOutput.add(new Watermark(0L)); expectedOutput.add(new StreamRecord<>(17, 42L)); expectedOutput.add(new StreamRecord<>(1777, 5L)); expectedOutput.add(new Watermark(5L)); TestHarnessUtil.assertOutputEquals("Output was not correct.", expectedOutput, testHarness.getOutput()); testHarness.close(); }
@Test public void testEventTimeTimers() throws Exception { final int expectedKey = 17; KeyedProcessOperator<Integer, Integer, Integer> operator = new KeyedProcessOperator<>(new TriggeringFlatMapFunction(TimeDomain.EVENT_TIME, expectedKey)); OneInputStreamOperatorTestHarness<Integer, Integer> testHarness = new KeyedOneInputStreamOperatorTestHarness<>(operator, new IdentityKeySelector<Integer>(), BasicTypeInfo.INT_TYPE_INFO); testHarness.setup(); testHarness.open(); testHarness.processWatermark(new Watermark(0)); testHarness.processElement(new StreamRecord<>(expectedKey, 42L)); testHarness.processWatermark(new Watermark(5)); ConcurrentLinkedQueue<Object> expectedOutput = new ConcurrentLinkedQueue<>(); expectedOutput.add(new Watermark(0L)); expectedOutput.add(new StreamRecord<>(expectedKey, 42L)); expectedOutput.add(new StreamRecord<>(1777, 5L)); expectedOutput.add(new Watermark(5L)); TestHarnessUtil.assertOutputEquals("Output was not correct.", expectedOutput, testHarness.getOutput()); testHarness.close(); }
/** * Verify that timers for the different time domains don't clash. */ @Test public void testProcessingTimeAndEventTimeDontInterfere() throws Exception { TestOperator testOperator = new TestOperator(); KeyedOneInputStreamOperatorTestHarness<Integer, Tuple2<Integer, String>, String> testHarness = new KeyedOneInputStreamOperatorTestHarness<>(testOperator, new TestKeySelector(), BasicTypeInfo.INT_TYPE_INFO); testHarness.open(); testHarness.setProcessingTime(0L); testHarness.processWatermark(0L); testHarness.processElement(new Tuple2<>(0, "SET_PROC_TIME_TIMER:10"), 0); testHarness.processElement(new Tuple2<>(0, "SET_EVENT_TIME_TIMER:20"), 0); testHarness.processElement(new Tuple2<>(0, "SET_STATE:HELLO"), 0); testHarness.processWatermark(20L); assertThat( extractResult(testHarness), contains("ON_EVENT_TIME:HELLO")); testHarness.setProcessingTime(10L); assertThat( extractResult(testHarness), contains("ON_PROC_TIME:HELLO")); }
@Test public void testGroupedReduce() throws Exception { KeySelector<Integer, Integer> keySelector = new IntegerKeySelector(); StreamGroupedReduce<Integer> operator = new StreamGroupedReduce<>(new MyReducer(), IntSerializer.INSTANCE); OneInputStreamOperatorTestHarness<Integer, Integer> testHarness = new KeyedOneInputStreamOperatorTestHarness<>(operator, keySelector, BasicTypeInfo.INT_TYPE_INFO); long initialTime = 0L; ConcurrentLinkedQueue<Object> expectedOutput = new ConcurrentLinkedQueue<>(); testHarness.open(); testHarness.processElement(new StreamRecord<>(1, initialTime + 1)); testHarness.processElement(new StreamRecord<>(1, initialTime + 2)); testHarness.processWatermark(new Watermark(initialTime + 2)); testHarness.processElement(new StreamRecord<>(2, initialTime + 3)); testHarness.processElement(new StreamRecord<>(2, initialTime + 4)); testHarness.processElement(new StreamRecord<>(3, initialTime + 5)); expectedOutput.add(new StreamRecord<>(1, initialTime + 1)); expectedOutput.add(new StreamRecord<>(2, initialTime + 2)); expectedOutput.add(new Watermark(initialTime + 2)); expectedOutput.add(new StreamRecord<>(2, initialTime + 3)); expectedOutput.add(new StreamRecord<>(4, initialTime + 4)); expectedOutput.add(new StreamRecord<>(3, initialTime + 5)); TestHarnessUtil.assertOutputEquals("Output was not correct.", expectedOutput, testHarness.getOutput()); }
/** * Verify that firing processing-time timers see the state of the key that was active * when the timer was set. */ @Test public void testProcessingTimeTimersDontInterfere() throws Exception { TestOperator testOperator = new TestOperator(); KeyedOneInputStreamOperatorTestHarness<Integer, Tuple2<Integer, String>, String> testHarness = new KeyedOneInputStreamOperatorTestHarness<>(testOperator, new TestKeySelector(), BasicTypeInfo.INT_TYPE_INFO); testHarness.open(); testHarness.setProcessingTime(0L); testHarness.processElement(new Tuple2<>(1, "SET_PROC_TIME_TIMER:20"), 0); testHarness.processElement(new Tuple2<>(0, "SET_STATE:HELLO"), 0); testHarness.processElement(new Tuple2<>(1, "SET_STATE:CIAO"), 0); testHarness.processElement(new Tuple2<>(0, "SET_PROC_TIME_TIMER:10"), 0); testHarness.setProcessingTime(10L); assertThat( extractResult(testHarness), contains("ON_PROC_TIME:HELLO")); testHarness.setProcessingTime(20L); assertThat( extractResult(testHarness), contains("ON_PROC_TIME:CIAO")); }
@Test public void testTimestampAndWatermarkQuerying() throws Exception { KeyedProcessOperator<Integer, Integer, String> operator = new KeyedProcessOperator<>(new QueryingFlatMapFunction(TimeDomain.EVENT_TIME)); OneInputStreamOperatorTestHarness<Integer, String> testHarness = new KeyedOneInputStreamOperatorTestHarness<>(operator, new IdentityKeySelector<Integer>(), BasicTypeInfo.INT_TYPE_INFO); testHarness.setup(); testHarness.open(); testHarness.processWatermark(new Watermark(17)); testHarness.processElement(new StreamRecord<>(5, 12L)); testHarness.processWatermark(new Watermark(42)); testHarness.processElement(new StreamRecord<>(6, 13L)); ConcurrentLinkedQueue<Object> expectedOutput = new ConcurrentLinkedQueue<>(); expectedOutput.add(new Watermark(17L)); expectedOutput.add(new StreamRecord<>("5TIME:17 TS:12", 12L)); expectedOutput.add(new Watermark(42L)); expectedOutput.add(new StreamRecord<>("6TIME:42 TS:13", 13L)); TestHarnessUtil.assertOutputEquals("Output was not correct.", expectedOutput, testHarness.getOutput()); testHarness.close(); }
@Test public void testTimestampAndWatermarkQuerying() throws Exception { LegacyKeyedProcessOperator<Integer, Integer, String> operator = new LegacyKeyedProcessOperator<>(new QueryingFlatMapFunction(TimeDomain.EVENT_TIME)); OneInputStreamOperatorTestHarness<Integer, String> testHarness = new KeyedOneInputStreamOperatorTestHarness<>(operator, new IdentityKeySelector<Integer>(), BasicTypeInfo.INT_TYPE_INFO); testHarness.setup(); testHarness.open(); testHarness.processWatermark(new Watermark(17)); testHarness.processElement(new StreamRecord<>(5, 12L)); testHarness.processWatermark(new Watermark(42)); testHarness.processElement(new StreamRecord<>(6, 13L)); ConcurrentLinkedQueue<Object> expectedOutput = new ConcurrentLinkedQueue<>(); expectedOutput.add(new Watermark(17L)); expectedOutput.add(new StreamRecord<>("5TIME:17 TS:12", 12L)); expectedOutput.add(new Watermark(42L)); expectedOutput.add(new StreamRecord<>("6TIME:42 TS:13", 13L)); TestHarnessUtil.assertOutputEquals("Output was not correct.", expectedOutput, testHarness.getOutput()); testHarness.close(); }
private KeyedOneInputStreamOperatorTestHarness<Integer, Event, Map<String, List<Event>>> getTestHarness( int maxParallelism, int taskParallelism, int subtaskIdx) throws Exception { KeySelector<Event, Integer> keySelector = new TestKeySelector(); KeyedOneInputStreamOperatorTestHarness<Integer, Event, Map<String, List<Event>>> harness = new KeyedOneInputStreamOperatorTestHarness<>( getKeyedCepOpearator( false, new NFAFactory()), keySelector, BasicTypeInfo.INT_TYPE_INFO, maxParallelism, taskParallelism, subtaskIdx); harness.setStateBackend(new RocksDBStateBackend(new MemoryStateBackend())); return harness; }
@Test public void testTimestampAndProcessingTimeQuerying() throws Exception { LegacyKeyedProcessOperator<Integer, Integer, String> operator = new LegacyKeyedProcessOperator<>(new QueryingFlatMapFunction(TimeDomain.PROCESSING_TIME)); OneInputStreamOperatorTestHarness<Integer, String> testHarness = new KeyedOneInputStreamOperatorTestHarness<>(operator, new IdentityKeySelector<Integer>(), BasicTypeInfo.INT_TYPE_INFO); testHarness.setup(); testHarness.open(); testHarness.setProcessingTime(17); testHarness.processElement(new StreamRecord<>(5)); testHarness.setProcessingTime(42); testHarness.processElement(new StreamRecord<>(6)); ConcurrentLinkedQueue<Object> expectedOutput = new ConcurrentLinkedQueue<>(); expectedOutput.add(new StreamRecord<>("5TIME:17 TS:null")); expectedOutput.add(new StreamRecord<>("6TIME:42 TS:null")); TestHarnessUtil.assertOutputEquals("Output was not correct.", expectedOutput, testHarness.getOutput()); testHarness.close(); }
@Test public void testTimestampAndProcessingTimeQuerying() throws Exception { KeyedProcessOperator<Integer, Integer, String> operator = new KeyedProcessOperator<>(new QueryingFlatMapFunction(TimeDomain.PROCESSING_TIME)); OneInputStreamOperatorTestHarness<Integer, String> testHarness = new KeyedOneInputStreamOperatorTestHarness<>(operator, new IdentityKeySelector<Integer>(), BasicTypeInfo.INT_TYPE_INFO); testHarness.setup(); testHarness.open(); testHarness.setProcessingTime(17); testHarness.processElement(new StreamRecord<>(5)); testHarness.setProcessingTime(42); testHarness.processElement(new StreamRecord<>(6)); ConcurrentLinkedQueue<Object> expectedOutput = new ConcurrentLinkedQueue<>(); expectedOutput.add(new StreamRecord<>("5TIME:17 TS:null")); expectedOutput.add(new StreamRecord<>("6TIME:42 TS:null")); TestHarnessUtil.assertOutputEquals("Output was not correct.", expectedOutput, testHarness.getOutput()); testHarness.close(); }
@Test public void testOpenClose() throws Exception { KeySelector<Integer, Integer> keySelector = new IntegerKeySelector(); StreamGroupedReduce<Integer> operator = new StreamGroupedReduce<>(new TestOpenCloseReduceFunction(), IntSerializer.INSTANCE); OneInputStreamOperatorTestHarness<Integer, Integer> testHarness = new KeyedOneInputStreamOperatorTestHarness<>(operator, keySelector, BasicTypeInfo.INT_TYPE_INFO); long initialTime = 0L; testHarness.open(); testHarness.processElement(new StreamRecord<>(1, initialTime)); testHarness.processElement(new StreamRecord<>(2, initialTime)); testHarness.close(); Assert.assertTrue("RichFunction methods where not called.", TestOpenCloseReduceFunction.closeCalled); Assert.assertTrue("Output contains no elements.", testHarness.getOutput().size() > 0); }
@Test public void testOpenClose() throws Exception { KeySelector<Integer, Integer> keySelector = new KeySelector<Integer, Integer>() { @Override public Integer getKey(Integer value) { return value; } }; StreamGroupedFold<Integer, String, Integer> operator = new StreamGroupedFold<>( new TestOpenCloseFoldFunction(), "init"); operator.setOutputType(BasicTypeInfo.STRING_TYPE_INFO, new ExecutionConfig()); OneInputStreamOperatorTestHarness<Integer, String> testHarness = new KeyedOneInputStreamOperatorTestHarness<>(operator, keySelector, BasicTypeInfo.INT_TYPE_INFO); long initialTime = 0L; testHarness.open(); testHarness.processElement(new StreamRecord<>(1, initialTime)); testHarness.processElement(new StreamRecord<>(2, initialTime)); testHarness.close(); assertTrue("RichFunction methods where not called.", TestOpenCloseFoldFunction.closeCalled); assertTrue("Output contains no elements.", testHarness.getOutput().size() > 0); }
@Test public void testNullOutputTagRefusal() throws Exception { KeyedProcessOperator<Integer, Integer, String> operator = new KeyedProcessOperator<>(new NullOutputTagEmittingProcessFunction()); OneInputStreamOperatorTestHarness<Integer, String> testHarness = new KeyedOneInputStreamOperatorTestHarness<>( operator, new IdentityKeySelector<>(), BasicTypeInfo.INT_TYPE_INFO); testHarness.setup(); testHarness.open(); testHarness.setProcessingTime(17); try { expectedException.expect(IllegalArgumentException.class); testHarness.processElement(new StreamRecord<>(5)); } finally { testHarness.close(); } }
@Test public void testNullOutputTagRefusal() throws Exception { LegacyKeyedProcessOperator<Integer, Integer, String> operator = new LegacyKeyedProcessOperator<>(new NullOutputTagEmittingProcessFunction()); OneInputStreamOperatorTestHarness<Integer, String> testHarness = new KeyedOneInputStreamOperatorTestHarness<>( operator, new IdentityKeySelector<>(), BasicTypeInfo.INT_TYPE_INFO); testHarness.setup(); testHarness.open(); testHarness.setProcessingTime(17); try { expectedException.expect(IllegalArgumentException.class); testHarness.processElement(new StreamRecord<>(5)); } finally { testHarness.close(); } }