public void advanceTime(OneInputStreamOperatorTestHarness testHarness, long timestamp) throws Exception { testHarness.setProcessingTime(timestamp); }
@Test public void testProcessingTimestampisPassedToNFA() throws Exception { final NFA<Event> nfa = NFACompiler.compileFactory(Pattern.<Event>begin("begin"), true).createNFA(); final NFA<Event> spyNFA = spy(nfa); try ( OneInputStreamOperatorTestHarness<Event, Map<String, List<Event>>> harness = CepOperatorTestUtilities.getCepTestHarness(createOperatorForNFA(spyNFA).build())) { long timestamp = 5; harness.open(); harness.setProcessingTime(timestamp); StreamRecord<Event> event = event().withTimestamp(3).asStreamRecord(); harness.processElement(event); verify(spyNFA).process( any(SharedBufferAccessor.class), any(NFAState.class), eq(event.getValue()), eq(timestamp), any(AfterMatchSkipStrategy.class), any(TimerService.class)); } }
/** * Verifies that we don't have leakage between different keys. */ @Test public void testProcessingTimeTimerWithState() throws Exception { LegacyKeyedProcessOperator<Integer, Integer, String> operator = new LegacyKeyedProcessOperator<>(new TriggeringStatefulFlatMapFunction(TimeDomain.PROCESSING_TIME)); OneInputStreamOperatorTestHarness<Integer, String> testHarness = new KeyedOneInputStreamOperatorTestHarness<>(operator, new IdentityKeySelector<Integer>(), BasicTypeInfo.INT_TYPE_INFO); testHarness.setup(); testHarness.open(); testHarness.setProcessingTime(1); testHarness.processElement(new StreamRecord<>(17)); // should set timer for 6 testHarness.setProcessingTime(2); testHarness.processElement(new StreamRecord<>(42)); // should set timer for 7 testHarness.setProcessingTime(6); testHarness.setProcessingTime(7); ConcurrentLinkedQueue<Object> expectedOutput = new ConcurrentLinkedQueue<>(); expectedOutput.add(new StreamRecord<>("INPUT:17")); expectedOutput.add(new StreamRecord<>("INPUT:42")); expectedOutput.add(new StreamRecord<>("STATE:17")); expectedOutput.add(new StreamRecord<>("STATE:42")); TestHarnessUtil.assertOutputEquals("Output was not correct.", expectedOutput, testHarness.getOutput()); testHarness.close(); }
@Test public void testTimestampAndProcessingTimeQuerying() throws Exception { ProcessOperator<Integer, String> operator = new ProcessOperator<>(new QueryingProcessFunction(TimeDomain.PROCESSING_TIME)); OneInputStreamOperatorTestHarness<Integer, String> testHarness = new OneInputStreamOperatorTestHarness<>(operator); testHarness.setup(); testHarness.open(); testHarness.setProcessingTime(17); testHarness.processElement(new StreamRecord<>(5)); testHarness.setProcessingTime(42); testHarness.processElement(new StreamRecord<>(6)); ConcurrentLinkedQueue<Object> expectedOutput = new ConcurrentLinkedQueue<>(); expectedOutput.add(new StreamRecord<>("5TIME:17 TS:null")); expectedOutput.add(new StreamRecord<>("6TIME:42 TS:null")); TestHarnessUtil.assertOutputEquals("Output was not correct.", expectedOutput, testHarness.getOutput()); testHarness.close(); }
@Test public void testTimestampAndProcessingTimeQuerying() throws Exception { LegacyKeyedProcessOperator<Integer, Integer, String> operator = new LegacyKeyedProcessOperator<>(new QueryingFlatMapFunction(TimeDomain.PROCESSING_TIME)); OneInputStreamOperatorTestHarness<Integer, String> testHarness = new KeyedOneInputStreamOperatorTestHarness<>(operator, new IdentityKeySelector<Integer>(), BasicTypeInfo.INT_TYPE_INFO); testHarness.setup(); testHarness.open(); testHarness.setProcessingTime(17); testHarness.processElement(new StreamRecord<>(5)); testHarness.setProcessingTime(42); testHarness.processElement(new StreamRecord<>(6)); ConcurrentLinkedQueue<Object> expectedOutput = new ConcurrentLinkedQueue<>(); expectedOutput.add(new StreamRecord<>("5TIME:17 TS:null")); expectedOutput.add(new StreamRecord<>("6TIME:42 TS:null")); TestHarnessUtil.assertOutputEquals("Output was not correct.", expectedOutput, testHarness.getOutput()); testHarness.close(); }
@Test public void testTimestampAndProcessingTimeQuerying() throws Exception { KeyedProcessOperator<Integer, Integer, String> operator = new KeyedProcessOperator<>(new QueryingFlatMapFunction(TimeDomain.PROCESSING_TIME)); OneInputStreamOperatorTestHarness<Integer, String> testHarness = new KeyedOneInputStreamOperatorTestHarness<>(operator, new IdentityKeySelector<Integer>(), BasicTypeInfo.INT_TYPE_INFO); testHarness.setup(); testHarness.open(); testHarness.setProcessingTime(17); testHarness.processElement(new StreamRecord<>(5)); testHarness.setProcessingTime(42); testHarness.processElement(new StreamRecord<>(6)); ConcurrentLinkedQueue<Object> expectedOutput = new ConcurrentLinkedQueue<>(); expectedOutput.add(new StreamRecord<>("5TIME:17 TS:null")); expectedOutput.add(new StreamRecord<>("6TIME:42 TS:null")); TestHarnessUtil.assertOutputEquals("Output was not correct.", expectedOutput, testHarness.getOutput()); testHarness.close(); }
/** * Verify that we can correctly query watermark, processing time and the timestamp from the * context. */ @Test public void testTimeQuerying() throws Exception { BufferingQueryingSink<String> bufferingSink = new BufferingQueryingSink<>(); StreamSink<String> operator = new StreamSink<>(bufferingSink); OneInputStreamOperatorTestHarness<String, Object> testHarness = new OneInputStreamOperatorTestHarness<>(operator); testHarness.setup(); testHarness.open(); testHarness.processWatermark(new Watermark(17)); testHarness.setProcessingTime(12); testHarness.processElement(new StreamRecord<>("Hello", 12L)); testHarness.processWatermark(new Watermark(42)); testHarness.setProcessingTime(15); testHarness.processElement(new StreamRecord<>("Ciao", 13L)); testHarness.processWatermark(new Watermark(42)); testHarness.setProcessingTime(15); testHarness.processElement(new StreamRecord<>("Ciao")); assertThat(bufferingSink.data.size(), is(3)); assertThat(bufferingSink.data, contains( new Tuple4<>(17L, 12L, 12L, "Hello"), new Tuple4<>(42L, 15L, 13L, "Ciao"), new Tuple4<>(42L, 15L, null, "Ciao"))); testHarness.close(); }
@Test public void testProcessingTimeTimers() throws Exception { LegacyKeyedProcessOperator<Integer, Integer, Integer> operator = new LegacyKeyedProcessOperator<>(new TriggeringFlatMapFunction(TimeDomain.PROCESSING_TIME)); OneInputStreamOperatorTestHarness<Integer, Integer> testHarness = new KeyedOneInputStreamOperatorTestHarness<>(operator, new IdentityKeySelector<Integer>(), BasicTypeInfo.INT_TYPE_INFO); testHarness.setup(); testHarness.open(); testHarness.processElement(new StreamRecord<>(17)); testHarness.setProcessingTime(5); ConcurrentLinkedQueue<Object> expectedOutput = new ConcurrentLinkedQueue<>(); expectedOutput.add(new StreamRecord<>(17)); expectedOutput.add(new StreamRecord<>(1777)); TestHarnessUtil.assertOutputEquals("Output was not correct.", expectedOutput, testHarness.getOutput()); testHarness.close(); }
@Test public void testProcessingTimeTimers() throws Exception { final int expectedKey = 17; KeyedProcessOperator<Integer, Integer, Integer> operator = new KeyedProcessOperator<>(new TriggeringFlatMapFunction(TimeDomain.PROCESSING_TIME, expectedKey)); OneInputStreamOperatorTestHarness<Integer, Integer> testHarness = new KeyedOneInputStreamOperatorTestHarness<>(operator, new IdentityKeySelector<Integer>(), BasicTypeInfo.INT_TYPE_INFO); testHarness.setup(); testHarness.open(); testHarness.processElement(new StreamRecord<>(expectedKey)); testHarness.setProcessingTime(5); ConcurrentLinkedQueue<Object> expectedOutput = new ConcurrentLinkedQueue<>(); expectedOutput.add(new StreamRecord<>(expectedKey)); expectedOutput.add(new StreamRecord<>(1777)); TestHarnessUtil.assertOutputEquals("Output was not correct.", expectedOutput, testHarness.getOutput()); testHarness.close(); }
@Test public void testTimestampPassingInProcessingTime() throws Exception { try ( OneInputStreamOperatorTestHarness<Event, String> harness = getCepTestHarness( createCepOperator( extractTimestampAndNames(1), new NFAForwardingFactory(), PROCESSING_TIME))) { harness.open(); harness.setProcessingTime(1); harness.processElement(event().withName("A").withTimestamp(5).asStreamRecord()); harness.setProcessingTime(2); harness.processElement(event().withName("B").withTimestamp(3).asStreamRecord()); harness.setProcessingTime(3); assertOutput(harness.getOutput()) .nextElementEquals("1:A") .nextElementEquals("2:B") .hasNoMoreElements(); } }
@Test public void testCurrentProcessingTimeInEventTime() throws Exception { try ( OneInputStreamOperatorTestHarness<Event, String> harness = getCepTestHarness( createCepOperator( extractCurrentProcessingTimeAndNames(1), new NFAForwardingFactory(), EVENT_TIME))) { harness.open(); harness.setProcessingTime(10); harness.processElement(event().withName("A").withTimestamp(5).asStreamRecord()); harness.setProcessingTime(100); harness.processWatermark(6); assertOutput(harness.getOutput()) .nextElementEquals("100:A") .watermarkEquals(6) .hasNoMoreElements(); } }
@Test public void testNullOutputTagRefusal() throws Exception { ProcessOperator<Integer, String> operator = new ProcessOperator<>(new NullOutputTagEmittingProcessFunction()); OneInputStreamOperatorTestHarness<Integer, String> testHarness = new OneInputStreamOperatorTestHarness<>(operator); testHarness.setup(); testHarness.open(); testHarness.setProcessingTime(17); try { expectedException.expect(IllegalArgumentException.class); testHarness.processElement(new StreamRecord<>(5)); } finally { testHarness.close(); } }
@Test public void testCurrentProcessingTimeInProcessingTime() throws Exception { try ( OneInputStreamOperatorTestHarness<Event, String> harness = getCepTestHarness( createCepOperator( extractCurrentProcessingTimeAndNames(1), new NFAForwardingFactory(), PROCESSING_TIME))) { harness.open(); harness.setProcessingTime(15); harness.processElement(event().withName("A").asStreamRecord()); harness.setProcessingTime(35); harness.processElement(event().withName("B").asStreamRecord()); assertOutput(harness.getOutput()) .nextElementEquals("15:A") .nextElementEquals("35:B") .hasNoMoreElements(); } }
@Test public void testCurrentProcessingTimeForTimedOutInProcessingTime() throws Exception { OutputTag<String> sideOutputTag = new OutputTag<String>("timedOut") {}; try ( OneInputStreamOperatorTestHarness<Event, String> harness = getCepTestHarness( createCepOperator( extractCurrentProcessingTimeAndNames(2, sideOutputTag), new NFATimingOutFactory(), PROCESSING_TIME))) { harness.open(); harness.setProcessingTime(3); harness.processElement(event().withName("A").asStreamRecord()); harness.setProcessingTime(5); harness.processElement(event().withName("B").asStreamRecord()); harness.setProcessingTime(20); harness.processElement(event().withName("C").asStreamRecord()); assertOutput(harness.getOutput()) .nextElementEquals("5:A:B") .hasNoMoreElements(); // right now we time out only on next event in processing time, therefore the 20 assertOutput(harness.getSideOutput(sideOutputTag)) .nextElementEquals("20:B") .hasNoMoreElements(); } }
@Test public void testTimestampPassingForTimedOutInProcessingTime() throws Exception { OutputTag<String> timedOut = new OutputTag<String>("timedOut") {}; try ( OneInputStreamOperatorTestHarness<Event, String> harness = getCepTestHarness( createCepOperator( extractTimestampAndNames(2, timedOut), new NFATimingOutFactory(), PROCESSING_TIME))) { harness.open(); harness.setProcessingTime(3); harness.processElement(event().withName("A").withTimestamp(3).asStreamRecord()); harness.setProcessingTime(5); harness.processElement(event().withName("C").withTimestamp(5).asStreamRecord()); harness.setProcessingTime(20); harness.processElement(event().withName("B").withTimestamp(20).asStreamRecord()); assertOutput(harness.getOutput()) .nextElementEquals("5:A:C") .hasNoMoreElements(); assertOutput(harness.getSideOutput(timedOut)) .nextElementEquals("15:C") .hasNoMoreElements(); } }
private void testTimeoutExceptionHandling(AsyncDataStream.OutputMode outputMode) throws Exception { AsyncFunction<Integer, Integer> asyncFunction = new NoOpAsyncFunction<>(); long timeout = 10L; // 1 milli second AsyncWaitOperator<Integer, Integer> asyncWaitOperator = new AsyncWaitOperator<>( asyncFunction, timeout, 2, outputMode); final MockEnvironment mockEnvironment = createMockEnvironment(); mockEnvironment.setExpectedExternalFailureCause(Throwable.class); OneInputStreamOperatorTestHarness<Integer, Integer> harness = new OneInputStreamOperatorTestHarness<>( asyncWaitOperator, IntSerializer.INSTANCE, mockEnvironment); harness.open(); synchronized (harness.getCheckpointLock()) { harness.processElement(1, 1L); } harness.setProcessingTime(10L); synchronized (harness.getCheckpointLock()) { harness.close(); } }
@Test public void testSnapshotAndRestore() throws Exception { LegacyKeyedProcessOperator<Integer, Integer, String> operator = new LegacyKeyedProcessOperator<>(new BothTriggeringFlatMapFunction()); OneInputStreamOperatorTestHarness<Integer, String> testHarness = new KeyedOneInputStreamOperatorTestHarness<>(operator, new IdentityKeySelector<Integer>(), BasicTypeInfo.INT_TYPE_INFO); testHarness.setup(); testHarness.open(); testHarness.processElement(new StreamRecord<>(5, 12L)); // snapshot and restore from scratch OperatorSubtaskState snapshot = testHarness.snapshot(0, 0); testHarness.close(); operator = new LegacyKeyedProcessOperator<>(new BothTriggeringFlatMapFunction()); testHarness = new KeyedOneInputStreamOperatorTestHarness<>(operator, new IdentityKeySelector<Integer>(), BasicTypeInfo.INT_TYPE_INFO); testHarness.setup(); testHarness.initializeState(snapshot); testHarness.open(); testHarness.setProcessingTime(5); testHarness.processWatermark(new Watermark(6)); ConcurrentLinkedQueue<Object> expectedOutput = new ConcurrentLinkedQueue<>(); expectedOutput.add(new StreamRecord<>("PROC:1777")); expectedOutput.add(new StreamRecord<>("EVENT:1777", 6L)); expectedOutput.add(new Watermark(6)); TestHarnessUtil.assertOutputEquals("Output was not correct.", expectedOutput, testHarness.getOutput()); testHarness.close(); }
@Test public void testNullOutputTagRefusal() throws Exception { KeyedProcessOperator<Integer, Integer, String> operator = new KeyedProcessOperator<>(new NullOutputTagEmittingProcessFunction()); OneInputStreamOperatorTestHarness<Integer, String> testHarness = new KeyedOneInputStreamOperatorTestHarness<>( operator, new IdentityKeySelector<>(), BasicTypeInfo.INT_TYPE_INFO); testHarness.setup(); testHarness.open(); testHarness.setProcessingTime(17); try { expectedException.expect(IllegalArgumentException.class); testHarness.processElement(new StreamRecord<>(5)); } finally { testHarness.close(); } }
@Test public void testNullOutputTagRefusal() throws Exception { LegacyKeyedProcessOperator<Integer, Integer, String> operator = new LegacyKeyedProcessOperator<>(new NullOutputTagEmittingProcessFunction()); OneInputStreamOperatorTestHarness<Integer, String> testHarness = new KeyedOneInputStreamOperatorTestHarness<>( operator, new IdentityKeySelector<>(), BasicTypeInfo.INT_TYPE_INFO); testHarness.setup(); testHarness.open(); testHarness.setProcessingTime(17); try { expectedException.expect(IllegalArgumentException.class); testHarness.processElement(new StreamRecord<>(5)); } finally { testHarness.close(); } }
@Test public void testCurrentProcessingTimeForTimedOutInEventTime() throws Exception { OutputTag<String> sideOutputTag = new OutputTag<String>("timedOut") {}; try ( OneInputStreamOperatorTestHarness<Event, String> harness = getCepTestHarness( createCepOperator( extractCurrentProcessingTimeAndNames(2, sideOutputTag), new NFATimingOutFactory(), EVENT_TIME))) { harness.open(); // events out of order to test if internal sorting does not mess up the timestamps harness.processElement(event().withName("A").withTimestamp(5).asStreamRecord()); harness.processElement(event().withName("B").withTimestamp(20).asStreamRecord()); harness.processElement(event().withName("C").withTimestamp(3).asStreamRecord()); harness.setProcessingTime(100); harness.processWatermark(22); assertOutput(harness.getOutput()) .nextElementEquals("100:C:A") .watermarkEquals(22) .hasNoMoreElements(); assertOutput(harness.getSideOutput(sideOutputTag)) .nextElementEquals("100:A") .hasNoMoreElements(); } }