public void processWatermark(long watermark) throws Exception { processWatermark(new Watermark(watermark)); }
public void advanceTime(OneInputStreamOperatorTestHarness testHarness, long timestamp) throws Exception { testHarness.processWatermark(new Watermark(timestamp)); }
/** * Verify that there is no late-data side output if the {@code WindowAssigner} does * not assign any windows. */ @Test public void testNoLateSideOutputForSkippedWindows() throws Exception { OutputTag<Integer> lateOutputTag = new OutputTag<Integer>("late"){}; WindowAssigner<Integer, TimeWindow> mockAssigner = mockTimeWindowAssigner(); Trigger<Integer, TimeWindow> mockTrigger = mockTrigger(); InternalWindowFunction<Iterable<Integer>, Void, Integer, TimeWindow> mockWindowFunction = mockWindowFunction(); OneInputStreamOperatorTestHarness<Integer, Void> testHarness = createWindowOperator(mockAssigner, mockTrigger, 0L, mockWindowFunction, lateOutputTag); testHarness.open(); when(mockAssigner.assignWindows(anyInt(), anyLong(), anyAssignerContext())) .thenReturn(Collections.<TimeWindow>emptyList()); testHarness.processWatermark(0); testHarness.processElement(new StreamRecord<>(0, 5L)); verify(mockAssigner, times(1)).assignWindows(eq(0), eq(5L), anyAssignerContext()); assertTrue(testHarness.getSideOutput(lateOutputTag) == null || testHarness.getSideOutput(lateOutputTag).isEmpty()); }
@Test public void testKeyedCEPOperatorWatermarkForwarding() throws Exception { OneInputStreamOperatorTestHarness<Event, Map<String, List<Event>>> harness = getCepTestHarness(false); try { harness.open(); Watermark expectedWatermark = new Watermark(42L); harness.processWatermark(expectedWatermark); verifyWatermark(harness.getOutput().poll(), 42L); } finally { harness.close(); } }
@Test public void testLateSideOutput() throws Exception { OutputTag<Integer> lateOutputTag = new OutputTag<Integer>("late"){}; WindowAssigner<Integer, TimeWindow> mockAssigner = mockTimeWindowAssigner(); Trigger<Integer, TimeWindow> mockTrigger = mockTrigger(); InternalWindowFunction<Iterable<Integer>, Void, Integer, TimeWindow> mockWindowFunction = mockWindowFunction(); OneInputStreamOperatorTestHarness<Integer, Void> testHarness = createWindowOperator(mockAssigner, mockTrigger, 0L, mockWindowFunction, lateOutputTag); testHarness.open(); when(mockAssigner.assignWindows(anyInt(), anyLong(), anyAssignerContext())) .thenReturn(Collections.singletonList(new TimeWindow(0, 0))); testHarness.processWatermark(20); testHarness.processElement(new StreamRecord<>(0, 5L)); verify(mockAssigner, times(1)).assignWindows(eq(0), eq(5L), anyAssignerContext()); assertThat(testHarness.getSideOutput(lateOutputTag), contains(isStreamRecord(0, 5L))); // we should also see side output if the WindowAssigner assigns no windows when(mockAssigner.assignWindows(anyInt(), anyLong(), anyAssignerContext())) .thenReturn(Collections.<TimeWindow>emptyList()); testHarness.processElement(new StreamRecord<>(0, 10L)); verify(mockAssigner, times(1)).assignWindows(eq(0), eq(5L), anyAssignerContext()); verify(mockAssigner, times(1)).assignWindows(eq(0), eq(10L), anyAssignerContext()); assertThat(testHarness.getSideOutput(lateOutputTag), contains(isStreamRecord(0, 5L), isStreamRecord(0, 10L))); }
@Test public void testEventTimeTimers() throws Exception { LegacyKeyedProcessOperator<Integer, Integer, Integer> operator = new LegacyKeyedProcessOperator<>(new TriggeringFlatMapFunction(TimeDomain.EVENT_TIME)); OneInputStreamOperatorTestHarness<Integer, Integer> testHarness = new KeyedOneInputStreamOperatorTestHarness<>(operator, new IdentityKeySelector<Integer>(), BasicTypeInfo.INT_TYPE_INFO); testHarness.setup(); testHarness.open(); testHarness.processWatermark(new Watermark(0)); testHarness.processElement(new StreamRecord<>(17, 42L)); testHarness.processWatermark(new Watermark(5)); ConcurrentLinkedQueue<Object> expectedOutput = new ConcurrentLinkedQueue<>(); expectedOutput.add(new Watermark(0L)); expectedOutput.add(new StreamRecord<>(17, 42L)); expectedOutput.add(new StreamRecord<>(1777, 5L)); expectedOutput.add(new Watermark(5L)); TestHarnessUtil.assertOutputEquals("Output was not correct.", expectedOutput, testHarness.getOutput()); testHarness.close(); }
@Test public void testTimestampAndWatermarkQuerying() throws Exception { ProcessOperator<Integer, String> operator = new ProcessOperator<>(new QueryingProcessFunction(TimeDomain.EVENT_TIME)); OneInputStreamOperatorTestHarness<Integer, String> testHarness = new OneInputStreamOperatorTestHarness<>(operator); testHarness.setup(); testHarness.open(); testHarness.processWatermark(new Watermark(17)); testHarness.processElement(new StreamRecord<>(5, 12L)); testHarness.processWatermark(new Watermark(42)); testHarness.processElement(new StreamRecord<>(6, 13L)); ConcurrentLinkedQueue<Object> expectedOutput = new ConcurrentLinkedQueue<>(); expectedOutput.add(new Watermark(17L)); expectedOutput.add(new StreamRecord<>("5TIME:17 TS:12", 12L)); expectedOutput.add(new Watermark(42L)); expectedOutput.add(new StreamRecord<>("6TIME:42 TS:13", 13L)); TestHarnessUtil.assertOutputEquals("Output was not correct.", expectedOutput, testHarness.getOutput()); testHarness.close(); }
@Test public void testEventTimeTimers() throws Exception { final int expectedKey = 17; KeyedProcessOperator<Integer, Integer, Integer> operator = new KeyedProcessOperator<>(new TriggeringFlatMapFunction(TimeDomain.EVENT_TIME, expectedKey)); OneInputStreamOperatorTestHarness<Integer, Integer> testHarness = new KeyedOneInputStreamOperatorTestHarness<>(operator, new IdentityKeySelector<Integer>(), BasicTypeInfo.INT_TYPE_INFO); testHarness.setup(); testHarness.open(); testHarness.processWatermark(new Watermark(0)); testHarness.processElement(new StreamRecord<>(expectedKey, 42L)); testHarness.processWatermark(new Watermark(5)); ConcurrentLinkedQueue<Object> expectedOutput = new ConcurrentLinkedQueue<>(); expectedOutput.add(new Watermark(0L)); expectedOutput.add(new StreamRecord<>(expectedKey, 42L)); expectedOutput.add(new StreamRecord<>(1777, 5L)); expectedOutput.add(new Watermark(5L)); TestHarnessUtil.assertOutputEquals("Output was not correct.", expectedOutput, testHarness.getOutput()); testHarness.close(); }
@Test public void testTimestampAndWatermarkQuerying() throws Exception { KeyedProcessOperator<Integer, Integer, String> operator = new KeyedProcessOperator<>(new QueryingFlatMapFunction(TimeDomain.EVENT_TIME)); OneInputStreamOperatorTestHarness<Integer, String> testHarness = new KeyedOneInputStreamOperatorTestHarness<>(operator, new IdentityKeySelector<Integer>(), BasicTypeInfo.INT_TYPE_INFO); testHarness.setup(); testHarness.open(); testHarness.processWatermark(new Watermark(17)); testHarness.processElement(new StreamRecord<>(5, 12L)); testHarness.processWatermark(new Watermark(42)); testHarness.processElement(new StreamRecord<>(6, 13L)); ConcurrentLinkedQueue<Object> expectedOutput = new ConcurrentLinkedQueue<>(); expectedOutput.add(new Watermark(17L)); expectedOutput.add(new StreamRecord<>("5TIME:17 TS:12", 12L)); expectedOutput.add(new Watermark(42L)); expectedOutput.add(new StreamRecord<>("6TIME:42 TS:13", 13L)); TestHarnessUtil.assertOutputEquals("Output was not correct.", expectedOutput, testHarness.getOutput()); testHarness.close(); }
@Test public void testTimestampAndWatermarkQuerying() throws Exception { LegacyKeyedProcessOperator<Integer, Integer, String> operator = new LegacyKeyedProcessOperator<>(new QueryingFlatMapFunction(TimeDomain.EVENT_TIME)); OneInputStreamOperatorTestHarness<Integer, String> testHarness = new KeyedOneInputStreamOperatorTestHarness<>(operator, new IdentityKeySelector<Integer>(), BasicTypeInfo.INT_TYPE_INFO); testHarness.setup(); testHarness.open(); testHarness.processWatermark(new Watermark(17)); testHarness.processElement(new StreamRecord<>(5, 12L)); testHarness.processWatermark(new Watermark(42)); testHarness.processElement(new StreamRecord<>(6, 13L)); ConcurrentLinkedQueue<Object> expectedOutput = new ConcurrentLinkedQueue<>(); expectedOutput.add(new Watermark(17L)); expectedOutput.add(new StreamRecord<>("5TIME:17 TS:12", 12L)); expectedOutput.add(new Watermark(42L)); expectedOutput.add(new StreamRecord<>("6TIME:42 TS:13", 13L)); TestHarnessUtil.assertOutputEquals("Output was not correct.", expectedOutput, testHarness.getOutput()); testHarness.close(); }
@Test public void testMap() throws Exception { StreamMap<Integer, String> operator = new StreamMap<Integer, String>(new Map()); OneInputStreamOperatorTestHarness<Integer, String> testHarness = new OneInputStreamOperatorTestHarness<Integer, String>(operator); long initialTime = 0L; ConcurrentLinkedQueue<Object> expectedOutput = new ConcurrentLinkedQueue<Object>(); testHarness.open(); testHarness.processElement(new StreamRecord<Integer>(1, initialTime + 1)); testHarness.processElement(new StreamRecord<Integer>(2, initialTime + 2)); testHarness.processWatermark(new Watermark(initialTime + 2)); testHarness.processElement(new StreamRecord<Integer>(3, initialTime + 3)); expectedOutput.add(new StreamRecord<String>("+2", initialTime + 1)); expectedOutput.add(new StreamRecord<String>("+3", initialTime + 2)); expectedOutput.add(new Watermark(initialTime + 2)); expectedOutput.add(new StreamRecord<String>("+4", initialTime + 3)); TestHarnessUtil.assertOutputEquals("Output was not correct.", expectedOutput, testHarness.getOutput()); }
@Test public void testGroupedReduce() throws Exception { KeySelector<Integer, Integer> keySelector = new IntegerKeySelector(); StreamGroupedReduce<Integer> operator = new StreamGroupedReduce<>(new MyReducer(), IntSerializer.INSTANCE); OneInputStreamOperatorTestHarness<Integer, Integer> testHarness = new KeyedOneInputStreamOperatorTestHarness<>(operator, keySelector, BasicTypeInfo.INT_TYPE_INFO); long initialTime = 0L; ConcurrentLinkedQueue<Object> expectedOutput = new ConcurrentLinkedQueue<>(); testHarness.open(); testHarness.processElement(new StreamRecord<>(1, initialTime + 1)); testHarness.processElement(new StreamRecord<>(1, initialTime + 2)); testHarness.processWatermark(new Watermark(initialTime + 2)); testHarness.processElement(new StreamRecord<>(2, initialTime + 3)); testHarness.processElement(new StreamRecord<>(2, initialTime + 4)); testHarness.processElement(new StreamRecord<>(3, initialTime + 5)); expectedOutput.add(new StreamRecord<>(1, initialTime + 1)); expectedOutput.add(new StreamRecord<>(2, initialTime + 2)); expectedOutput.add(new Watermark(initialTime + 2)); expectedOutput.add(new StreamRecord<>(2, initialTime + 3)); expectedOutput.add(new StreamRecord<>(4, initialTime + 4)); expectedOutput.add(new StreamRecord<>(3, initialTime + 5)); TestHarnessUtil.assertOutputEquals("Output was not correct.", expectedOutput, testHarness.getOutput()); }
@Test public void testFlatMap() throws Exception { StreamFlatMap<Integer, Integer> operator = new StreamFlatMap<Integer, Integer>(new MyFlatMap()); OneInputStreamOperatorTestHarness<Integer, Integer> testHarness = new OneInputStreamOperatorTestHarness<Integer, Integer>(operator); long initialTime = 0L; ConcurrentLinkedQueue<Object> expectedOutput = new ConcurrentLinkedQueue<Object>(); testHarness.open(); testHarness.processElement(new StreamRecord<Integer>(1, initialTime + 1)); testHarness.processElement(new StreamRecord<Integer>(2, initialTime + 2)); testHarness.processWatermark(new Watermark(initialTime + 2)); testHarness.processElement(new StreamRecord<Integer>(3, initialTime + 3)); testHarness.processElement(new StreamRecord<Integer>(4, initialTime + 4)); testHarness.processElement(new StreamRecord<Integer>(5, initialTime + 5)); testHarness.processElement(new StreamRecord<Integer>(6, initialTime + 6)); testHarness.processElement(new StreamRecord<Integer>(7, initialTime + 7)); testHarness.processElement(new StreamRecord<Integer>(8, initialTime + 8)); expectedOutput.add(new StreamRecord<Integer>(2, initialTime + 2)); expectedOutput.add(new StreamRecord<Integer>(4, initialTime + 2)); expectedOutput.add(new Watermark(initialTime + 2)); expectedOutput.add(new StreamRecord<Integer>(4, initialTime + 4)); expectedOutput.add(new StreamRecord<Integer>(16, initialTime + 4)); expectedOutput.add(new StreamRecord<Integer>(6, initialTime + 6)); expectedOutput.add(new StreamRecord<Integer>(36, initialTime + 6)); expectedOutput.add(new StreamRecord<Integer>(8, initialTime + 8)); expectedOutput.add(new StreamRecord<Integer>(64, initialTime + 8)); TestHarnessUtil.assertOutputEquals("Output was not correct.", expectedOutput, testHarness.getOutput()); }
@Test @SuppressWarnings("unchecked") public void testFilter() throws Exception { StreamFilter<Integer> operator = new StreamFilter<Integer>(new MyFilter()); OneInputStreamOperatorTestHarness<Integer, Integer> testHarness = new OneInputStreamOperatorTestHarness<Integer, Integer>(operator); long initialTime = 0L; ConcurrentLinkedQueue<Object> expectedOutput = new ConcurrentLinkedQueue<Object>(); testHarness.open(); testHarness.processElement(new StreamRecord<Integer>(1, initialTime + 1)); testHarness.processElement(new StreamRecord<Integer>(2, initialTime + 2)); testHarness.processWatermark(new Watermark(initialTime + 2)); testHarness.processElement(new StreamRecord<Integer>(3, initialTime + 3)); testHarness.processElement(new StreamRecord<Integer>(4, initialTime + 4)); testHarness.processElement(new StreamRecord<Integer>(5, initialTime + 5)); testHarness.processElement(new StreamRecord<Integer>(6, initialTime + 6)); testHarness.processElement(new StreamRecord<Integer>(7, initialTime + 7)); expectedOutput.add(new StreamRecord<Integer>(2, initialTime + 2)); expectedOutput.add(new Watermark(initialTime + 2)); expectedOutput.add(new StreamRecord<Integer>(4, initialTime + 4)); expectedOutput.add(new StreamRecord<Integer>(6, initialTime + 6)); TestHarnessUtil.assertOutputEquals("Output was not correct.", expectedOutput, testHarness.getOutput()); }
@Test public void testSnapshotAndRestore() throws Exception { LegacyKeyedProcessOperator<Integer, Integer, String> operator = new LegacyKeyedProcessOperator<>(new BothTriggeringFlatMapFunction()); OneInputStreamOperatorTestHarness<Integer, String> testHarness = new KeyedOneInputStreamOperatorTestHarness<>(operator, new IdentityKeySelector<Integer>(), BasicTypeInfo.INT_TYPE_INFO); testHarness.setup(); testHarness.open(); testHarness.processElement(new StreamRecord<>(5, 12L)); // snapshot and restore from scratch OperatorSubtaskState snapshot = testHarness.snapshot(0, 0); testHarness.close(); operator = new LegacyKeyedProcessOperator<>(new BothTriggeringFlatMapFunction()); testHarness = new KeyedOneInputStreamOperatorTestHarness<>(operator, new IdentityKeySelector<Integer>(), BasicTypeInfo.INT_TYPE_INFO); testHarness.setup(); testHarness.initializeState(snapshot); testHarness.open(); testHarness.setProcessingTime(5); testHarness.processWatermark(new Watermark(6)); ConcurrentLinkedQueue<Object> expectedOutput = new ConcurrentLinkedQueue<>(); expectedOutput.add(new StreamRecord<>("PROC:1777")); expectedOutput.add(new StreamRecord<>("EVENT:1777", 6L)); expectedOutput.add(new Watermark(6)); TestHarnessUtil.assertOutputEquals("Output was not correct.", expectedOutput, testHarness.getOutput()); testHarness.close(); }
@Test public void testSnapshotAndRestore() throws Exception { final int expectedKey = 5; KeyedProcessOperator<Integer, Integer, String> operator = new KeyedProcessOperator<>(new BothTriggeringFlatMapFunction(expectedKey)); OneInputStreamOperatorTestHarness<Integer, String> testHarness = new KeyedOneInputStreamOperatorTestHarness<>(operator, new IdentityKeySelector<Integer>(), BasicTypeInfo.INT_TYPE_INFO); testHarness.setup(); testHarness.open(); testHarness.processElement(new StreamRecord<>(expectedKey, 12L)); // snapshot and restore from scratch OperatorSubtaskState snapshot = testHarness.snapshot(0, 0); testHarness.close(); operator = new KeyedProcessOperator<>(new BothTriggeringFlatMapFunction(expectedKey)); testHarness = new KeyedOneInputStreamOperatorTestHarness<>(operator, new IdentityKeySelector<Integer>(), BasicTypeInfo.INT_TYPE_INFO); testHarness.setup(); testHarness.initializeState(snapshot); testHarness.open(); testHarness.setProcessingTime(5); testHarness.processWatermark(new Watermark(6)); ConcurrentLinkedQueue<Object> expectedOutput = new ConcurrentLinkedQueue<>(); expectedOutput.add(new StreamRecord<>("PROC:1777")); expectedOutput.add(new StreamRecord<>("EVENT:1777", 6L)); expectedOutput.add(new Watermark(6)); TestHarnessUtil.assertOutputEquals("Output was not correct.", expectedOutput, testHarness.getOutput()); testHarness.close(); }
@Test public void testCurrentProcessingTimeInEventTime() throws Exception { try ( OneInputStreamOperatorTestHarness<Event, String> harness = getCepTestHarness( createCepOperator( extractCurrentProcessingTimeAndNames(1), new NFAForwardingFactory(), EVENT_TIME))) { harness.open(); harness.setProcessingTime(10); harness.processElement(event().withName("A").withTimestamp(5).asStreamRecord()); harness.setProcessingTime(100); harness.processWatermark(6); assertOutput(harness.getOutput()) .nextElementEquals("100:A") .watermarkEquals(6) .hasNoMoreElements(); } }
@Test public void testTimestampPassingInEventTime() throws Exception { try ( OneInputStreamOperatorTestHarness<Event, String> harness = getCepTestHarness( createCepOperator( extractTimestampAndNames(1), new NFAForwardingFactory(), EVENT_TIME))) { harness.open(); // events out of order to test if internal sorting does not mess up the timestamps harness.processElement(event().withName("A").withTimestamp(5).asStreamRecord()); harness.processElement(event().withName("B").withTimestamp(3).asStreamRecord()); harness.processWatermark(6); assertOutput(harness.getOutput()) .nextElementEquals("3:B") .nextElementEquals("5:A") .watermarkEquals(6L) .hasNoMoreElements(); } }
@Test public void testTimestampPassingForTimedOutInEventTime() throws Exception { OutputTag<String> timedOut = new OutputTag<String>("timedOut") {}; try ( OneInputStreamOperatorTestHarness<Event, String> harness = getCepTestHarness( createCepOperator( extractTimestampAndNames(2, timedOut), new NFATimingOutFactory(), EVENT_TIME))) { harness.open(); // events out of order to test if internal sorting does not mess up the timestamps harness.processElement(event().withName("A").withTimestamp(5).asStreamRecord()); harness.processElement(event().withName("C").withTimestamp(20).asStreamRecord()); harness.processElement(event().withName("B").withTimestamp(3).asStreamRecord()); harness.processWatermark(22); assertOutput(harness.getOutput()) .nextElementEquals("5:B:A") .watermarkEquals(22) .hasNoMoreElements(); assertOutput(harness.getSideOutput(timedOut)) .nextElementEquals("15:A") .hasNoMoreElements(); } }
@Test public void testCurrentProcessingTimeForTimedOutInEventTime() throws Exception { OutputTag<String> sideOutputTag = new OutputTag<String>("timedOut") {}; try ( OneInputStreamOperatorTestHarness<Event, String> harness = getCepTestHarness( createCepOperator( extractCurrentProcessingTimeAndNames(2, sideOutputTag), new NFATimingOutFactory(), EVENT_TIME))) { harness.open(); // events out of order to test if internal sorting does not mess up the timestamps harness.processElement(event().withName("A").withTimestamp(5).asStreamRecord()); harness.processElement(event().withName("B").withTimestamp(20).asStreamRecord()); harness.processElement(event().withName("C").withTimestamp(3).asStreamRecord()); harness.setProcessingTime(100); harness.processWatermark(22); assertOutput(harness.getOutput()) .nextElementEquals("100:C:A") .watermarkEquals(22) .hasNoMoreElements(); assertOutput(harness.getSideOutput(sideOutputTag)) .nextElementEquals("100:A") .hasNoMoreElements(); } }