@Test public void testClosingWithoutInitializingStateShouldNotFail() throws Exception { final File outDir = TEMP_FOLDER.newFolder(); try (OneInputStreamOperatorTestHarness<Tuple2<String, Integer>, Object> testHarness = TestUtils.createRescalingTestSink(outDir, 1, 0, 100L, 124L)) { testHarness.setup(); } }
@Test public void testClosingWithoutInput() throws Exception { final File outDir = TEMP_FOLDER.newFolder(); try ( OneInputStreamOperatorTestHarness<Tuple2<String, Integer>, Object> testHarness = TestUtils.createRescalingTestSink(outDir, 1, 0, 100L, 124L); ) { testHarness.setup(); testHarness.open(); } }
private void setUpTestHarness() throws Exception { sinkFunction = new ContentDumpSinkFunction(); harness = new OneInputStreamOperatorTestHarness<>(new StreamSink<>(sinkFunction), StringSerializer.INSTANCE); harness.setup(); }
new KeyedOneInputStreamOperatorTestHarness<>(operator, keySelector, keyType); testHarness.setup(); testHarness.open();
@Test public void testProcessingTimeTimers() throws Exception { LegacyKeyedProcessOperator<Integer, Integer, Integer> operator = new LegacyKeyedProcessOperator<>(new TriggeringFlatMapFunction(TimeDomain.PROCESSING_TIME)); OneInputStreamOperatorTestHarness<Integer, Integer> testHarness = new KeyedOneInputStreamOperatorTestHarness<>(operator, new IdentityKeySelector<Integer>(), BasicTypeInfo.INT_TYPE_INFO); testHarness.setup(); testHarness.open(); testHarness.processElement(new StreamRecord<>(17)); testHarness.setProcessingTime(5); ConcurrentLinkedQueue<Object> expectedOutput = new ConcurrentLinkedQueue<>(); expectedOutput.add(new StreamRecord<>(17)); expectedOutput.add(new StreamRecord<>(1777)); TestHarnessUtil.assertOutputEquals("Output was not correct.", expectedOutput, testHarness.getOutput()); testHarness.close(); }
@Test public void testProcessingTimeTimers() throws Exception { final int expectedKey = 17; KeyedProcessOperator<Integer, Integer, Integer> operator = new KeyedProcessOperator<>(new TriggeringFlatMapFunction(TimeDomain.PROCESSING_TIME, expectedKey)); OneInputStreamOperatorTestHarness<Integer, Integer> testHarness = new KeyedOneInputStreamOperatorTestHarness<>(operator, new IdentityKeySelector<Integer>(), BasicTypeInfo.INT_TYPE_INFO); testHarness.setup(); testHarness.open(); testHarness.processElement(new StreamRecord<>(expectedKey)); testHarness.setProcessingTime(5); ConcurrentLinkedQueue<Object> expectedOutput = new ConcurrentLinkedQueue<>(); expectedOutput.add(new StreamRecord<>(expectedKey)); expectedOutput.add(new StreamRecord<>(1777)); TestHarnessUtil.assertOutputEquals("Output was not correct.", expectedOutput, testHarness.getOutput()); testHarness.close(); }
@Test public void testEventTimeTimers() throws Exception { LegacyKeyedProcessOperator<Integer, Integer, Integer> operator = new LegacyKeyedProcessOperator<>(new TriggeringFlatMapFunction(TimeDomain.EVENT_TIME)); OneInputStreamOperatorTestHarness<Integer, Integer> testHarness = new KeyedOneInputStreamOperatorTestHarness<>(operator, new IdentityKeySelector<Integer>(), BasicTypeInfo.INT_TYPE_INFO); testHarness.setup(); testHarness.open(); testHarness.processWatermark(new Watermark(0)); testHarness.processElement(new StreamRecord<>(17, 42L)); testHarness.processWatermark(new Watermark(5)); ConcurrentLinkedQueue<Object> expectedOutput = new ConcurrentLinkedQueue<>(); expectedOutput.add(new Watermark(0L)); expectedOutput.add(new StreamRecord<>(17, 42L)); expectedOutput.add(new StreamRecord<>(1777, 5L)); expectedOutput.add(new Watermark(5L)); TestHarnessUtil.assertOutputEquals("Output was not correct.", expectedOutput, testHarness.getOutput()); testHarness.close(); }
@Test public void testTimestampAndWatermarkQuerying() throws Exception { ProcessOperator<Integer, String> operator = new ProcessOperator<>(new QueryingProcessFunction(TimeDomain.EVENT_TIME)); OneInputStreamOperatorTestHarness<Integer, String> testHarness = new OneInputStreamOperatorTestHarness<>(operator); testHarness.setup(); testHarness.open(); testHarness.processWatermark(new Watermark(17)); testHarness.processElement(new StreamRecord<>(5, 12L)); testHarness.processWatermark(new Watermark(42)); testHarness.processElement(new StreamRecord<>(6, 13L)); ConcurrentLinkedQueue<Object> expectedOutput = new ConcurrentLinkedQueue<>(); expectedOutput.add(new Watermark(17L)); expectedOutput.add(new StreamRecord<>("5TIME:17 TS:12", 12L)); expectedOutput.add(new Watermark(42L)); expectedOutput.add(new StreamRecord<>("6TIME:42 TS:13", 13L)); TestHarnessUtil.assertOutputEquals("Output was not correct.", expectedOutput, testHarness.getOutput()); testHarness.close(); }
@Test public void testEventTimeTimers() throws Exception { final int expectedKey = 17; KeyedProcessOperator<Integer, Integer, Integer> operator = new KeyedProcessOperator<>(new TriggeringFlatMapFunction(TimeDomain.EVENT_TIME, expectedKey)); OneInputStreamOperatorTestHarness<Integer, Integer> testHarness = new KeyedOneInputStreamOperatorTestHarness<>(operator, new IdentityKeySelector<Integer>(), BasicTypeInfo.INT_TYPE_INFO); testHarness.setup(); testHarness.open(); testHarness.processWatermark(new Watermark(0)); testHarness.processElement(new StreamRecord<>(expectedKey, 42L)); testHarness.processWatermark(new Watermark(5)); ConcurrentLinkedQueue<Object> expectedOutput = new ConcurrentLinkedQueue<>(); expectedOutput.add(new Watermark(0L)); expectedOutput.add(new StreamRecord<>(expectedKey, 42L)); expectedOutput.add(new StreamRecord<>(1777, 5L)); expectedOutput.add(new Watermark(5L)); TestHarnessUtil.assertOutputEquals("Output was not correct.", expectedOutput, testHarness.getOutput()); testHarness.close(); }
@Test public void testTimestampAndProcessingTimeQuerying() throws Exception { ProcessOperator<Integer, String> operator = new ProcessOperator<>(new QueryingProcessFunction(TimeDomain.PROCESSING_TIME)); OneInputStreamOperatorTestHarness<Integer, String> testHarness = new OneInputStreamOperatorTestHarness<>(operator); testHarness.setup(); testHarness.open(); testHarness.setProcessingTime(17); testHarness.processElement(new StreamRecord<>(5)); testHarness.setProcessingTime(42); testHarness.processElement(new StreamRecord<>(6)); ConcurrentLinkedQueue<Object> expectedOutput = new ConcurrentLinkedQueue<>(); expectedOutput.add(new StreamRecord<>("5TIME:17 TS:null")); expectedOutput.add(new StreamRecord<>("6TIME:42 TS:null")); TestHarnessUtil.assertOutputEquals("Output was not correct.", expectedOutput, testHarness.getOutput()); testHarness.close(); }
@Test public void testTimestampAndWatermarkQuerying() throws Exception { KeyedProcessOperator<Integer, Integer, String> operator = new KeyedProcessOperator<>(new QueryingFlatMapFunction(TimeDomain.EVENT_TIME)); OneInputStreamOperatorTestHarness<Integer, String> testHarness = new KeyedOneInputStreamOperatorTestHarness<>(operator, new IdentityKeySelector<Integer>(), BasicTypeInfo.INT_TYPE_INFO); testHarness.setup(); testHarness.open(); testHarness.processWatermark(new Watermark(17)); testHarness.processElement(new StreamRecord<>(5, 12L)); testHarness.processWatermark(new Watermark(42)); testHarness.processElement(new StreamRecord<>(6, 13L)); ConcurrentLinkedQueue<Object> expectedOutput = new ConcurrentLinkedQueue<>(); expectedOutput.add(new Watermark(17L)); expectedOutput.add(new StreamRecord<>("5TIME:17 TS:12", 12L)); expectedOutput.add(new Watermark(42L)); expectedOutput.add(new StreamRecord<>("6TIME:42 TS:13", 13L)); TestHarnessUtil.assertOutputEquals("Output was not correct.", expectedOutput, testHarness.getOutput()); testHarness.close(); }
@Test public void testTimestampAndWatermarkQuerying() throws Exception { LegacyKeyedProcessOperator<Integer, Integer, String> operator = new LegacyKeyedProcessOperator<>(new QueryingFlatMapFunction(TimeDomain.EVENT_TIME)); OneInputStreamOperatorTestHarness<Integer, String> testHarness = new KeyedOneInputStreamOperatorTestHarness<>(operator, new IdentityKeySelector<Integer>(), BasicTypeInfo.INT_TYPE_INFO); testHarness.setup(); testHarness.open(); testHarness.processWatermark(new Watermark(17)); testHarness.processElement(new StreamRecord<>(5, 12L)); testHarness.processWatermark(new Watermark(42)); testHarness.processElement(new StreamRecord<>(6, 13L)); ConcurrentLinkedQueue<Object> expectedOutput = new ConcurrentLinkedQueue<>(); expectedOutput.add(new Watermark(17L)); expectedOutput.add(new StreamRecord<>("5TIME:17 TS:12", 12L)); expectedOutput.add(new Watermark(42L)); expectedOutput.add(new StreamRecord<>("6TIME:42 TS:13", 13L)); TestHarnessUtil.assertOutputEquals("Output was not correct.", expectedOutput, testHarness.getOutput()); testHarness.close(); }
@Test public void testTimestampAndProcessingTimeQuerying() throws Exception { LegacyKeyedProcessOperator<Integer, Integer, String> operator = new LegacyKeyedProcessOperator<>(new QueryingFlatMapFunction(TimeDomain.PROCESSING_TIME)); OneInputStreamOperatorTestHarness<Integer, String> testHarness = new KeyedOneInputStreamOperatorTestHarness<>(operator, new IdentityKeySelector<Integer>(), BasicTypeInfo.INT_TYPE_INFO); testHarness.setup(); testHarness.open(); testHarness.setProcessingTime(17); testHarness.processElement(new StreamRecord<>(5)); testHarness.setProcessingTime(42); testHarness.processElement(new StreamRecord<>(6)); ConcurrentLinkedQueue<Object> expectedOutput = new ConcurrentLinkedQueue<>(); expectedOutput.add(new StreamRecord<>("5TIME:17 TS:null")); expectedOutput.add(new StreamRecord<>("6TIME:42 TS:null")); TestHarnessUtil.assertOutputEquals("Output was not correct.", expectedOutput, testHarness.getOutput()); testHarness.close(); }
@Test public void testTimestampAndProcessingTimeQuerying() throws Exception { KeyedProcessOperator<Integer, Integer, String> operator = new KeyedProcessOperator<>(new QueryingFlatMapFunction(TimeDomain.PROCESSING_TIME)); OneInputStreamOperatorTestHarness<Integer, String> testHarness = new KeyedOneInputStreamOperatorTestHarness<>(operator, new IdentityKeySelector<Integer>(), BasicTypeInfo.INT_TYPE_INFO); testHarness.setup(); testHarness.open(); testHarness.setProcessingTime(17); testHarness.processElement(new StreamRecord<>(5)); testHarness.setProcessingTime(42); testHarness.processElement(new StreamRecord<>(6)); ConcurrentLinkedQueue<Object> expectedOutput = new ConcurrentLinkedQueue<>(); expectedOutput.add(new StreamRecord<>("5TIME:17 TS:null")); expectedOutput.add(new StreamRecord<>("6TIME:42 TS:null")); TestHarnessUtil.assertOutputEquals("Output was not correct.", expectedOutput, testHarness.getOutput()); testHarness.close(); }
@Test public void testSnapshotAndRestore() throws Exception { LegacyKeyedProcessOperator<Integer, Integer, String> operator = new LegacyKeyedProcessOperator<>(new BothTriggeringFlatMapFunction()); OneInputStreamOperatorTestHarness<Integer, String> testHarness = new KeyedOneInputStreamOperatorTestHarness<>(operator, new IdentityKeySelector<Integer>(), BasicTypeInfo.INT_TYPE_INFO); testHarness.setup(); testHarness.open(); testHarness.processElement(new StreamRecord<>(5, 12L)); // snapshot and restore from scratch OperatorSubtaskState snapshot = testHarness.snapshot(0, 0); testHarness.close(); operator = new LegacyKeyedProcessOperator<>(new BothTriggeringFlatMapFunction()); testHarness = new KeyedOneInputStreamOperatorTestHarness<>(operator, new IdentityKeySelector<Integer>(), BasicTypeInfo.INT_TYPE_INFO); testHarness.setup(); testHarness.initializeState(snapshot); testHarness.open(); testHarness.setProcessingTime(5); testHarness.processWatermark(new Watermark(6)); ConcurrentLinkedQueue<Object> expectedOutput = new ConcurrentLinkedQueue<>(); expectedOutput.add(new StreamRecord<>("PROC:1777")); expectedOutput.add(new StreamRecord<>("EVENT:1777", 6L)); expectedOutput.add(new Watermark(6)); TestHarnessUtil.assertOutputEquals("Output was not correct.", expectedOutput, testHarness.getOutput()); testHarness.close(); }
@Test public void testNullOutputTagRefusal() throws Exception { ProcessOperator<Integer, String> operator = new ProcessOperator<>(new NullOutputTagEmittingProcessFunction()); OneInputStreamOperatorTestHarness<Integer, String> testHarness = new OneInputStreamOperatorTestHarness<>(operator); testHarness.setup(); testHarness.open(); testHarness.setProcessingTime(17); try { expectedException.expect(IllegalArgumentException.class); testHarness.processElement(new StreamRecord<>(5)); } finally { testHarness.close(); } }
@Test public void testNullOutputTagRefusal() throws Exception { LegacyKeyedProcessOperator<Integer, Integer, String> operator = new LegacyKeyedProcessOperator<>(new NullOutputTagEmittingProcessFunction()); OneInputStreamOperatorTestHarness<Integer, String> testHarness = new KeyedOneInputStreamOperatorTestHarness<>( operator, new IdentityKeySelector<>(), BasicTypeInfo.INT_TYPE_INFO); testHarness.setup(); testHarness.open(); testHarness.setProcessingTime(17); try { expectedException.expect(IllegalArgumentException.class); testHarness.processElement(new StreamRecord<>(5)); } finally { testHarness.close(); } }
@Test public void testNullOutputTagRefusal() throws Exception { KeyedProcessOperator<Integer, Integer, String> operator = new KeyedProcessOperator<>(new NullOutputTagEmittingProcessFunction()); OneInputStreamOperatorTestHarness<Integer, String> testHarness = new KeyedOneInputStreamOperatorTestHarness<>( operator, new IdentityKeySelector<>(), BasicTypeInfo.INT_TYPE_INFO); testHarness.setup(); testHarness.open(); testHarness.setProcessingTime(17); try { expectedException.expect(IllegalArgumentException.class); testHarness.processElement(new StreamRecord<>(5)); } finally { testHarness.close(); } }
@Test public void testSnapshotAndRestoreWrappedCheckpointedFunction() throws Exception { StreamMap<Integer, Integer> operator = new StreamMap<>( new WrappingTestFun(new WrappingTestFun(new InnerTestFun()))); OneInputStreamOperatorTestHarness<Integer, Integer> testHarness = new OneInputStreamOperatorTestHarness<>(operator); testHarness.setup(); testHarness.open(); testHarness.processElement(new StreamRecord<>(5, 12L)); // snapshot and restore from scratch OperatorSubtaskState snapshot = testHarness.snapshot(0, 0); testHarness.close(); InnerTestFun innerTestFun = new InnerTestFun(); operator = new StreamMap<>(new WrappingTestFun(new WrappingTestFun(innerTestFun))); testHarness = new OneInputStreamOperatorTestHarness<>(operator); testHarness.setup(); testHarness.initializeState(snapshot); testHarness.open(); Assert.assertTrue(innerTestFun.wasRestored); testHarness.close(); }
@Test public void testSnapshotAndRestoreWrappedListCheckpointed() throws Exception { StreamMap<Integer, Integer> operator = new StreamMap<>( new WrappingTestFun(new WrappingTestFun(new InnerTestFunList()))); OneInputStreamOperatorTestHarness<Integer, Integer> testHarness = new OneInputStreamOperatorTestHarness<>(operator); testHarness.setup(); testHarness.open(); testHarness.processElement(new StreamRecord<>(5, 12L)); // snapshot and restore from scratch OperatorSubtaskState snapshot = testHarness.snapshot(0, 0); testHarness.close(); InnerTestFunList innerTestFun = new InnerTestFunList(); operator = new StreamMap<>(new WrappingTestFun(new WrappingTestFun(innerTestFun))); testHarness = new OneInputStreamOperatorTestHarness<>(operator); testHarness.setup(); testHarness.initializeState(snapshot); testHarness.open(); Assert.assertTrue(innerTestFun.wasRestored); testHarness.close(); }