private void setUpTestHarness() throws Exception { sinkFunction = new ContentDumpSinkFunction(); harness = new OneInputStreamOperatorTestHarness<>(new StreamSink<>(sinkFunction), StringSerializer.INSTANCE); harness.setup(); }
private OneInputStreamOperatorTestHarness<TimestampedFileInputSplit, String> getTestHarness( BlockingFileInputFormat format, int noOfTasks, int taksIdx) throws Exception { ContinuousFileReaderOperator<String> reader = new ContinuousFileReaderOperator<>(format); reader.setOutputType(TypeExtractor.getInputFormatTypes(format), new ExecutionConfig()); OneInputStreamOperatorTestHarness<TimestampedFileInputSplit, String> testHarness = new OneInputStreamOperatorTestHarness<>(reader, 10, noOfTasks, taksIdx); testHarness.setTimeCharacteristic(TimeCharacteristic.EventTime); return testHarness; }
/** * This test is meant to assure that testAtLeastOnceSink is valid by testing that if flushing is disabled, * the snapshot method does indeed finishes without waiting for pending requests; * we set a timeout because the test will not finish if the logic is broken. */ @Test(timeout = 5000) public void testDoesNotWaitForPendingRequestsIfFlushingDisabled() throws Exception { final DummyElasticsearchSink<String> sink = new DummyElasticsearchSink<>( new HashMap<String, String>(), new SimpleSinkFunction<String>(), new DummyRetryFailureHandler()); sink.disableFlushOnCheckpoint(); // disable flushing final OneInputStreamOperatorTestHarness<String, Object> testHarness = new OneInputStreamOperatorTestHarness<>(new StreamSink<>(sink)); testHarness.open(); // setup the next bulk request, and let bulk request succeed sink.setMockItemFailuresListForNextBulkItemResponses(Collections.singletonList(new Exception("artificial failure for record"))); testHarness.processElement(new StreamRecord<>("msg-1")); verify(sink.getMockBulkProcessor(), times(1)).add(any(IndexRequest.class)); // the snapshot should not block even though we haven't flushed the bulk request testHarness.snapshot(1L, 1000L); testHarness.close(); }
static OneInputStreamOperatorTestHarness<Tuple2<String, Integer>, Object> createTestSinkWithBulkEncoder( final File outDir, final int totalParallelism, final int taskIdx, final long bucketCheckInterval, final BucketAssigner<Tuple2<String, Integer>, String> bucketer, final BulkWriter.Factory<Tuple2<String, Integer>> writer, final BucketFactory<Tuple2<String, Integer>, String> bucketFactory) throws Exception { StreamingFileSink<Tuple2<String, Integer>> sink = StreamingFileSink .forBulkFormat(new Path(outDir.toURI()), writer) .withBucketAssigner(bucketer) .withBucketCheckInterval(bucketCheckInterval) .withBucketFactory(bucketFactory) .build(); return new OneInputStreamOperatorTestHarness<>(new StreamSink<>(sink), 10, totalParallelism, taskIdx); }
@Test public void testMap() throws Exception { StreamMap<Integer, String> operator = new StreamMap<Integer, String>(new Map()); OneInputStreamOperatorTestHarness<Integer, String> testHarness = new OneInputStreamOperatorTestHarness<Integer, String>(operator); long initialTime = 0L; ConcurrentLinkedQueue<Object> expectedOutput = new ConcurrentLinkedQueue<Object>(); testHarness.open(); testHarness.processElement(new StreamRecord<Integer>(1, initialTime + 1)); testHarness.processElement(new StreamRecord<Integer>(2, initialTime + 2)); testHarness.processWatermark(new Watermark(initialTime + 2)); testHarness.processElement(new StreamRecord<Integer>(3, initialTime + 3)); expectedOutput.add(new StreamRecord<String>("+2", initialTime + 1)); expectedOutput.add(new StreamRecord<String>("+3", initialTime + 2)); expectedOutput.add(new Watermark(initialTime + 2)); expectedOutput.add(new StreamRecord<String>("+4", initialTime + 3)); TestHarnessUtil.assertOutputEquals("Output was not correct.", expectedOutput, testHarness.getOutput()); }
@Test public void testOpenClose() throws Exception { StreamMap<String, String> operator = new StreamMap<String, String>(new TestOpenCloseMapFunction()); OneInputStreamOperatorTestHarness<String, String> testHarness = new OneInputStreamOperatorTestHarness<String, String>(operator); long initialTime = 0L; testHarness.open(); testHarness.processElement(new StreamRecord<String>("Hello", initialTime)); testHarness.close(); Assert.assertTrue("RichFunction methods where not called.", TestOpenCloseMapFunction.closeCalled); Assert.assertTrue("Output contains no elements.", testHarness.getOutput().size() > 0); }
/** * This test is meant to assure that testAtLeastOnceProducer is valid by testing that if flushing is disabled, * the snapshot method does indeed finishes without waiting for pending records; * we set a timeout because the test will not finish if the logic is broken. */ @SuppressWarnings("unchecked") @Test(timeout = 5000) public void testDoesNotWaitForPendingRecordsIfFlushingDisabled() throws Throwable { final DummyFlinkKafkaProducer<String> producer = new DummyFlinkKafkaProducer<>( FakeStandardProducerConfig.get(), new KeyedSerializationSchemaWrapper<>(new SimpleStringSchema()), null); producer.setFlushOnCheckpoint(false); final KafkaProducer<?, ?> mockProducer = producer.getMockKafkaProducer(); final OneInputStreamOperatorTestHarness<String, Object> testHarness = new OneInputStreamOperatorTestHarness<>(new StreamSink<>(producer)); testHarness.open(); testHarness.processElement(new StreamRecord<>("msg")); // make sure that all callbacks have not been completed verify(mockProducer, times(1)).send(any(ProducerRecord.class), any(Callback.class)); // should return even if there are pending records testHarness.snapshot(123L, 123L); testHarness.close(); }
@Test public void testOpenClose() throws Exception { StreamFlatMap<String, String> operator = new StreamFlatMap<String, String>(new TestOpenCloseFlatMapFunction()); OneInputStreamOperatorTestHarness<String, String> testHarness = new OneInputStreamOperatorTestHarness<String, String>(operator); long initialTime = 0L; testHarness.open(); testHarness.processElement(new StreamRecord<String>("Hello", initialTime)); testHarness.close(); Assert.assertTrue("RichFunction methods where not called.", TestOpenCloseFlatMapFunction.closeCalled); Assert.assertTrue("Output contains no elements.", testHarness.getOutput().size() > 0); }
@Test public void testZeroOnNegativeTimestamps() throws Exception { final AssignerWithPunctuatedWatermarks<Long> assigner = new NeverWatermarkExtractor(); final TimestampsAndPunctuatedWatermarksOperator<Long> operator = new TimestampsAndPunctuatedWatermarksOperator<Long>(assigner); OneInputStreamOperatorTestHarness<Long, Long> testHarness = new OneInputStreamOperatorTestHarness<Long, Long>(operator); testHarness.open(); long[] values = { Long.MIN_VALUE, -1L, 0L, 1L, 2L, 3L, Long.MAX_VALUE }; for (long value : values) { testHarness.processElement(new StreamRecord<>(value)); } ConcurrentLinkedQueue<Object> output = testHarness.getOutput(); for (long value: values) { assertEquals(value, ((StreamRecord<?>) output.poll()).getTimestamp()); } }
static OneInputStreamOperatorTestHarness<Tuple2<String, Integer>, Object> createCustomRescalingTestSink( final File outDir, final int totalParallelism, final int taskIdx, final long bucketCheckInterval, final BucketAssigner<Tuple2<String, Integer>, String> bucketer, final Encoder<Tuple2<String, Integer>> writer, final RollingPolicy<Tuple2<String, Integer>, String> rollingPolicy, final BucketFactory<Tuple2<String, Integer>, String> bucketFactory) throws Exception { StreamingFileSink<Tuple2<String, Integer>> sink = StreamingFileSink .forRowFormat(new Path(outDir.toURI()), writer) .withBucketAssigner(bucketer) .withRollingPolicy(rollingPolicy) .withBucketCheckInterval(bucketCheckInterval) .withBucketFactory(bucketFactory) .build(); return new OneInputStreamOperatorTestHarness<>(new StreamSink<>(sink), 10, totalParallelism, taskIdx); }
/** * Test ensuring that if a snapshot call happens right after an async exception is caught, it should be rethrown. */ @Test public void testAsyncErrorRethrownOnCheckpoint() throws Throwable { final DummyFlinkKafkaProducer<String> producer = new DummyFlinkKafkaProducer<>( FakeStandardProducerConfig.get(), new KeyedSerializationSchemaWrapper<>(new SimpleStringSchema()), null); OneInputStreamOperatorTestHarness<String, Object> testHarness = new OneInputStreamOperatorTestHarness<>(new StreamSink<>(producer)); testHarness.open(); testHarness.processElement(new StreamRecord<>("msg-1")); // let the message request return an async exception producer.getPendingCallbacks().get(0).onCompletion(null, new Exception("artificial async exception")); try { testHarness.snapshot(123L, 123L); } catch (Exception e) { // the next invoke should rethrow the async exception Assert.assertTrue(e.getCause().getMessage().contains("artificial async exception")); // test succeeded return; } Assert.fail(); }
@Test public void testTimestampAndWatermarkQuerying() throws Exception { ProcessOperator<Integer, String> operator = new ProcessOperator<>(new QueryingProcessFunction(TimeDomain.EVENT_TIME)); OneInputStreamOperatorTestHarness<Integer, String> testHarness = new OneInputStreamOperatorTestHarness<>(operator); testHarness.setup(); testHarness.open(); testHarness.processWatermark(new Watermark(17)); testHarness.processElement(new StreamRecord<>(5, 12L)); testHarness.processWatermark(new Watermark(42)); testHarness.processElement(new StreamRecord<>(6, 13L)); ConcurrentLinkedQueue<Object> expectedOutput = new ConcurrentLinkedQueue<>(); expectedOutput.add(new Watermark(17L)); expectedOutput.add(new StreamRecord<>("5TIME:17 TS:12", 12L)); expectedOutput.add(new Watermark(42L)); expectedOutput.add(new StreamRecord<>("6TIME:42 TS:13", 13L)); TestHarnessUtil.assertOutputEquals("Output was not correct.", expectedOutput, testHarness.getOutput()); testHarness.close(); }
@Test public void testOpenClose() throws Exception { StreamFilter<String> operator = new StreamFilter<String>(new TestOpenCloseFilterFunction()); OneInputStreamOperatorTestHarness<String, String> testHarness = new OneInputStreamOperatorTestHarness<String, String>(operator); long initialTime = 0L; testHarness.open(); testHarness.processElement(new StreamRecord<String>("fooHello", initialTime)); testHarness.processElement(new StreamRecord<String>("bar", initialTime)); testHarness.close(); Assert.assertTrue("RichFunction methods where not called.", TestOpenCloseFilterFunction.closeCalled); Assert.assertTrue("Output contains no elements.", testHarness.getOutput().size() > 0); }
@Test public void testTimestampAndProcessingTimeQuerying() throws Exception { ProcessOperator<Integer, String> operator = new ProcessOperator<>(new QueryingProcessFunction(TimeDomain.PROCESSING_TIME)); OneInputStreamOperatorTestHarness<Integer, String> testHarness = new OneInputStreamOperatorTestHarness<>(operator); testHarness.setup(); testHarness.open(); testHarness.setProcessingTime(17); testHarness.processElement(new StreamRecord<>(5)); testHarness.setProcessingTime(42); testHarness.processElement(new StreamRecord<>(6)); ConcurrentLinkedQueue<Object> expectedOutput = new ConcurrentLinkedQueue<>(); expectedOutput.add(new StreamRecord<>("5TIME:17 TS:null")); expectedOutput.add(new StreamRecord<>("6TIME:42 TS:null")); TestHarnessUtil.assertOutputEquals("Output was not correct.", expectedOutput, testHarness.getOutput()); testHarness.close(); }
@Test public void testNegativeTimestamps() throws Exception { final AssignerWithPeriodicWatermarks<Long> assigner = new NeverWatermarkExtractor(); final TimestampsAndPeriodicWatermarksOperator<Long> operator = new TimestampsAndPeriodicWatermarksOperator<Long>(assigner); OneInputStreamOperatorTestHarness<Long, Long> testHarness = new OneInputStreamOperatorTestHarness<Long, Long>(operator); testHarness.getExecutionConfig().setAutoWatermarkInterval(50); testHarness.open(); long[] values = { Long.MIN_VALUE, -1L, 0L, 1L, 2L, 3L, Long.MAX_VALUE }; for (long value : values) { testHarness.processElement(new StreamRecord<>(value)); } ConcurrentLinkedQueue<Object> output = testHarness.getOutput(); for (long value: values) { assertEquals(value, ((StreamRecord<?>) output.poll()).getTimestamp()); } }
@Test @SuppressWarnings("unchecked") public void testFilter() throws Exception { StreamFilter<Integer> operator = new StreamFilter<Integer>(new MyFilter()); OneInputStreamOperatorTestHarness<Integer, Integer> testHarness = new OneInputStreamOperatorTestHarness<Integer, Integer>(operator); long initialTime = 0L; ConcurrentLinkedQueue<Object> expectedOutput = new ConcurrentLinkedQueue<Object>(); testHarness.open(); testHarness.processElement(new StreamRecord<Integer>(1, initialTime + 1)); testHarness.processElement(new StreamRecord<Integer>(2, initialTime + 2)); testHarness.processWatermark(new Watermark(initialTime + 2)); testHarness.processElement(new StreamRecord<Integer>(3, initialTime + 3)); testHarness.processElement(new StreamRecord<Integer>(4, initialTime + 4)); testHarness.processElement(new StreamRecord<Integer>(5, initialTime + 5)); testHarness.processElement(new StreamRecord<Integer>(6, initialTime + 6)); testHarness.processElement(new StreamRecord<Integer>(7, initialTime + 7)); expectedOutput.add(new StreamRecord<Integer>(2, initialTime + 2)); expectedOutput.add(new Watermark(initialTime + 2)); expectedOutput.add(new StreamRecord<Integer>(4, initialTime + 4)); expectedOutput.add(new StreamRecord<Integer>(6, initialTime + 6)); TestHarnessUtil.assertOutputEquals("Output was not correct.", expectedOutput, testHarness.getOutput()); }
private void testUserExceptionHandling(AsyncDataStream.OutputMode outputMode) throws Exception { UserExceptionAsyncFunction asyncWaitFunction = new UserExceptionAsyncFunction(); long timeout = 2000L; AsyncWaitOperator<Integer, Integer> asyncWaitOperator = new AsyncWaitOperator<>( asyncWaitFunction, TIMEOUT, 2, outputMode); final MockEnvironment mockEnvironment = createMockEnvironment(); mockEnvironment.setExpectedExternalFailureCause(Throwable.class); OneInputStreamOperatorTestHarness<Integer, Integer> harness = new OneInputStreamOperatorTestHarness<>( asyncWaitOperator, IntSerializer.INSTANCE, mockEnvironment); harness.open(); synchronized (harness.getCheckpointLock()) { harness.processElement(1, 1L); } synchronized (harness.getCheckpointLock()) { harness.close(); } assertTrue(harness.getEnvironment().getActualExternalFailureCause().isPresent()); }
@Test public void testNullOutputTagRefusal() throws Exception { ProcessOperator<Integer, String> operator = new ProcessOperator<>(new NullOutputTagEmittingProcessFunction()); OneInputStreamOperatorTestHarness<Integer, String> testHarness = new OneInputStreamOperatorTestHarness<>(operator); testHarness.setup(); testHarness.open(); testHarness.setProcessingTime(17); try { expectedException.expect(IllegalArgumentException.class); testHarness.processElement(new StreamRecord<>(5)); } finally { testHarness.close(); } }
@Test public void testSnapshotAndRestoreWrappedCheckpointedFunction() throws Exception { StreamMap<Integer, Integer> operator = new StreamMap<>( new WrappingTestFun(new WrappingTestFun(new InnerTestFun()))); OneInputStreamOperatorTestHarness<Integer, Integer> testHarness = new OneInputStreamOperatorTestHarness<>(operator); testHarness.setup(); testHarness.open(); testHarness.processElement(new StreamRecord<>(5, 12L)); // snapshot and restore from scratch OperatorSubtaskState snapshot = testHarness.snapshot(0, 0); testHarness.close(); InnerTestFun innerTestFun = new InnerTestFun(); operator = new StreamMap<>(new WrappingTestFun(new WrappingTestFun(innerTestFun))); testHarness = new OneInputStreamOperatorTestHarness<>(operator); testHarness.setup(); testHarness.initializeState(snapshot); testHarness.open(); Assert.assertTrue(innerTestFun.wasRestored); testHarness.close(); }
@Test public void testSnapshotAndRestoreWrappedListCheckpointed() throws Exception { StreamMap<Integer, Integer> operator = new StreamMap<>( new WrappingTestFun(new WrappingTestFun(new InnerTestFunList()))); OneInputStreamOperatorTestHarness<Integer, Integer> testHarness = new OneInputStreamOperatorTestHarness<>(operator); testHarness.setup(); testHarness.open(); testHarness.processElement(new StreamRecord<>(5, 12L)); // snapshot and restore from scratch OperatorSubtaskState snapshot = testHarness.snapshot(0, 0); testHarness.close(); InnerTestFunList innerTestFun = new InnerTestFunList(); operator = new StreamMap<>(new WrappingTestFun(new WrappingTestFun(innerTestFun))); testHarness = new OneInputStreamOperatorTestHarness<>(operator); testHarness.setup(); testHarness.initializeState(snapshot); testHarness.open(); Assert.assertTrue(innerTestFun.wasRestored); testHarness.close(); }