@Test public void testNullOutputTagRefusal() throws Exception { ProcessOperator<Integer, String> operator = new ProcessOperator<>(new NullOutputTagEmittingProcessFunction()); OneInputStreamOperatorTestHarness<Integer, String> testHarness = new OneInputStreamOperatorTestHarness<>(operator); testHarness.setup(); testHarness.open(); testHarness.setProcessingTime(17); try { expectedException.expect(IllegalArgumentException.class); testHarness.processElement(new StreamRecord<>(5)); } finally { testHarness.close(); } }
new OneInputStreamOperatorTestHarness<>(sink); testHarness.open(); testHarness.processElement(new StreamRecord<>(generateValue(elementCounter, 0))); elementCounter++; OperatorSubtaskState latestSnapshot = testHarness.snapshot(snapshotCount++, 0); testHarness.notifyOfCompletedCheckpoint(snapshotCount - 1); testHarness.processElement(new StreamRecord<>(generateValue(elementCounter, 1))); elementCounter++; testHarness.close(); testHarness = new OneInputStreamOperatorTestHarness<>(sink); testHarness.setup(); testHarness.initializeState(latestSnapshot); testHarness.open(); testHarness.processElement(new StreamRecord<>(generateValue(elementCounter, 2))); elementCounter++; testHarness.snapshot(snapshotCount++, 0); testHarness.notifyOfCompletedCheckpoint(snapshotCount - 1);
@Test public void testOpenClose() throws Exception { StreamFilter<String> operator = new StreamFilter<String>(new TestOpenCloseFilterFunction()); OneInputStreamOperatorTestHarness<String, String> testHarness = new OneInputStreamOperatorTestHarness<String, String>(operator); long initialTime = 0L; testHarness.open(); testHarness.processElement(new StreamRecord<String>("fooHello", initialTime)); testHarness.processElement(new StreamRecord<String>("bar", initialTime)); testHarness.close(); Assert.assertTrue("RichFunction methods where not called.", TestOpenCloseFilterFunction.closeCalled); Assert.assertTrue("Output contains no elements.", testHarness.getOutput().size() > 0); }
@Override public void go() throws Exception { synchronized (snapshotHarness.getCheckpointLock()) { lastElement.trigger(); snapshotHarness.processElement(capacity, 0L); } } };
@Test public void testNotifyOfCompletedCheckpoint() throws Exception { harness.open(); harness.processElement("42", 0); harness.snapshot(0, 1); harness.processElement("43", 2); harness.snapshot(1, 3); harness.processElement("44", 4); harness.snapshot(2, 5); harness.notifyOfCompletedCheckpoint(1); assertExactlyOnce(Arrays.asList("42", "43")); assertEquals(2, tmpDirectory.listFiles().size()); // one for checkpointId 2 and second for the currentTransaction }
/** * Extracts the result values form the test harness and clear the output queue. */ @SuppressWarnings({"unchecked", "rawtypes"}) private <T> List<T> extractResult(OneInputStreamOperatorTestHarness<?, T> testHarness) { List<StreamRecord<? extends T>> streamRecords = testHarness.extractOutputStreamRecords(); List<T> result = new ArrayList<>(); for (Object in : streamRecords) { if (in instanceof StreamRecord) { result.add((T) ((StreamRecord) in).getValue()); } } testHarness.getOutput().clear(); return result; }
new OneInputStreamOperatorTestHarness<>(operator); testHarness.setup(); testHarness.open(); testHarness.processElement(new StreamRecord<>(42, 17L /* timestamp */)); TestHarnessUtil.assertOutputEquals("Output was not correct.", expectedOutput, testHarness.getOutput()); testHarness.getSideOutput(SideOutputProcessFunction.INTEGER_OUTPUT_TAG); TestHarnessUtil.assertOutputEquals( "Side output was not correct.", expectedLongSideOutput.add(new StreamRecord<>(42L, 17L /* timestamp */)); ConcurrentLinkedQueue<StreamRecord<Long>> longSideOutput = testHarness.getSideOutput(SideOutputProcessFunction.LONG_OUTPUT_TAG); TestHarnessUtil.assertOutputEquals( "Side output was not correct.", longSideOutput); testHarness.close();
private void testTimeoutExceptionHandling(AsyncDataStream.OutputMode outputMode) throws Exception { AsyncFunction<Integer, Integer> asyncFunction = new NoOpAsyncFunction<>(); long timeout = 10L; // 1 milli second AsyncWaitOperator<Integer, Integer> asyncWaitOperator = new AsyncWaitOperator<>( asyncFunction, timeout, 2, outputMode); final MockEnvironment mockEnvironment = createMockEnvironment(); mockEnvironment.setExpectedExternalFailureCause(Throwable.class); OneInputStreamOperatorTestHarness<Integer, Integer> harness = new OneInputStreamOperatorTestHarness<>( asyncWaitOperator, IntSerializer.INSTANCE, mockEnvironment); harness.open(); synchronized (harness.getCheckpointLock()) { harness.processElement(1, 1L); } harness.setProcessingTime(10L); synchronized (harness.getCheckpointLock()) { harness.close(); } }
private void testUserExceptionHandling(AsyncDataStream.OutputMode outputMode) throws Exception { UserExceptionAsyncFunction asyncWaitFunction = new UserExceptionAsyncFunction(); long timeout = 2000L; AsyncWaitOperator<Integer, Integer> asyncWaitOperator = new AsyncWaitOperator<>( asyncWaitFunction, TIMEOUT, 2, outputMode); final MockEnvironment mockEnvironment = createMockEnvironment(); mockEnvironment.setExpectedExternalFailureCause(Throwable.class); OneInputStreamOperatorTestHarness<Integer, Integer> harness = new OneInputStreamOperatorTestHarness<>( asyncWaitOperator, IntSerializer.INSTANCE, mockEnvironment); harness.open(); synchronized (harness.getCheckpointLock()) { harness.processElement(1, 1L); } synchronized (harness.getCheckpointLock()) { harness.close(); } assertTrue(harness.getEnvironment().getActualExternalFailureCause().isPresent()); }
@Test public void testAssignerIsInvokedOncePerElement() throws Exception { WindowAssigner<Integer, TimeWindow> mockAssigner = mockTimeWindowAssigner(); Trigger<Integer, TimeWindow> mockTrigger = mockTrigger(); InternalWindowFunction<Iterable<Integer>, Void, Integer, TimeWindow> mockWindowFunction = mockWindowFunction(); OneInputStreamOperatorTestHarness<Integer, Void> testHarness = createWindowOperator(mockAssigner, mockTrigger, 0L, mockWindowFunction); testHarness.open(); when(mockAssigner.assignWindows(anyInt(), anyLong(), anyAssignerContext())) .thenReturn(Collections.singletonList(new TimeWindow(0, 0))); testHarness.processElement(new StreamRecord<>(0, 0L)); verify(mockAssigner, times(1)).assignWindows(eq(0), eq(0L), anyAssignerContext()); testHarness.processElement(new StreamRecord<>(0, 0L)); verify(mockAssigner, times(2)).assignWindows(eq(0), eq(0L), anyAssignerContext()); }
@Test public void testKeyedCEPOperatorWatermarkForwarding() throws Exception { OneInputStreamOperatorTestHarness<Event, Map<String, List<Event>>> harness = getCepTestHarness(false); try { harness.open(); Watermark expectedWatermark = new Watermark(42L); harness.processWatermark(expectedWatermark); verifyWatermark(harness.getOutput().poll(), 42L); } finally { harness.close(); } }
@Test public void testLogTimeoutAlmostReachedWarningDuringRecovery() throws Exception { clock.setEpochMilli(0); final long transactionTimeout = 1000; final double warningRatio = 0.5; sinkFunction.setTransactionTimeout(transactionTimeout); sinkFunction.enableTransactionTimeoutWarnings(warningRatio); harness.open(); final OperatorSubtaskState snapshot = harness.snapshot(0, 1); final long elapsedTime = (long) ((double) transactionTimeout * warningRatio + 2); clock.setEpochMilli(elapsedTime); closeTestHarness(); setUpTestHarness(); sinkFunction.setTransactionTimeout(transactionTimeout); sinkFunction.enableTransactionTimeoutWarnings(warningRatio); harness.initializeState(snapshot); harness.open(); final List<String> logMessages = loggingEvents.stream().map(LoggingEvent::getRenderedMessage).collect(Collectors.toList()); closeTestHarness(); assertThat( logMessages, hasItem(containsString("has been open for 502 ms. " + "This is close to or even exceeding the transaction timeout of 1000 ms."))); }
@Test public void testProcessingTimestampisPassedToNFA() throws Exception { final NFA<Event> nfa = NFACompiler.compileFactory(Pattern.<Event>begin("begin"), true).createNFA(); final NFA<Event> spyNFA = spy(nfa); try ( OneInputStreamOperatorTestHarness<Event, Map<String, List<Event>>> harness = CepOperatorTestUtilities.getCepTestHarness(createOperatorForNFA(spyNFA).build())) { long timestamp = 5; harness.open(); harness.setProcessingTime(timestamp); StreamRecord<Event> event = event().withTimestamp(3).asStreamRecord(); harness.processElement(event); verify(spyNFA).process( any(SharedBufferAccessor.class), any(NFAState.class), eq(event.getValue()), eq(timestamp), any(AfterMatchSkipStrategy.class), any(TimerService.class)); } }
createWindowOperator(mockAssigner, mockTrigger, 0L, windowFunction); testHarness.open(); testHarness.processElement(new StreamRecord<>(17, 5L)); assertThat(testHarness.getSideOutput(integerOutputTag), contains(isStreamRecord(17, windowEnd - 1))); assertThat(testHarness.getSideOutput(longOutputTag), contains(isStreamRecord(17L, windowEnd - 1)));
@Test public void testLogTimeoutAlmostReachedWarningDuringCommit() throws Exception { clock.setEpochMilli(0); final long transactionTimeout = 1000; final double warningRatio = 0.5; sinkFunction.setTransactionTimeout(transactionTimeout); sinkFunction.enableTransactionTimeoutWarnings(warningRatio); harness.open(); harness.snapshot(0, 1); final long elapsedTime = (long) ((double) transactionTimeout * warningRatio + 2); clock.setEpochMilli(elapsedTime); harness.notifyOfCompletedCheckpoint(1); final List<String> logMessages = loggingEvents.stream().map(LoggingEvent::getRenderedMessage).collect(Collectors.toList()); assertThat( logMessages, hasItem(containsString("has been open for 502 ms. " + "This is close to or even exceeding the transaction timeout of 1000 ms."))); }
@Override public void go() throws Exception { testHarness.snapshot(1L, 1000L); } };
@Test public void testClosingWithoutInput() throws Exception { final File outDir = TEMP_FOLDER.newFolder(); try ( OneInputStreamOperatorTestHarness<Tuple2<String, Integer>, Object> testHarness = TestUtils.createRescalingTestSink(outDir, 1, 0, 100L, 124L); ) { testHarness.setup(); testHarness.open(); } }
private void setUpTestHarness() throws Exception { sinkFunction = new ContentDumpSinkFunction(); harness = new OneInputStreamOperatorTestHarness<>(new StreamSink<>(sinkFunction), StringSerializer.INSTANCE); harness.setup(); }
private static void closeSilently(OneInputStreamOperatorTestHarness<?, ?> harness) { if (harness != null) { try { harness.close(); } catch (Throwable ignored) { } } }
static OneInputStreamOperatorTestHarness<Tuple2<String, Integer>, Object> createTestSinkWithBulkEncoder( final File outDir, final int totalParallelism, final int taskIdx, final long bucketCheckInterval, final BucketAssigner<Tuple2<String, Integer>, String> bucketer, final BulkWriter.Factory<Tuple2<String, Integer>> writer, final BucketFactory<Tuple2<String, Integer>, String> bucketFactory) throws Exception { StreamingFileSink<Tuple2<String, Integer>> sink = StreamingFileSink .forBulkFormat(new Path(outDir.toURI()), writer) .withBucketAssigner(bucketer) .withBucketCheckInterval(bucketCheckInterval) .withBucketFactory(bucketFactory) .build(); return new OneInputStreamOperatorTestHarness<>(new StreamSink<>(sink), 10, totalParallelism, taskIdx); }