new KeyedOneInputStreamOperatorTestHarness<>(testOperator, new TestKeySelector(), BasicTypeInfo.INT_TYPE_INFO); testHarness.open(); testHarness.setProcessingTime(0L); testHarness.processElement(new Tuple2<>(1, "SET_PROC_TIME_TIMER:20"), 0); testHarness.processElement(new Tuple2<>(0, "SET_STATE:HELLO"), 0); testHarness.processElement(new Tuple2<>(1, "SET_STATE:CIAO"), 0); testHarness.processElement(new Tuple2<>(0, "SET_PROC_TIME_TIMER:10"), 0); OperatorSubtaskState snapshot = testHarness.snapshot(0, 0); new KeyedOneInputStreamOperatorTestHarness<>( testOperator1, new TestKeySelector(), BasicTypeInfo.INT_TYPE_INFO); testHarness1.setProcessingTime(0L); testHarness1.setup(); testHarness1.initializeState(snapshot); testHarness1.open(); testHarness1.setProcessingTime(10L); testHarness1.setProcessingTime(20L);
new OutputTag<Tuple2<Map<String, List<Event>>, Long>>("timedOut") {}; final KeyedOneInputStreamOperatorTestHarness<Integer, Event, Map<String, List<Event>>> harness = new KeyedOneInputStreamOperatorTestHarness<>( new CepOperator<>( Event.createTypeSerializer(), rocksDBStateBackend.setDbStoragePath(rocksDbPath); harness.setStateBackend(rocksDBStateBackend); harness.setup( new KryoSerializer<>( (Class<Map<String, List<Event>>>) (Object) Map.class, new ExecutionConfig())); harness.open(); harness.processElement(new StreamRecord<>(startEvent, 3L)); harness.processWatermark(new Watermark(watermarkTimestamp1)); harness.processWatermark(new Watermark(watermarkTimestamp2)); Queue<Object> result = harness.getOutput(); Queue<StreamRecord<Tuple2<Map<String, List<Event>>, Long>>> sideOutput = harness.getSideOutput(timedOut); harness.close();
@Test public void testNoEventTimeGarbageCollectionTimerForLongMax() throws Exception { WindowAssigner<Integer, TimeWindow> mockAssigner = mockTimeWindowAssigner(); Trigger<Integer, TimeWindow> mockTrigger = mockTrigger(); InternalWindowFunction<Iterable<Integer>, Void, Integer, TimeWindow> mockWindowFunction = mockWindowFunction(); KeyedOneInputStreamOperatorTestHarness<Integer, Integer, Void> testHarness = createWindowOperator(mockAssigner, mockTrigger, 20L, mockWindowFunction); testHarness.open(); when(mockAssigner.assignWindows(anyInt(), anyLong(), anyAssignerContext())) .thenReturn(Arrays.asList(new TimeWindow(0, Long.MAX_VALUE - 10))); assertEquals(0, testHarness.getOutput().size()); assertEquals(0, testHarness.numKeyedStateEntries()); testHarness.processElement(new StreamRecord<>(0, 0L)); // just the window contents assertEquals(1, testHarness.numKeyedStateEntries()); // no GC timer assertEquals(0, testHarness.numEventTimeTimers()); assertEquals(0, testHarness.numProcessingTimeTimers()); }
/** * Ensure that we get some output from the given operator when pushing in an element and * setting watermark and processing time to {@code Long.MAX_VALUE}. */ private static <K, IN, OUT> void processElementAndEnsureOutput( OneInputStreamOperator<IN, OUT> operator, KeySelector<IN, K> keySelector, TypeInformation<K> keyType, IN element) throws Exception { KeyedOneInputStreamOperatorTestHarness<K, IN, OUT> testHarness = new KeyedOneInputStreamOperatorTestHarness<>( operator, keySelector, keyType); testHarness.open(); testHarness.setProcessingTime(0); testHarness.processWatermark(Long.MIN_VALUE); testHarness.processElement(new StreamRecord<>(element, 0)); // provoke any processing-time/event-time triggers testHarness.setProcessingTime(Long.MAX_VALUE); testHarness.processWatermark(Long.MAX_VALUE); // we at least get the two watermarks and should also see an output element assertTrue(testHarness.getOutput().size() >= 3); testHarness.close(); }
@Test public void testStateDoesNotInterfere() throws Exception { TestOperator testOperator = new TestOperator(); KeyedOneInputStreamOperatorTestHarness<Integer, Tuple2<Integer, String>, String> testHarness = new KeyedOneInputStreamOperatorTestHarness<>(testOperator, new TestKeySelector(), BasicTypeInfo.INT_TYPE_INFO); testHarness.open(); testHarness.processElement(new Tuple2<>(0, "SET_STATE:HELLO"), 0); testHarness.processElement(new Tuple2<>(1, "SET_STATE:CIAO"), 0); testHarness.processElement(new Tuple2<>(1, "EMIT_STATE"), 0); testHarness.processElement(new Tuple2<>(0, "EMIT_STATE"), 0); assertThat( extractResult(testHarness), contains("ON_ELEMENT:1:CIAO", "ON_ELEMENT:0:HELLO")); }
/** * Verify that timers for the different time domains don't clash. */ @Test public void testProcessingTimeAndEventTimeDontInterfere() throws Exception { TestOperator testOperator = new TestOperator(); KeyedOneInputStreamOperatorTestHarness<Integer, Tuple2<Integer, String>, String> testHarness = new KeyedOneInputStreamOperatorTestHarness<>(testOperator, new TestKeySelector(), BasicTypeInfo.INT_TYPE_INFO); testHarness.open(); testHarness.setProcessingTime(0L); testHarness.processWatermark(0L); testHarness.processElement(new Tuple2<>(0, "SET_PROC_TIME_TIMER:10"), 0); testHarness.processElement(new Tuple2<>(0, "SET_EVENT_TIME_TIMER:20"), 0); testHarness.processElement(new Tuple2<>(0, "SET_STATE:HELLO"), 0); testHarness.processWatermark(20L); assertThat( extractResult(testHarness), contains("ON_EVENT_TIME:HELLO")); testHarness.setProcessingTime(10L); assertThat( extractResult(testHarness), contains("ON_PROC_TIME:HELLO")); }
String, WindowedValue<KV<String, Integer>>, WindowedValue<KV<String, Integer>>> testHarness = new KeyedOneInputStreamOperatorTestHarness<>( doFnOperator, kvWindowedValue -> kvWindowedValue.getValue().getKey(), new CoderTypeInformation<>(StringUtf8Coder.of())); testHarness.open(); testHarness.processWatermark(0); assertEquals(0, testHarness.numKeyedStateEntries()); testHarness.processElement( new StreamRecord<>( WindowedValue.of(KV.of("key1", 5), new Instant(1), window1, PaneInfo.NO_FIRING))); testHarness.processElement( new StreamRecord<>( WindowedValue.of(KV.of("key2", 7), new Instant(3), window1, PaneInfo.NO_FIRING))); stripStreamRecordFromWindowedValue(testHarness.getOutput()), contains( WindowedValue.of( assertEquals(4, testHarness.numKeyedStateEntries()); testHarness.getOutput().clear();
harness.open(); harness.processElement(new StreamRecord<>( WindowedValue.valueInGlobalWindow(new ValueWithRecordId<>(key1, key1.getBytes())))); harness.processElement(new StreamRecord<>( WindowedValue.valueInGlobalWindow(new ValueWithRecordId<>(key2, key2.getBytes())))); harness.processElement(new StreamRecord<>( WindowedValue.valueInGlobalWindow(new ValueWithRecordId<>(key1, key1.getBytes())))); this.<String>stripStreamRecordFromWindowedValue(harness.getOutput()), contains(WindowedValue.valueInGlobalWindow(key1), WindowedValue.valueInGlobalWindow(key2))); OperatorStateHandles snapshot = harness.snapshot(0L, 0L); harness.close(); harness.setup(); harness.initializeState(snapshot); harness.open(); harness.processElement(new StreamRecord<>( WindowedValue.valueInGlobalWindow(new ValueWithRecordId<>(key2, key2.getBytes())))); harness.processElement(new StreamRecord<>( WindowedValue.valueInGlobalWindow(new ValueWithRecordId<>(key3, key3.getBytes()))));
createWindowOperator(mockAssigner, mockTrigger, 0L, mockWindowFunction); testHarness.open(); assertEquals(0, testHarness.getOutput().size()); assertEquals(0, testHarness.numKeyedStateEntries()); testHarness.processElement(new StreamRecord<>(0, 0L)); assertEquals(5, testHarness.numKeyedStateEntries()); assertEquals(4, testHarness.numEventTimeTimers()); // timers/gc timers for two windows OperatorSubtaskState snapshot = testHarness.snapshot(0, 0); testHarness.setup(); testHarness.initializeState(snapshot); testHarness.open(); assertEquals(0, testHarness.extractOutputStreamRecords().size()); assertEquals(5, testHarness.numKeyedStateEntries()); assertEquals(4, testHarness.numEventTimeTimers()); // timers/gc timers for two windows testHarness.processWatermark(new Watermark(20L)); assertEquals(0, testHarness.numKeyedStateEntries()); assertEquals(0, testHarness.numEventTimeTimers());
public static <T> OneInputStreamOperatorTestHarness<Event, T> getCepTestHarness( CepOperator<Event, Integer, T> cepOperator) throws Exception { KeySelector<Event, Integer> keySelector = new TestKeySelector(); return new KeyedOneInputStreamOperatorTestHarness<>( cepOperator, keySelector, BasicTypeInfo.INT_TYPE_INFO); }
@Test public void testLateWindowDropping() throws Exception { WindowAssigner<Integer, TimeWindow> mockAssigner = mockTimeWindowAssigner(); Trigger<Integer, TimeWindow> mockTrigger = mockTrigger(); InternalWindowFunction<Iterable<Integer>, Void, Integer, TimeWindow> mockWindowFunction = mockWindowFunction(); KeyedOneInputStreamOperatorTestHarness<Integer, Integer, Void> testHarness = createWindowOperator(mockAssigner, mockTrigger, 20L, mockWindowFunction); testHarness.open(); when(mockAssigner.assignWindows(anyInt(), anyLong(), anyAssignerContext())) .thenReturn(Arrays.asList(new TimeWindow(0, 2))); assertEquals(0, testHarness.getOutput().size()); assertEquals(0, testHarness.numKeyedStateEntries()); shouldFireOnElement(mockTrigger); // window.maxTime() == 1 plus 20L of allowed lateness testHarness.processWatermark(new Watermark(21)); testHarness.processElement(new StreamRecord<>(0, 0L)); // there should be nothing assertEquals(0, testHarness.numKeyedStateEntries()); assertEquals(0, testHarness.numEventTimeTimers()); assertEquals(0, testHarness.numProcessingTimeTimers()); // there should be two elements now assertEquals(0, testHarness.extractOutputStreamRecords().size()); }
new KeyedOneInputStreamOperatorTestHarness<>( op, (KeySelector<Integer, Integer>) value -> value, mockEnvironment); testHarness.setStateBackend(stateBackend); testHarness.open(); testHarness.processElement(new StreamRecord<>(i)); OperatorSnapshotFinalizer snapshotWithLocalState = testHarness.snapshotWithLocalState(1L, 1L); testHarness.close(); testHarness = new KeyedOneInputStreamOperatorTestHarness<Integer, Integer, Integer>( op, (KeySelector<Integer, Integer>) value -> value, testHarness.setStateBackend(stateBackend); testHarness.initializeState(jobManagerOwnedState, taskLocalState); testHarness.open(); testHarness.processElement(new StreamRecord<>(i)); testHarness.close();
@Test public void testWindowStateNotAvailableToMergingWindows() throws Exception { WindowAssigner<Integer, TimeWindow> mockAssigner = mockMergingAssigner(); Trigger<Integer, TimeWindow> mockTrigger = mockTrigger(); InternalWindowFunction<Iterable<Integer>, Void, Integer, TimeWindow> mockWindowFunction = mockWindowFunction(); KeyedOneInputStreamOperatorTestHarness<Integer, Integer, Void> testHarness = createWindowOperator(mockAssigner, mockTrigger, 20L, mockWindowFunction); testHarness.open(); when(mockTrigger.onElement(anyInt(), anyLong(), anyTimeWindow(), anyTriggerContext())) .thenReturn(TriggerResult.FIRE); when(mockAssigner.assignWindows(anyInt(), anyLong(), anyAssignerContext())) .thenReturn(Arrays.asList(new TimeWindow(0, 20))); doAnswer(new Answer<Object>() { @Override public Object answer(InvocationOnMock invocationOnMock) throws Throwable { InternalWindowFunction.InternalWindowContext context = (InternalWindowFunction.InternalWindowContext) invocationOnMock.getArguments()[2]; context.windowState().getState(valueStateDescriptor).update("hello"); return null; } }).when(mockWindowFunction).process(anyInt(), anyTimeWindow(), anyInternalWindowContext(), anyIntIterable(), WindowOperatorContractTest.<Void>anyCollector()); expectedException.expect(UnsupportedOperationException.class); expectedException.expectMessage("Per-window state is not allowed when using merging windows."); testHarness.processElement(new StreamRecord<>(0, 0L)); }
ByteBuffer, WindowedValue<KeyedWorkItem<Long, Long>>, WindowedValue<KV<Long, Long>>> testHarness = createTestHarness(windowDoFnOperator); testHarness.open(); testHarness.processWatermark(0L); testHarness.processElement( Item.builder().key(1L).timestamp(1L).value(100L).window(window).build().toStreamRecord()); testHarness.processElement( Item.builder() .key(1L) testHarness.processWatermark(200L); testHarness.close();
private void testEmittingFromWindowFunction(TimeDomainAdaptor timeAdaptor) throws Exception { WindowAssigner<Integer, TimeWindow> mockAssigner = mockTimeWindowAssigner(); Trigger<Integer, TimeWindow> mockTrigger = mockTrigger(); InternalWindowFunction<Iterable<Integer>, String, Integer, TimeWindow> mockWindowFunction = mockWindowFunction(); KeyedOneInputStreamOperatorTestHarness<Integer, Integer, String> testHarness = createWindowOperator(mockAssigner, mockTrigger, 0L, mockWindowFunction); testHarness.open(); when(mockAssigner.assignWindows(anyInt(), anyLong(), anyAssignerContext())) .thenReturn(Collections.singletonList(new TimeWindow(0, 2))); doAnswer(new Answer<Void>() { @Override public Void answer(InvocationOnMock invocation) throws Exception { @SuppressWarnings("unchecked") Collector<String> out = invocation.getArgument(4); out.collect("Hallo"); out.collect("Ciao"); return null; } }).when(mockWindowFunction).process(eq(0), eq(new TimeWindow(0, 2)), anyInternalWindowContext(), intIterable(0), WindowOperatorContractTest.<String>anyCollector()); timeAdaptor.shouldRegisterTimerOnElement(mockTrigger, 1); testHarness.processElement(new StreamRecord<>(0, 0L)); verify(mockWindowFunction, never()).process(anyInt(), anyTimeWindow(), anyInternalWindowContext(), anyIntIterable(), WindowOperatorContractTest.<String>anyCollector()); assertTrue(testHarness.extractOutputStreamRecords().isEmpty()); timeAdaptor.shouldFireOnTime(mockTrigger); timeAdaptor.advanceTime(testHarness, 1L); verify(mockWindowFunction, times(1)).process(eq(0), eq(new TimeWindow(0, 2)), anyInternalWindowContext(), intIterable(0), WindowOperatorContractTest.<String>anyCollector()); assertThat(testHarness.extractOutputStreamRecords(), contains(isStreamRecord("Hallo", 1L), isStreamRecord("Ciao", 1L))); }
private KeyedOneInputStreamOperatorTestHarness<Integer, Event, Map<String, List<Event>>> getTestHarness( int maxParallelism, int taskParallelism, int subtaskIdx) throws Exception { KeySelector<Event, Integer> keySelector = new TestKeySelector(); KeyedOneInputStreamOperatorTestHarness<Integer, Event, Map<String, List<Event>>> harness = new KeyedOneInputStreamOperatorTestHarness<>( getKeyedCepOpearator( false, new NFAFactory()), keySelector, BasicTypeInfo.INT_TYPE_INFO, maxParallelism, taskParallelism, subtaskIdx); harness.setStateBackend(new RocksDBStateBackend(new MemoryStateBackend())); return harness; }
String, WindowedValue<KV<String, Integer>>, WindowedValue<KV<String, Integer>>> testHarness = new KeyedOneInputStreamOperatorTestHarness<>( doFnOperator, kvWindowedValue -> kvWindowedValue.getValue().getKey(), new CoderTypeInformation<>(StringUtf8Coder.of())); testHarness.open(); testHarness.processWatermark(0); assertEquals(0, testHarness.numKeyedStateEntries()); testHarness.processElement( new StreamRecord<>( WindowedValue.of(KV.of("key1", 5), new Instant(1), window1, PaneInfo.NO_FIRING))); testHarness.processElement( new StreamRecord<>( WindowedValue.of(KV.of("key2", 7), new Instant(3), window1, PaneInfo.NO_FIRING))); stripStreamRecordFromWindowedValue(testHarness.getOutput()), contains( WindowedValue.of( assertEquals(4, testHarness.numKeyedStateEntries()); testHarness.getOutput().clear();
harness.open(); harness.processElement( new StreamRecord<>( WindowedValue.valueInGlobalWindow( new ValueWithRecordId<>(key1, key1.getBytes(StandardCharsets.UTF_8))))); harness.processElement( new StreamRecord<>( WindowedValue.valueInGlobalWindow( new ValueWithRecordId<>(key2, key2.getBytes(StandardCharsets.UTF_8))))); harness.processElement( new StreamRecord<>( WindowedValue.valueInGlobalWindow( stripStreamRecordFromWindowedValue(harness.getOutput()), contains(WindowedValue.valueInGlobalWindow(key1), WindowedValue.valueInGlobalWindow(key2))); OperatorSubtaskState snapshot = harness.snapshot(0L, 0L); harness.close(); harness.setup(); harness.initializeState(snapshot); harness.open(); harness.processElement( new StreamRecord<>(
/** * Ensure that we get some output from the given operator when pushing in an element and * setting watermark and processing time to {@code Long.MAX_VALUE}. */ private static <K, IN, OUT> void processElementAndEnsureOutput( OneInputStreamOperator<IN, OUT> operator, KeySelector<IN, K> keySelector, TypeInformation<K> keyType, IN element) throws Exception { KeyedOneInputStreamOperatorTestHarness<K, IN, OUT> testHarness = new KeyedOneInputStreamOperatorTestHarness<>( operator, keySelector, keyType); if (operator instanceof OutputTypeConfigurable) { // use a dummy type since window functions just need the ExecutionConfig // this is also only needed for Fold, which we're getting rid off soon. ((OutputTypeConfigurable) operator).setOutputType(BasicTypeInfo.STRING_TYPE_INFO, new ExecutionConfig()); } testHarness.open(); testHarness.setProcessingTime(0); testHarness.processWatermark(Long.MIN_VALUE); testHarness.processElement(new StreamRecord<>(element, 0)); // provoke any processing-time/event-time triggers testHarness.setProcessingTime(Long.MAX_VALUE); testHarness.processWatermark(Long.MAX_VALUE); // we at least get the two watermarks and should also see an output element assertTrue(testHarness.getOutput().size() >= 3); testHarness.close(); }
private static <OUT> OneInputStreamOperatorTestHarness<Tuple2<String, Integer>, OUT> createTestHarness(OneInputStreamOperator<Tuple2<String, Integer>, OUT> operator) throws Exception { return new KeyedOneInputStreamOperatorTestHarness<>(operator, new TupleKeySelector(), BasicTypeInfo.STRING_TYPE_INFO); }