@Test public void testInvalidParameters() { try { EventTimeSessionWindows.withGap(Time.seconds(-1)); fail("should fail"); } catch (IllegalArgumentException e) { assertThat(e.toString(), containsString("0 < size")); } try { EventTimeSessionWindows.withGap(Time.seconds(0)); fail("should fail"); } catch (IllegalArgumentException e) { assertThat(e.toString(), containsString("0 < size")); } }
@Test public void testMergeSingleWindow() { MergingWindowAssigner.MergeCallback callback = mock(MergingWindowAssigner.MergeCallback.class); EventTimeSessionWindows assigner = EventTimeSessionWindows.withGap(Time.milliseconds(5000)); assigner.mergeWindows(Lists.newArrayList(new TimeWindow(0, 1)), callback); verify(callback, never()).merge(anyCollection(), Matchers.anyObject()); }
@Test public void testMergeSinglePointWindow() { MergingWindowAssigner.MergeCallback callback = mock(MergingWindowAssigner.MergeCallback.class); EventTimeSessionWindows assigner = EventTimeSessionWindows.withGap(Time.milliseconds(5000)); assigner.mergeWindows(Lists.newArrayList(new TimeWindow(0, 0)), callback); verify(callback, never()).merge(anyCollection(), Matchers.anyObject()); }
@Test public void testTimeUnits() { // sanity check with one other time unit final int sessionGap = 5000; WindowAssigner.WindowAssignerContext mockContext = mock(WindowAssigner.WindowAssignerContext.class); EventTimeSessionWindows assigner = EventTimeSessionWindows.withGap(Time.seconds(sessionGap / 1000)); assertThat(assigner.assignWindows("String", 0L, mockContext), contains(timeWindow(0, 0 + sessionGap))); assertThat(assigner.assignWindows("String", 4999L, mockContext), contains(timeWindow(4999, 4999 + sessionGap))); assertThat(assigner.assignWindows("String", 5000L, mockContext), contains(timeWindow(5000, 5000 + sessionGap))); }
@Test public void testWindowAssignment() { final int sessionGap = 5000; WindowAssigner.WindowAssignerContext mockContext = mock(WindowAssigner.WindowAssignerContext.class); EventTimeSessionWindows assigner = EventTimeSessionWindows.withGap(Time.milliseconds(sessionGap)); assertThat(assigner.assignWindows("String", 0L, mockContext), contains(timeWindow(0, 0 + sessionGap))); assertThat(assigner.assignWindows("String", 4999L, mockContext), contains(timeWindow(4999, 4999 + sessionGap))); assertThat(assigner.assignWindows("String", 5000L, mockContext), contains(timeWindow(5000, 5000 + sessionGap))); }
@Test public void testMergeCoveringWindow() { MergingWindowAssigner.MergeCallback callback = mock(MergingWindowAssigner.MergeCallback.class); EventTimeSessionWindows assigner = EventTimeSessionWindows.withGap(Time.milliseconds(5000)); assigner.mergeWindows( Lists.newArrayList( new TimeWindow(1, 1), new TimeWindow(0, 2), new TimeWindow(4, 7), new TimeWindow(5, 6)), callback); verify(callback, times(1)).merge( (Collection<TimeWindow>) argThat(containsInAnyOrder(new TimeWindow(1, 1), new TimeWindow(0, 2))), eq(new TimeWindow(0, 2))); verify(callback, times(1)).merge( (Collection<TimeWindow>) argThat(containsInAnyOrder(new TimeWindow(5, 6), new TimeWindow(4, 7))), eq(new TimeWindow(4, 7))); verify(callback, times(2)).merge(anyCollection(), Matchers.anyObject()); }
@Test public void testProperties() { EventTimeSessionWindows assigner = EventTimeSessionWindows.withGap(Time.seconds(5)); assertTrue(assigner.isEventTime()); assertEquals(new TimeWindow.Serializer(), assigner.getWindowSerializer(new ExecutionConfig())); assertThat(assigner.getDefaultTrigger(mock(StreamExecutionEnvironment.class)), instanceOf(EventTimeTrigger.class)); }
@Test public void testMergeConsecutiveWindows() { MergingWindowAssigner.MergeCallback callback = mock(MergingWindowAssigner.MergeCallback.class); EventTimeSessionWindows assigner = EventTimeSessionWindows.withGap(Time.milliseconds(5000)); assigner.mergeWindows( Lists.newArrayList( new TimeWindow(0, 1), new TimeWindow(1, 2), new TimeWindow(2, 3), new TimeWindow(4, 5), new TimeWindow(5, 6)), callback); verify(callback, times(1)).merge( (Collection<TimeWindow>) argThat(containsInAnyOrder(new TimeWindow(0, 1), new TimeWindow(1, 2), new TimeWindow(2, 3))), eq(new TimeWindow(0, 3))); verify(callback, times(1)).merge( (Collection<TimeWindow>) argThat(containsInAnyOrder(new TimeWindow(4, 5), new TimeWindow(5, 6))), eq(new TimeWindow(4, 6))); verify(callback, times(2)).merge(anyCollection(), Matchers.anyObject()); }
@Test public void testPersistOnlyIfHaveUpdates() throws Exception { @SuppressWarnings("unchecked") ListState<Tuple2<TimeWindow, TimeWindow>> mockState = mock(ListState.class); when(mockState.get()).thenReturn(Lists.newArrayList( new Tuple2<>(new TimeWindow(17, 42), new TimeWindow(42, 17)), new Tuple2<>(new TimeWindow(1, 2), new TimeWindow(3, 4)) )); MergingWindowSet<TimeWindow> windowSet = new MergingWindowSet<>(EventTimeSessionWindows.withGap(Time.milliseconds(3)), mockState); assertEquals(new TimeWindow(42, 17), windowSet.getStateWindow(new TimeWindow(17, 42))); assertEquals(new TimeWindow(3, 4), windowSet.getStateWindow(new TimeWindow(1, 2))); windowSet.persist(); verify(mockState, times(0)).add(Matchers.<Tuple2<TimeWindow, TimeWindow>>anyObject()); }
@Test public void testPersist() throws Exception { @SuppressWarnings("unchecked") ListState<Tuple2<TimeWindow, TimeWindow>> mockState = mock(ListState.class); MergingWindowSet<TimeWindow> windowSet = new MergingWindowSet<>(EventTimeSessionWindows.withGap(Time.milliseconds(3)), mockState); TestingMergeFunction mergeFunction = new TestingMergeFunction(); windowSet.addWindow(new TimeWindow(1, 2), mergeFunction); windowSet.addWindow(new TimeWindow(17, 42), mergeFunction); assertEquals(new TimeWindow(1, 2), windowSet.getStateWindow(new TimeWindow(1, 2))); assertEquals(new TimeWindow(17, 42), windowSet.getStateWindow(new TimeWindow(17, 42))); windowSet.persist(); verify(mockState).add(eq(new Tuple2<>(new TimeWindow(1, 2), new TimeWindow(1, 2)))); verify(mockState).add(eq(new Tuple2<>(new TimeWindow(17, 42), new TimeWindow(17, 42)))); verify(mockState, times(2)).add(Matchers.<Tuple2<TimeWindow, TimeWindow>>anyObject()); }
@Test public void testSessionWithFoldFails() throws Exception { // verify that fold does not work with merging windows StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment(); AllWindowedStream<String, TimeWindow> windowedStream = env.fromElements("Hello", "Ciao") .windowAll(EventTimeSessionWindows.withGap(Time.seconds(5))); try { windowedStream.fold("", new FoldFunction<String, String>() { private static final long serialVersionUID = -4567902917104921706L; @Override public String fold(String accumulator, String value) throws Exception { return accumulator; } }); } catch (UnsupportedOperationException e) { // expected // use a catch to ensure that the exception is thrown by the fold return; } fail("The fold call should fail."); }
@Test public void testRestoreFromState() throws Exception { @SuppressWarnings("unchecked") ListState<Tuple2<TimeWindow, TimeWindow>> mockState = mock(ListState.class); when(mockState.get()).thenReturn(Lists.newArrayList( new Tuple2<>(new TimeWindow(17, 42), new TimeWindow(42, 17)), new Tuple2<>(new TimeWindow(1, 2), new TimeWindow(3, 4)) )); MergingWindowSet<TimeWindow> windowSet = new MergingWindowSet<>(EventTimeSessionWindows.withGap(Time.milliseconds(3)), mockState); assertEquals(new TimeWindow(42, 17), windowSet.getStateWindow(new TimeWindow(17, 42))); assertEquals(new TimeWindow(3, 4), windowSet.getStateWindow(new TimeWindow(1, 2))); }
@Test public void testSessionWithFoldFails() throws Exception { // verify that fold does not work with merging windows StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment(); WindowedStream<String, String, TimeWindow> windowedStream = env.fromElements("Hello", "Ciao") .keyBy(new KeySelector<String, String>() { @Override public String getKey(String value) throws Exception { return value; } }) .window(EventTimeSessionWindows.withGap(Time.seconds(5))); try { windowedStream.fold("", new FoldFunction<String, String>() { private static final long serialVersionUID = -4567902917104921706L; @Override public String fold(String accumulator, String value) throws Exception { return accumulator; } }); } catch (UnsupportedOperationException e) { // expected // use a catch to ensure that the exception is thrown by the fold return; } fail("The fold call should fail."); }
/** * Test adding a new window that is identical to an existing window. This should not cause * a merge. */ @Test public void testAddingIdenticalWindows() throws Exception { @SuppressWarnings("unchecked") ListState<Tuple2<TimeWindow, TimeWindow>> mockState = mock(ListState.class); MergingWindowSet<TimeWindow> windowSet = new MergingWindowSet<>(EventTimeSessionWindows.withGap(Time.milliseconds(3)), mockState); TestingMergeFunction mergeFunction = new TestingMergeFunction(); mergeFunction.reset(); assertEquals(new TimeWindow(1, 2), windowSet.addWindow(new TimeWindow(1, 2), mergeFunction)); assertFalse(mergeFunction.hasMerged()); assertEquals(new TimeWindow(1, 2), windowSet.getStateWindow(new TimeWindow(1, 2))); mergeFunction.reset(); assertEquals(new TimeWindow(1, 2), windowSet.addWindow(new TimeWindow(1, 2), mergeFunction)); assertFalse(mergeFunction.hasMerged()); assertEquals(new TimeWindow(1, 2), windowSet.getStateWindow(new TimeWindow(1, 2))); }
private void runTest( SourceFunction<SessionEvent<Integer, TestEventPayload>> dataSource, WindowFunction<SessionEvent<Integer, TestEventPayload>, String, Tuple, TimeWindow> windowFunction) throws Exception { StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment(); env.setStreamTimeCharacteristic(TimeCharacteristic.EventTime); WindowedStream<SessionEvent<Integer, TestEventPayload>, Tuple, TimeWindow> windowedStream = env.addSource(dataSource).keyBy("sessionKey") .window(EventTimeSessionWindows.withGap(Time.milliseconds(MAX_SESSION_EVENT_GAP_MS))); if (ALLOWED_LATENESS_MS != Long.MAX_VALUE) { windowedStream = windowedStream.allowedLateness(Time.milliseconds(ALLOWED_LATENESS_MS)); } if (PURGE_WINDOW_ON_FIRE) { windowedStream = windowedStream.trigger(PurgingTrigger.of(EventTimeTrigger.create())); } windowedStream.apply(windowFunction).print(); JobExecutionResult result = env.execute(); // check that overall event counts match with our expectations. remember that late events within lateness will // each trigger a window! Assert.assertEquals( (LATE_EVENTS_PER_SESSION + 1) * NUMBER_OF_SESSIONS * EVENTS_PER_SESSION, (long) result.getAccumulatorResult(SESSION_COUNTER_ON_TIME_KEY)); Assert.assertEquals( NUMBER_OF_SESSIONS * (LATE_EVENTS_PER_SESSION * (LATE_EVENTS_PER_SESSION + 1) / 2), (long) result.getAccumulatorResult(SESSION_COUNTER_LATE_KEY)); }
/** * Test merging of a large new window that covers one existing windows. */ @Test public void testMergeLargeWindowCoveringSingleWindow() throws Exception { @SuppressWarnings("unchecked") ListState<Tuple2<TimeWindow, TimeWindow>> mockState = mock(ListState.class); MergingWindowSet<TimeWindow> windowSet = new MergingWindowSet<>(EventTimeSessionWindows.withGap(Time.milliseconds(3)), mockState); TestingMergeFunction mergeFunction = new TestingMergeFunction(); // add an initial small window mergeFunction.reset(); assertEquals(new TimeWindow(1, 2), windowSet.addWindow(new TimeWindow(1, 2), mergeFunction)); assertFalse(mergeFunction.hasMerged()); assertEquals(new TimeWindow(1, 2), windowSet.getStateWindow(new TimeWindow(1, 2))); // add a new window that completely covers the existing window mergeFunction.reset(); assertEquals(new TimeWindow(0, 3), windowSet.addWindow(new TimeWindow(0, 3), mergeFunction)); assertTrue(mergeFunction.hasMerged()); assertEquals(new TimeWindow(1, 2), windowSet.getStateWindow(new TimeWindow(0, 3))); }
@Test @SuppressWarnings("rawtypes") public void testMergingWindowsWithEvictor() throws Exception { StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment(); env.setStreamTimeCharacteristic(TimeCharacteristic.IngestionTime); DataStream<Tuple2<String, Integer>> source = env.fromElements(Tuple2.of("hello", 1), Tuple2.of("hello", 2)); DataStream<Tuple3<String, String, Integer>> window1 = source .windowAll(EventTimeSessionWindows.withGap(Time.seconds(5))) .evictor(CountEvictor.of(5)) .process(new TestProcessAllWindowFunction()); OneInputTransformation<Tuple2<String, Integer>, Tuple3<String, String, Integer>> transform = (OneInputTransformation<Tuple2<String, Integer>, Tuple3<String, String, Integer>>) window1.getTransformation(); OneInputStreamOperator<Tuple2<String, Integer>, Tuple3<String, String, Integer>> operator = transform.getOperator(); Assert.assertTrue(operator instanceof WindowOperator); WindowOperator<String, Tuple2<String, Integer>, ?, ?, ?> winOperator = (WindowOperator<String, Tuple2<String, Integer>, ?, ?, ?>) operator; Assert.assertTrue(winOperator.getTrigger() instanceof EventTimeTrigger); Assert.assertTrue(winOperator.getWindowAssigner() instanceof EventTimeSessionWindows); Assert.assertTrue(winOperator.getStateDescriptor() instanceof ListStateDescriptor); processElementAndEnsureOutput(winOperator, winOperator.getKeySelector(), BasicTypeInfo.STRING_TYPE_INFO, new Tuple2<>("hello", 1)); }
@Test @SuppressWarnings("rawtypes") public void testMergingWindowsWithEvictor() throws Exception { StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment(); env.setStreamTimeCharacteristic(TimeCharacteristic.IngestionTime); DataStream<Integer> source = env.fromElements(1, 2); DataStream<String> window1 = source .keyBy(new KeySelector<Integer, String>() { @Override public String getKey(Integer value) throws Exception { return value.toString(); } }) .window(EventTimeSessionWindows.withGap(Time.seconds(5))) .evictor(CountEvictor.of(5)) .process(new TestProcessWindowFunction()); final OneInputTransformation<Integer, String> transform = (OneInputTransformation<Integer, String>) window1.getTransformation(); final OneInputStreamOperator<Integer, String> operator = transform.getOperator(); Assert.assertTrue(operator instanceof WindowOperator); WindowOperator<String, Integer, ?, ?, ?> winOperator = (WindowOperator<String, Integer, ?, ?, ?>) operator; Assert.assertTrue(winOperator.getTrigger() instanceof EventTimeTrigger); Assert.assertTrue(winOperator.getWindowAssigner() instanceof EventTimeSessionWindows); Assert.assertTrue(winOperator.getStateDescriptor() instanceof ListStateDescriptor); processElementAndEnsureOutput(winOperator, winOperator.getKeySelector(), BasicTypeInfo.STRING_TYPE_INFO, 1); }
EventTimeSessionWindows.withGap(Time.seconds(sessionSize)), new TimeWindow.Serializer(), new TupleKeySelector<String>(),
/** * Manually run this to write binary snapshot data. */ @Ignore @Test public void writeSessionWindowsWithCountTriggerInMintConditionSnapshot() throws Exception { final int sessionSize = 3; ListStateDescriptor<Tuple2<String, Integer>> stateDesc = new ListStateDescriptor<>("window-contents", STRING_INT_TUPLE.createSerializer(new ExecutionConfig())); WindowOperator<String, Tuple2<String, Integer>, Iterable<Tuple2<String, Integer>>, Tuple3<String, Long, Long>, TimeWindow> operator = new WindowOperator<>( EventTimeSessionWindows.withGap(Time.seconds(sessionSize)), new TimeWindow.Serializer(), new TupleKeySelector<String>(), BasicTypeInfo.STRING_TYPE_INFO.createSerializer(new ExecutionConfig()), stateDesc, new InternalIterableWindowFunction<>(new SessionWindowFunction()), PurgingTrigger.of(CountTrigger.of(4)), 0, null /* late data output tag */); OneInputStreamOperatorTestHarness<Tuple2<String, Integer>, Tuple3<String, Long, Long>> testHarness = new KeyedOneInputStreamOperatorTestHarness<>(operator, new TupleKeySelector<>(), BasicTypeInfo.STRING_TYPE_INFO); testHarness.setup(); testHarness.open(); // do snapshot and save to file OperatorSubtaskState snapshot = testHarness.snapshot(0, 0); OperatorSnapshotUtil.writeStateHandle( snapshot, "src/test/resources/win-op-migration-test-session-with-stateful-trigger-mint-flink" + flinkGenerateSavepointVersion + "-snapshot"); testHarness.close(); }