@Test public void testTrackTwoStreams() throws Exception { Set<GlobalStreamId> streams = new HashSet<>(); streams.add(streamId("s1")); streams.add(streamId("s2")); waterMarkEventGenerator = new WaterMarkEventGenerator<>(windowManager, 100000, 5, streams); waterMarkEventGenerator.track(streamId("s1"), 100); waterMarkEventGenerator.track(streamId("s1"), 110); waterMarkEventGenerator.run(); assertTrue(eventList.isEmpty()); waterMarkEventGenerator.track(streamId("s2"), 95); waterMarkEventGenerator.track(streamId("s2"), 98); waterMarkEventGenerator.run(); assertTrue(eventList.get(0).isWatermark()); assertEquals(93, eventList.get(0).getTimestamp()); }
@Override public void cleanup() { if (waterMarkEventGenerator != null) { waterMarkEventGenerator.shutdown(); } windowManager.shutdown(); bolt.cleanup(); }
@Before public void setUp() { windowManager = new WindowManager<Integer>(null) { @Override public void add(Event<Integer> event) { eventList.add(event); } }; // set watermark interval to a high value and trigger manually to fix timing issues waterMarkEventGenerator = new WaterMarkEventGenerator<>(windowManager, 100000, 5, Collections.singleton(streamId("s1"))); waterMarkEventGenerator.start(); }
@Test public void testTrackSingleStreamOutOfOrder() throws Exception { waterMarkEventGenerator.track(streamId("s1"), 100); waterMarkEventGenerator.track(streamId("s1"), 110); waterMarkEventGenerator.track(streamId("s1"), 104); waterMarkEventGenerator.run(); assertTrue(eventList.get(0).isWatermark()); assertEquals(105, eventList.get(0).getTimestamp()); }
@Test public void testExecuteTuple() throws Exception { Mockito.when(mockWaterMarkEventGenerator.track(Mockito.any(), Mockito.anyLong())).thenReturn(true); Tuple mockTuple = Mockito.mock(Tuple.class); executor.initState(null); executor.waterMarkEventGenerator = mockWaterMarkEventGenerator; executor.execute(mockTuple); // should be ack-ed once Mockito.verify(mockOutputCollector, Mockito.times(1)).ack(mockTuple); }
/** * Start the trigger policy and waterMarkEventGenerator if set */ protected void start() { if (waterMarkEventGenerator != null) { LOG.debug("Starting waterMarkEventGenerator"); waterMarkEventGenerator.start(); } LOG.debug("Starting trigger policy"); triggerPolicy.start(); }
watermarkInterval = DEFAULT_WATERMARK_EVENT_INTERVAL_MS; waterMarkEventGenerator = new WaterMarkEventGenerator<>(manager, watermarkInterval, maxLagMs, getComponentStreams(context)); } else {
@Override public void run() { try { long waterMarkTs = computeWaterMarkTs(); if (waterMarkTs > lastWaterMarkTs) { this.windowManager.add(new WaterMarkEvent<>(waterMarkTs)); lastWaterMarkTs = waterMarkTs; } } catch (Throwable th) { LOG.error("Failed while processing watermark event ", th); throw th; } }
/** * Tracks the timestamp of the event in the stream, returns true if the event can be considered for processing or false if its a late * event. */ public boolean track(GlobalStreamId stream, long ts) { Long currentVal = streamToTs.get(stream); if (currentVal == null || ts > currentVal) { streamToTs.put(stream, ts); } checkFailures(); return ts >= lastWaterMarkTs; }
@Test public void testNoEvents() throws Exception { waterMarkEventGenerator.run(); assertTrue(eventList.isEmpty()); }
@Test public void testExecuteLatetuple() throws Exception { Mockito.when(mockWaterMarkEventGenerator.track(Mockito.any(), Mockito.anyLong())).thenReturn(false); Tuple mockTuple = Mockito.mock(Tuple.class); executor.initState(null); executor.waterMarkEventGenerator = mockWaterMarkEventGenerator; executor.execute(mockTuple); // ack-ed once Mockito.verify(mockOutputCollector, Mockito.times(1)).ack(mockTuple); // late tuple emitted ArgumentCaptor<String> stringCaptor = ArgumentCaptor.forClass(String.class); Mockito.verify(mockOutputCollector, Mockito.times(1)) .emit(stringCaptor.capture(), anchorCaptor.capture(), valuesCaptor.capture()); Assert.assertEquals(LATE_STREAM, stringCaptor.getValue()); Assert.assertEquals(Collections.singletonList(mockTuple), anchorCaptor.getValue()); Assert.assertEquals(new Values(mockTuple), valuesCaptor.getValue()); }
/** * Start the trigger policy and waterMarkEventGenerator if set */ protected void start() { if (waterMarkEventGenerator != null) { LOG.debug("Starting waterMarkEventGenerator"); waterMarkEventGenerator.start(); } LOG.debug("Starting trigger policy"); triggerPolicy.start(); }
watermarkInterval = DEFAULT_WATERMARK_EVENT_INTERVAL_MS; waterMarkEventGenerator = new WaterMarkEventGenerator<>(manager, watermarkInterval, maxLagMs, getComponentStreams(context)); } else {
@Override public void run() { try { long waterMarkTs = computeWaterMarkTs(); if (waterMarkTs > lastWaterMarkTs) { this.windowManager.add(new WaterMarkEvent<T>(waterMarkTs)); lastWaterMarkTs = waterMarkTs; } } catch (Throwable th) { LOG.error("Failed while processing watermark event ", th); throw th; } }
/** * Tracks the timestamp of the event in the stream, returns * true if the event can be considered for processing or * false if its a late event. */ public boolean track(GlobalStreamId stream, long ts) { Long currentVal = streamToTs.get(stream); if (currentVal == null || ts > currentVal) { streamToTs.put(stream, ts); } checkFailures(); return ts >= lastWaterMarkTs; }
@Test public void testExecuteWithTs() throws Exception { long[] timestamps = { 603, 605, 607, 618, 626, 636 }; for (long ts : timestamps) { executor.execute(getTuple("s1", new Fields("ts"), new Values(ts), "s1Src")); } //Thread.sleep(120); executor.waterMarkEventGenerator.run(); //System.out.println(testWindowedBolt.tupleWindows); assertEquals(3, testWindowedBolt.tupleWindows.size()); TupleWindow first = testWindowedBolt.tupleWindows.get(0); assertArrayEquals(new long[]{ 603, 605, 607 }, new long[]{ (long) first.get().get(0).getValue(0), (long) first.get().get(1).getValue(0), (long) first.get().get(2).getValue(0) }); TupleWindow second = testWindowedBolt.tupleWindows.get(1); assertArrayEquals(new long[]{ 603, 605, 607, 618 }, new long[]{ (long) second.get().get(0).getValue(0), (long) second.get().get(1).getValue(0), (long) second.get().get(2).getValue(0), (long) second.get().get(3).getValue(0) }); TupleWindow third = testWindowedBolt.tupleWindows.get(2); assertArrayEquals(new long[]{ 618, 626 }, new long[]{ (long) third.get().get(0).getValue(0), (long) third.get().get(1).getValue(0) }); }
@Override public void execute(Tuple input) { if (isTupleTs()) { long ts = timestampExtractor.extractTimestamp(input); if (waterMarkEventGenerator.track(input.getSourceGlobalStreamId(), ts)) { windowManager.add(input, ts); } else { if (lateTupleStream != null) { windowedOutputCollector.emit(lateTupleStream, input, new Values(input)); } else { LOG.info("Received a late tuple {} with ts {}. This will not be processed.", input, ts); } windowedOutputCollector.ack(input); } } else { windowManager.add(input); } }
@Test public void testLateEvent() throws Exception { assertTrue(waterMarkEventGenerator.track(streamId("s1"), 100)); assertTrue(waterMarkEventGenerator.track(streamId("s1"), 110)); waterMarkEventGenerator.run(); assertTrue(eventList.get(0).isWatermark()); assertEquals(105, eventList.get(0).getTimestamp()); eventList.clear(); assertTrue(waterMarkEventGenerator.track(streamId("s1"), 105)); assertTrue(waterMarkEventGenerator.track(streamId("s1"), 106)); assertTrue(waterMarkEventGenerator.track(streamId("s1"), 115)); assertFalse(waterMarkEventGenerator.track(streamId("s1"), 104)); waterMarkEventGenerator.run(); assertTrue(eventList.get(0).isWatermark()); assertEquals(110, eventList.get(0).getTimestamp()); } }
@After public void tearDown() { waterMarkEventGenerator.shutdown(); }