@Override public String toString() { return "TrackingSegmentWorker{" + "processor=" + getName() + ", segment=" + segment + '}'; } }
/** * Shut down the processor. */ @Override public void shutDown() { if (state.getAndSet(State.SHUT_DOWN).isRunning()) { logger.info("Shutdown state set for Processor '{}'. Awaiting termination...", getName()); try { while (threadFactory.activeThreads() > 0) { Thread.sleep(1); } } catch (InterruptedException e) { logger.info("Thread was interrupted while waiting for TrackingProcessor '{}' shutdown.", getName()); Thread.currentThread().interrupt(); } } }
private void releaseToken(Segment segment) { try { transactionManager.executeInTransaction(() -> tokenStore.releaseClaim(getName(), segment.getSegmentId())); } catch (Exception e) { // Ignore exception } }
private BlockingStream<TrackedEventMessage<?>> ensureEventStreamOpened( BlockingStream<TrackedEventMessage<?>> eventStreamIn, Segment segment) { BlockingStream<TrackedEventMessage<?>> eventStream = eventStreamIn; if (eventStream == null && state.get().isRunning()) { final TrackingToken trackingToken = transactionManager.fetchInTransaction( () -> tokenStore.fetchToken(getName(), segment.getSegmentId()) ); logger.info("Fetched token: {} for segment: {}", trackingToken, segment); eventStream = transactionManager.fetchInTransaction( () -> doOpenStream(trackingToken)); } return eventStream; }
@Test public void testTokenIsStoredWhenEventIsRead() throws Exception { CountDownLatch countDownLatch = new CountDownLatch(1); testSubject.registerHandlerInterceptor(((unitOfWork, interceptorChain) -> { unitOfWork.onCleanup(uow -> countDownLatch.countDown()); return interceptorChain.proceed(); })); testSubject.start(); // give it a bit of time to start Thread.sleep(200); eventBus.publish(createEvent()); assertTrue("Expected Unit of Work to have reached clean up phase", countDownLatch.await(5, TimeUnit.SECONDS)); verify(tokenStore).extendClaim(eq(testSubject.getName()), anyInt()); verify(tokenStore).storeToken(any(), any(), anyInt()); assertNotNull(tokenStore.fetchToken(testSubject.getName(), 0)); }
@Test public void testMultiThreadTokenIsStoredWhenEventIsRead() throws Exception { CountDownLatch countDownLatch = new CountDownLatch(2); testSubject.registerHandlerInterceptor(((unitOfWork, interceptorChain) -> { unitOfWork.onCleanup(uow -> countDownLatch.countDown()); return interceptorChain.proceed(); })); testSubject.start(); eventBus.publish(createEvents(2)); assertTrue("Expected Unit of Work to have reached clean up phase", countDownLatch.await(5, SECONDS)); verify(tokenStore, atLeastOnce()).storeToken(any(), any(), anyInt()); assertThat(tokenStore.fetchToken(testSubject.getName(), 0), notNullValue()); assertThat(tokenStore.fetchToken(testSubject.getName(), 1), notNullValue()); }
/** * Resets tokens to the given {@code startPosition}. This effectively causes a replay of events since that position. * <p> * Note that the new token must represent a position that is <em>before</em> the current position of the processor. * <p> * Before attempting to reset the tokens, the caller must stop this processor, as well as any instances of the * same logical processor that may be running in the cluster. Failure to do so will cause the reset to fail, * as a processor can only reset the tokens if it is able to claim them all. * * @param startPosition The token representing the position to reset the processor to. */ public void resetTokens(TrackingToken startPosition) { Assert.state(supportsReset(), () -> "The handlers assigned to this Processor do not support a reset"); Assert.state(!isRunning() && activeProcessorThreads() == 0, () -> "TrackingProcessor must be shut down before triggering a reset"); transactionManager.executeInTransaction(() -> { int[] segments = tokenStore.fetchSegments(getName()); TrackingToken[] tokens = new TrackingToken[segments.length]; for (int i = 0; i < segments.length; i++) { tokens[i] = tokenStore.fetchToken(getName(), segments[i]); } // we now have all tokens, hurray eventHandlerInvoker().performReset(); for (int i = 0; i < tokens.length; i++) { tokenStore.storeToken(ReplayToken.createReplayToken(tokens[i], startPosition), getName(), segments[i]); } }); }
TrackingToken finalLastToken = lastToken; transactionManager.executeInTransaction( () -> tokenStore.storeToken(finalLastToken, getName(), segment.getSegmentId()) ); activeSegments.computeIfPresent(segment.getSegmentId(), (k, v) -> v.advancedTo(finalLastToken)); () -> tokenStore.extendClaim(getName(), segment.getSegmentId()) ); return; logger.error(String.format("Event processor [%s] was interrupted. Shutting down.", getName()), e); this.shutDown(); Thread.currentThread().interrupt();
.setProcessorName(processor.getName()) .setMode("Tracking") .setActiveThreads(processor.activeProcessorThreads())
@Test public void testFirstTokenIsStoredWhenUnitOfWorkIsRolledBackOnSecondEvent() throws Exception { List<? extends EventMessage<?>> events = createEvents(2); CountDownLatch countDownLatch = new CountDownLatch(2); testSubject.registerHandlerInterceptor(((unitOfWork, interceptorChain) -> { unitOfWork.onCommit(uow -> { if (uow.getMessage().equals(events.get(1))) { throw new MockException(); } }); return interceptorChain.proceed(); })); testSubject.registerHandlerInterceptor(((unitOfWork, interceptorChain) -> { unitOfWork.onCleanup(uow -> countDownLatch.countDown()); return interceptorChain.proceed(); })); testSubject.start(); // give it a bit of time to start Thread.sleep(200); eventBus.publish(events); assertTrue("Expected Unit of Work to have reached clean up phase", countDownLatch.await(5, TimeUnit.SECONDS)); verify(tokenStore, atLeastOnce()).storeToken(any(), any(), anyInt()); assertNotNull(tokenStore.fetchToken(testSubject.getName(), 0)); }
@Test public void testProcessorInitializesMoreTokensThanWorkerCount() throws InterruptedException { configureProcessor(TrackingEventProcessorConfiguration.forParallelProcessing(2) .andInitialSegmentsCount(4)); testSubject.start(); // give it some time to split segments from the store and submit to executor service. Thread.sleep(200); assertThat(testSubject.activeProcessorThreads(), is(2)); int[] actual = tokenStore.fetchSegments(testSubject.getName()); Arrays.sort(actual); assertArrayEquals(new int[]{0, 1, 2, 3}, actual); }
@Test public void testTokenIsStoredOncePerEventBatch() throws Exception { testSubject = TrackingEventProcessor.builder() .name("test") .eventHandlerInvoker(eventHandlerInvoker) .messageSource(eventBus) .tokenStore(tokenStore) .transactionManager(NoTransactionManager.INSTANCE) .build(); CountDownLatch countDownLatch = new CountDownLatch(2); testSubject.registerHandlerInterceptor(((unitOfWork, interceptorChain) -> { unitOfWork.onCleanup(uow -> countDownLatch.countDown()); return interceptorChain.proceed(); })); testSubject.start(); // give it a bit of time to start Thread.sleep(200); eventBus.publish(createEvents(2)); assertTrue("Expected Unit of Work to have reached clean up phase for 2 messages", countDownLatch.await(5, TimeUnit.SECONDS)); InOrder inOrder = inOrder(tokenStore); inOrder.verify(tokenStore, times(1)).extendClaim(eq(testSubject.getName()), anyInt()); inOrder.verify(tokenStore, times(1)).storeToken(any(), any(), anyInt()); assertNotNull(tokenStore.fetchToken(testSubject.getName(), 0)); }
@Test public void testMultiThreadContinueFromPreviousToken() throws Exception { tokenStore = spy(new InMemoryTokenStore()); eventBus.publish(createEvents(10)); TrackedEventMessage<?> firstEvent = eventBus.openStream(null).nextAvailable(); tokenStore.storeToken(firstEvent.trackingToken(), testSubject.getName(), 0); assertEquals(firstEvent.trackingToken(), tokenStore.fetchToken(testSubject.getName(), 0)); final AcknowledgeByThread acknowledgeByThread = new AcknowledgeByThread(); CountDownLatch countDownLatch = new CountDownLatch(9); doAnswer(invocation -> { acknowledgeByThread.addMessage(Thread.currentThread(), (EventMessage<?>) invocation.getArguments()[0]); countDownLatch.countDown(); return null; }).when(mockHandler).handle(any()); configureProcessor(TrackingEventProcessorConfiguration.forParallelProcessing(2)); testSubject.start(); assertTrue("Expected 9 invocations on Event Handler by now, missing " + countDownLatch.getCount(), countDownLatch.await(60, SECONDS)); acknowledgeByThread.assertEventsAckedByMultipleThreads(); acknowledgeByThread.assertEventsAddUpTo(9); }
@Test @DirtiesContext public void testContinueFromPreviousToken() throws Exception { tokenStore = new InMemoryTokenStore(); eventBus.publish(createEvents(10)); TrackedEventMessage<?> firstEvent = eventBus.openStream(null).nextAvailable(); tokenStore.storeToken(firstEvent.trackingToken(), testSubject.getName(), 0); assertEquals(firstEvent.trackingToken(), tokenStore.fetchToken(testSubject.getName(), 0)); List<EventMessage<?>> ackedEvents = new ArrayList<>(); CountDownLatch countDownLatch = new CountDownLatch(9); doAnswer(invocation -> { ackedEvents.add((EventMessage<?>) invocation.getArguments()[0]); countDownLatch.countDown(); return null; }).when(mockHandler).handle(any()); testSubject = TrackingEventProcessor.builder() .name("test") .eventHandlerInvoker(eventHandlerInvoker) .messageSource(eventBus) .tokenStore(tokenStore) .transactionManager(NoTransactionManager.INSTANCE) .build(); testSubject.start(); // give it a bit of time to start Thread.sleep(200); assertTrue("Expected 9 invocations on Event Handler by now", countDownLatch.await(5, TimeUnit.SECONDS)); assertEquals(9, ackedEvents.size()); }
@Test public void testMultiThreadTokensAreStoredWhenUnitOfWorkIsRolledBackOnSecondEvent() throws Exception { List<? extends EventMessage<?>> events = createEvents(2); CountDownLatch countDownLatch = new CountDownLatch(2); //noinspection Duplicates testSubject.registerHandlerInterceptor(((unitOfWork, interceptorChain) -> { unitOfWork.onCommit(uow -> { if (uow.getMessage().equals(events.get(1))) { throw new MockException(); } }); return interceptorChain.proceed(); })); testSubject.registerHandlerInterceptor(((unitOfWork, interceptorChain) -> { unitOfWork.onCleanup(uow -> countDownLatch.countDown()); return interceptorChain.proceed(); })); testSubject.start(); eventBus.publish(events); assertTrue("Expected Unit of Work to have reached clean up phase", countDownLatch.await(5, SECONDS)); assertNotNull(tokenStore.fetchToken(testSubject.getName(), 0)); assertNotNull(tokenStore.fetchToken(testSubject.getName(), 1)); }
@Test public void testTokenIsNotStoredWhenUnitOfWorkIsRolledBack() throws Exception { CountDownLatch countDownLatch = new CountDownLatch(1); testSubject.registerHandlerInterceptor(((unitOfWork, interceptorChain) -> { unitOfWork.onCommit(uow -> { throw new MockException(); }); return interceptorChain.proceed(); })); testSubject.registerHandlerInterceptor(((unitOfWork, interceptorChain) -> { unitOfWork.onCleanup(uow -> countDownLatch.countDown()); return interceptorChain.proceed(); })); testSubject.start(); // give it a bit of time to start Thread.sleep(200); eventBus.publish(createEvent()); assertTrue("Expected Unit of Work to have reached clean up phase", countDownLatch.await(5, TimeUnit.SECONDS)); assertNull(tokenStore.fetchToken(testSubject.getName(), 0)); }
@Test public void testProcessorInitializesAndUsesSameTokens() { configureProcessor(TrackingEventProcessorConfiguration.forParallelProcessing(6) .andInitialSegmentsCount(6)); testSubject.start(); assertWithin(5, SECONDS, () -> assertThat(testSubject.activeProcessorThreads(), is(6))); int[] actual = tokenStore.fetchSegments(testSubject.getName()); Arrays.sort(actual); assertArrayEquals(new int[]{0, 1, 2, 3, 4, 5}, actual); }
assertNull(tokenStore.fetchToken(testSubject.getName(), 0));
tokenStore.extendClaim(getName(), unitOfWork.getResource(segmentIdResourceKey));