EventHandlerGroup<T> createWorkerPool( final Sequence[] barrierSequences, final WorkHandler<? super T>[] workHandlers) { final SequenceBarrier sequenceBarrier = ringBuffer.newBarrier(barrierSequences); final WorkerPool<T> workerPool = new WorkerPool<>(ringBuffer, sequenceBarrier, exceptionHandler, workHandlers); consumerRepository.add(workerPool, sequenceBarrier); final Sequence[] workerSequences = workerPool.getWorkerSequences(); updateGatingSequencesForNextInChain(barrierSequences, workerSequences); return new EventHandlerGroup<>(this, consumerRepository, workerSequences); }
@SuppressWarnings("unchecked") @Test public void shouldProcessOnlyOnceItHasBeenPublished() throws Exception { Executor executor = Executors.newCachedThreadPool(DaemonThreadFactory.INSTANCE); WorkerPool<AtomicLong> pool = new WorkerPool<AtomicLong>( new AtomicLongEventFactory(), new FatalExceptionHandler(), new AtomicLongWorkHandler(), new AtomicLongWorkHandler()); RingBuffer<AtomicLong> ringBuffer = pool.start(executor); ringBuffer.next(); ringBuffer.next(); Thread.sleep(1000); assertThat(ringBuffer.get(0).get(), is(0L)); assertThat(ringBuffer.get(1).get(), is(0L)); }
@SuppressWarnings("unchecked") @Test public void shouldProcessEachMessageByOnlyOneWorker() throws Exception { Executor executor = Executors.newCachedThreadPool(DaemonThreadFactory.INSTANCE); WorkerPool<AtomicLong> pool = new WorkerPool<AtomicLong>( new AtomicLongEventFactory(), new FatalExceptionHandler(), new AtomicLongWorkHandler(), new AtomicLongWorkHandler()); RingBuffer<AtomicLong> ringBuffer = pool.start(executor); ringBuffer.next(); ringBuffer.next(); ringBuffer.publish(0); ringBuffer.publish(1); Thread.sleep(500); assertThat(ringBuffer.get(0).get(), is(1L)); assertThat(ringBuffer.get(1).get(), is(1L)); }
workHandlers[i] = new DmlParserStage(); workerPool = new WorkerPool<MessageEvent>(disruptorMsgBuffer, dmlParserSequenceBarrier, exceptionHandler,
EventHandlerGroup<T> createWorkerPool( final Sequence[] barrierSequences, final WorkHandler<? super T>[] workHandlers) { final SequenceBarrier sequenceBarrier = ringBuffer.newBarrier(barrierSequences); final WorkerPool<T> workerPool = new WorkerPool<>(ringBuffer, sequenceBarrier, exceptionHandler, workHandlers); consumerRepository.add(workerPool, sequenceBarrier); final Sequence[] workerSequences = workerPool.getWorkerSequences(); updateGatingSequencesForNextInChain(barrierSequences, workerSequences); return new EventHandlerGroup<>(this, consumerRepository, workerSequences); }
@Override public void startup() { EventBus eventBus = disruptorDispatchThread.getEventBus(); executorService = new NonOrderedQueuePoolExecutor(poolName, excutorSize); cycleEventHandler = new CycleEventHandler[excutorSize]; for(int i = 0; i < excutorSize; i++){ cycleEventHandler[i] = new CycleEventHandler(eventBus); } RingBuffer ringBuffer = disruptorDispatchThread.getRingBuffer(); workerPool = new WorkerPool(ringBuffer, ringBuffer.newBarrier(), new FatalExceptionHandler(), cycleEventHandler); ringBuffer.addGatingSequences(workerPool.getWorkerSequences()); workerPool.start(executorService); // BatchEventProcessor<CycleEvent>[] batchEventProcessors = new BatchEventProcessor[excutorSize]; // for(int i = 0; i < excutorSize; i++){ // batchEventProcessors[i] = new BatchEventProcessor<>(ringBuffer, sequenceBarrier, cycleEventHandler[i]); // ringBuffer.addGatingSequences(batchEventProcessors[i].getSequence()); //// executorService.submit(batchEventProcessors[i]); // } }
var Workers = new WorkerPool(); var worker1 = Workers.addWorker(); worker1.enqueue(function(){
@Bean public WorkerPool<ServerEvent> workerPool(@Qualifier("request-dispatcher") RequestHandler requestHandler, @Qualifier("server-producer") Producer<String, Response> responseProducer) { final ServerEventHandler[] workHandlers = new ServerEventHandler[workerThreads]; IntStream.range(0, workerThreads).forEach( nbr -> workHandlers[nbr] = new ServerEventHandler(requestHandler, responseProducer) ); final RingBuffer<ServerEvent> ringBuffer = RingBuffer.createMultiProducer(ServerEvent::new, bufferSize, getWaitStrategy()); final SequenceBarrier barrier = ringBuffer.newBarrier(); WorkerPool<ServerEvent> workerPool = new WorkerPool<>(ringBuffer, barrier, new FatalExceptionHandler(), workHandlers); ringBuffer.addGatingSequences(workerPool.getWorkerSequences()); return workerPool; }
private void setUpWorkerPool() { // Executor that will be used to construct new threads for consumers final ThreadFactory namedThreadFactory = new ThreadFactoryBuilder() .setNameFormat("NewtsWriter-Consumer-%d").build(); final Executor executor = Executors.newCachedThreadPool(namedThreadFactory); @SuppressWarnings("unchecked") final WorkHandler<SampleBatchEvent> handlers[] = new WorkHandler[m_numWriterThreads]; for (int i = 0; i < m_numWriterThreads; i++) { handlers[i] = this; } m_ringBuffer = RingBuffer.createMultiProducer(SampleBatchEvent::new, m_ringBufferSize); m_workerPool = new WorkerPool<SampleBatchEvent>( m_ringBuffer, m_ringBuffer.newBarrier(), new FatalExceptionHandler(), handlers); m_ringBuffer.addGatingSequences(m_workerPool.getWorkerSequences()); m_workerPool.start(executor); }
EventHandlerGroup<T> createWorkerPool(final Sequence[] barrierSequences, final WorkHandler<? super T>[] workHandlers) { final SequenceBarrier sequenceBarrier = ringBuffer.newBarrier(barrierSequences); final WorkerPool<T> workerPool = new WorkerPool<T>(ringBuffer, sequenceBarrier, exceptionHandler, workHandlers); consumerRepository.add(workerPool, sequenceBarrier); return new EventHandlerGroup<T>(this, consumerRepository, workerPool.getWorkerSequences()); }
WorkerPool pool = new WorkerPool(); pool.submit("2", task1); pool.submit("2", task2); pool.submit("3", task3); pool.submit("2", task4);
EventHandlerGroup<T> createWorkerPool( final Sequence[] barrierSequences, final WorkHandler<? super T>[] workHandlers) { final SequenceBarrier sequenceBarrier = ringBuffer.newBarrier(barrierSequences); final WorkerPool<T> workerPool = new WorkerPool<T>(ringBuffer, sequenceBarrier, exceptionHandler, workHandlers); consumerRepository.add(workerPool, sequenceBarrier); return new EventHandlerGroup<T>(this, consumerRepository, workerPool.getWorkerSequences()); }
private void setUpWorkerPool() { // Executor that will be used to construct new threads for consumers final ThreadFactory namedThreadFactory = new ThreadFactoryBuilder() .setNameFormat("NewtsWriter-Consumer-%d").build(); final Executor executor = Executors.newCachedThreadPool(namedThreadFactory); @SuppressWarnings("unchecked") final WorkHandler<SampleBatchEvent> handlers[] = new WorkHandler[m_numWriterThreads]; for (int i = 0; i < m_numWriterThreads; i++) { handlers[i] = this; } m_ringBuffer = RingBuffer.createMultiProducer(SampleBatchEvent::new, m_ringBufferSize); m_workerPool = new WorkerPool<SampleBatchEvent>( m_ringBuffer, m_ringBuffer.newBarrier(), new FatalExceptionHandler(), handlers); m_ringBuffer.addGatingSequences(m_workerPool.getWorkerSequences()); m_workerPool.start(executor); }
public RpcServer build() { final int workerThreads = 3; Producer<String, Response> responseProducer = new KafkaProducer<>(producerProps, new StringSerializer(), producerValueSerializer); final ServerEventHandler[] workHandlers = new ServerEventHandler[workerThreads]; IntStream.range(0, workerThreads).forEach( nbr -> workHandlers[nbr] = new ServerEventHandler(requestHandler, responseProducer) ); final WorkerPool<ServerEvent> workerPool = new WorkerPool<>(ServerEvent::new, new FatalExceptionHandler(), workHandlers); RequestConsumer requestConsumer = new RequestConsumer(topic, consumerProps, consumerThreads, consumerValueDeserializer); return new KafkaRpcServer(workerPool, requestConsumer, new ServerEventHandler(requestHandler, responseProducer), workerThreads); }
EventHandlerGroup<T> createWorkerPool( final Sequence[] barrierSequences, final WorkHandler<? super T>[] workHandlers) { final SequenceBarrier sequenceBarrier = ringBuffer.newBarrier(barrierSequences); final WorkerPool<T> workerPool = new WorkerPool<>(ringBuffer, sequenceBarrier, exceptionHandler, workHandlers); consumerRepository.add(workerPool, sequenceBarrier); final Sequence[] workerSequences = workerPool.getWorkerSequences(); updateGatingSequencesForNextInChain(barrierSequences, workerSequences); return new EventHandlerGroup<>(this, consumerRepository, workerSequences); }
@Bean public WorkerPool<ServerEvent> workerPool(Gson gson, WebSocketKafkaProxyConfig proxyConfig, HandlersMapper requestHandlersMapper) { final ProxyServerEventHandler[] workHandlers = new ProxyServerEventHandler[proxyConfig.getWorkerThreads()]; IntStream.range(0, proxyConfig.getWorkerThreads()).forEach( nbr -> workHandlers[nbr] = new ProxyServerEventHandler(gson, proxyConfig, requestHandlersMapper) ); final RingBuffer<ServerEvent> ringBuffer = RingBuffer.createMultiProducer(ServerEvent::new, proxyConfig.getBufferSize(), getWaitStrategy()); final SequenceBarrier barrier = ringBuffer.newBarrier(); WorkerPool<ServerEvent> workerPool = new WorkerPool<>(ringBuffer, barrier, new FatalExceptionHandler(), workHandlers); ringBuffer.addGatingSequences(workerPool.getWorkerSequences()); return workerPool; }
@Inject PersistenceProcessorImpl(TSOServerConfig config, CommitTable commitTable, ObjectPool<Batch> batchPool, Panicker panicker, PersistenceProcessorHandler[] handlers, MetricsRegistry metrics) throws Exception { this.metrics = metrics; this.lowWatermarkWriter = commitTable.getWriter(); this.batchSequence = 0L; this.batchPool = batchPool; this.currentBatch = batchPool.borrowObject(); // Low Watermark writer ThreadFactoryBuilder lwmThreadFactory = new ThreadFactoryBuilder().setNameFormat("lwm-writer-%d"); lowWatermarkWriterExecutor = Executors.newSingleThreadExecutor(lwmThreadFactory.build()); // Disruptor configuration this.persistRing = RingBuffer.createSingleProducer(EVENT_FACTORY, 1 << 20, new BusySpinWaitStrategy()); ThreadFactoryBuilder threadFactory = new ThreadFactoryBuilder().setNameFormat("persist-%d"); ExecutorService requestExec = Executors.newFixedThreadPool(config.getNumConcurrentCTWriters(), threadFactory.build()); WorkerPool<PersistBatchEvent> persistProcessor = new WorkerPool<>(persistRing, persistRing.newBarrier(), new FatalExceptionHandler(panicker), handlers); this.persistRing.addGatingSequences(persistProcessor.getWorkerSequences()); persistProcessor.start(requestExec); // Metrics config this.lwmWriteTimer = metrics.timer(name("tso", "lwmWriter", "latency")); }
/** * Set up the thread that will handle the non-blocking reads, and writes. */ public SelectorThread(String name, int ringSize, ThreadPoolExecutor invoker, int numHandlers, boolean isSharedInvoker) throws IOException { super(name); this.newConnections = new ConcurrentLinkedQueue<>(); InvocationHandler handlers[] = new InvocationHandler[numHandlers]; for (int i = 0; i < handlers.length; i++) handlers[i] = new InvocationHandler(); this.invoker = invoker; this.isSharedInvoker = isSharedInvoker; /** * YieldingWaitStrategy claims to be better compromise between throughput/latency and CPU usage comparing to * BlockingWaitStrategy, but actual tests show quite the opposite, where YieldingWaitStrategy just constantly * burns CPU cycles with no performance benefit when coupled with networking. */ ringBuffer = RingBuffer.createSingleProducer(Message.Invocation.FACTORY, ringSize, new BlockingWaitStrategy()); workerPool = new WorkerPool<>(ringBuffer, ringBuffer.newBarrier(), new FatalExceptionHandler(), handlers); workerPool.start(invoker); }
@Bean public WorkerPool<ServerEvent> workerPool(Gson gson, RequestResponseMatcher requestResponseMatcher, WebSocketKafkaProxyConfig proxyConfig) { final ProxyResponseHandler[] workHandlers = new ProxyResponseHandler[proxyConfig.getWorkerThreads()]; IntStream.range(0, proxyConfig.getWorkerThreads()).forEach( nbr -> { ProxyResponseHandler handler = new ProxyResponseHandler(gson, REQUEST_TOPIC, RESPONSE_TOPIC, proxyConfig, requestResponseMatcher); handler.start(); workHandlers[nbr] = handler; } ); final RingBuffer<ServerEvent> ringBuffer = RingBuffer.createMultiProducer(ServerEvent::new, proxyConfig.getBufferSize(), getWaitStrategy(proxyConfig.getWaitStrategy())); final SequenceBarrier barrier = ringBuffer.newBarrier(); WorkerPool<ServerEvent> workerPool = new WorkerPool<>(ringBuffer, barrier, new FatalExceptionHandler(), workHandlers); ringBuffer.addGatingSequences(workerPool.getWorkerSequences()); return workerPool; }
/** * @param maxQueueSz * @param numThreads * @param name */ public RequestQueueProcessor(int maxQueueSz, int numThreads, String name) { m_name = name; m_maxQueueSz = maxQueueSz; m_ringBuffer = RingBuffer.createMultiProducer(m_eventFactory, normalizeBufferSize(m_maxQueueSz), new BlockingWaitStrategy()); m_barrier = m_ringBuffer.newBarrier(); m_numThreads = numThreads; QueueProcessorWorkHandler[] handlers = new QueueProcessorWorkHandler[m_numThreads]; for (int i = 0; i < m_numThreads; i++) { handlers[i] = new QueueProcessorWorkHandler(); } m_worker = new WorkerPool(m_ringBuffer, m_barrier, new QueueProcessorExceptionHandler(m_dropCounter, name), handlers); m_ringBuffer.addGatingSequences(m_worker.getWorkerSequences()); m_executor = Executors.newFixedThreadPool(m_numThreads, new NameableThreadFactory(name)); m_ringBuffer = m_worker.start(m_executor); }