private CompletableFuture<Void> handleHostRemoved(Host host) { return Futures.allOf(sweepers.stream().map(sweeper -> { if (sweeper.isReady()) { // Note: if we find sweeper to be ready, it is possible that this processes can be swept by both // sweepFailedProcesses and handleFailedProcess. A sweep is safe and idempotent operation. return RetryHelper.withIndefiniteRetriesAsync(() -> sweeper.handleFailedProcess(host.getHostId()), e -> log.warn(e.getMessage()), executor); } else { return CompletableFuture.completedFuture((Void) null); } }).collect(Collectors.toList())); }
@Before public void setup() throws Exception { // 1. Start ZK server. zkServer = new TestingServerStarter().start(); // 2. Start ZK client. curatorClient = CuratorFrameworkFactory.newClient(zkServer.getConnectString(), new ExponentialBackoffRetry(200, 10, 5000)); curatorClient.start(); // 3. Start executor service. executor = Executors.newScheduledThreadPool(5); // 4. start cluster event listener clusterZK = new ClusterZKImpl(curatorClient, ClusterType.CONTROLLER); clusterZK.addListener((eventType, host) -> { switch (eventType) { case HOST_ADDED: nodeAddedQueue.offer(host.getHostId()); break; case HOST_REMOVED: nodeRemovedQueue.offer(host.getHostId()); break; case ERROR: default: break; } }); }
serviceConfig.getGRPCServerConfig().get().getTokenSigningKey()); streamMetadataTasks = new StreamMetadataTasks(streamStore, hostStore, taskMetadataStore, segmentHelper, controllerExecutor, host.getHostId(), connectionFactory, authHelper, requestTracker); streamTransactionMetadataTasks = new StreamTransactionMetadataTasks(streamStore, hostStore, segmentHelper, controllerExecutor, host.getHostId(), serviceConfig.getTimeoutServiceConfig(), connectionFactory, authHelper); streamCutService = new StreamCutService(Config.BUCKET_COUNT, host.getHostId(), streamStore, streamMetadataTasks, retentionExecutor, requestTracker); log.info("starting auto retention service asynchronously"); TaskSweeper taskSweeper = new TaskSweeper(taskMetadataStore, host.getHostId(), controllerExecutor, streamMetadataTasks); controllerEventProcessors = new ControllerEventProcessors(host.getHostId(), serviceConfig.getEventProcessorConfig().get(), localController, checkpointStore, streamStore, connectionFactory, streamMetadataTasks, streamTransactionMetadataTasks,
TaskSweeper taskSweeper = spy(new TaskSweeper(taskStore, host.getHostId(), executor, new TestTasks(taskStore, executor, host.getHostId()))); segmentHelper, executor, host.getHostId(), connectionFactory, AuthHelper.getDisabledAuthHelper()); verify(taskSweeper, times(0)).handleFailedProcess(anyString()); verify(txnSweeper, times(0)).handleFailedProcess(anyString()); validateAddedNode(host.getHostId()); validateAddedNode(newHost.getHostId()); clusterZK.deregisterHost(newHost); validateRemovedNode(newHost.getHostId()); log.info("deregistering new host"); validateAddedNode(newHost.getHostId()); clusterZK.deregisterHost(newHost); log.info("removing newhost2"); validateRemovedNode(newHost.getHostId()); assertTrue(Futures.await(taskHostSweep2, 3000)); assertTrue(Futures.await(txnHostSweep2, 3000));
TaskSweeper taskSweeper = new TaskSweeper(taskStore, host.getHostId(), executor, new TestTasks(taskStore, executor, host.getHostId())); ConnectionFactory connectionFactory = mock(ConnectionFactory.class); StreamTransactionMetadataTasks txnTasks = new StreamTransactionMetadataTasks(streamStore, hostStore, segmentHelper, executor, host.getHostId(), connectionFactory, AuthHelper.getDisabledAuthHelper()); txnTasks.initializeStreamWriters("commitStream", new EventStreamWriterMock<>(), "abortStream", new EventStreamWriterMock<>()); validateAddedNode(host.getHostId()); validateAddedNode(host1.getHostId()); validateRemovedNode(host1.getHostId()); validateRemovedNode(host.getHostId());