private void check() { now = timer.now(); long duration = now - lastCheckTime; if (duration > checkInIntervalMS) { lastCheckTime = now; final boolean failing = healthManager.isFailing(); if (!failing) { healthServiceAsync.checkInOk(serviceName); } else { healthServiceAsync.checkIn(serviceName, HealthStatus.FAIL); } healthServiceAsync.clientProxyFlush(); } }
@Test public void testCheckInOk() throws Exception { final CountDownLatch countDownLatch = new CountDownLatch(1); final AtomicBoolean result = new AtomicBoolean(); healthService.register("foo", 1, TimeUnit.SECONDS); healthService.checkInOk("foo"); healthService.findHealthyNodes(names -> { result.set(names.stream().anyMatch(s -> s.equals("foo"))); countDownLatch.countDown(); } ); countDownLatch.await(10, TimeUnit.SECONDS); assertTrue("foo is in healthy list", result.get()); }
@Override public void queueShutdown() { healthServiceAsync.unregister(serviceName); }
@RequestMapping(value = "/healthy-nodes/", summary = "List of healthy nodes", description = "List of nodes that are healthy.", returnDescription = "List of healthy nodes") public void findAllHealthyNodes(final Callback<List<String>> callback) { healthService.findHealthyNodes(callback::accept); healthService.clientProxyFlush(); }
@RequestMapping(value = "/load-nodes/", summary = "Load all health info about all nodes", description = "Load all health info about all nodes", returnDescription = "list of healthy nodes") public void loadNodes(final Callback<List<NodeHealthStat>> callback) { healthService.loadNodes(callback); healthService.clientProxyFlush(); }
/** * Read annotation. * * @param callback callback */ @RequestMapping(value = "/all-nodes/", summary = "Finds all nodes that have registered with health system", description = "Finds all service actors and endpoints that are registered with the health system." + "Each node will periodically check in with the health system." + "Nodes can mark themselves unhealthy or just fail to check in", returnDescription = "List of node names") public void findAllNodes(final Callback<List<String>> callback) { healthService.findAllNodes(callback::accept); healthService.clientProxyFlush(); }
@Test public void forceTTLExpire() throws Exception { final CountDownLatch countDownLatch = new CountDownLatch(1); final AtomicBoolean result = new AtomicBoolean(); healthService.register("foo", 1, TimeUnit.SECONDS); healthService.checkIn("foo", HealthStatus.PASS); healthService.findAllNodes(names -> { result.set(names.stream().anyMatch(s -> s.equals("foo"))); countDownLatch.countDown(); } ); countDownLatch.await(10, TimeUnit.SECONDS); assertTrue("foo is found", result.get()); timer.setTime(); timer.minutes(1); Sys.sleep(1000); healthService.clientProxyFlush(); final CountDownLatch countDownLatch2 = new CountDownLatch(1); result.set(false); healthService.findHealthyNodes(names -> { result.set(!names.stream().anyMatch(s -> s.equals("foo"))); countDownLatch2.countDown(); } ); countDownLatch2.await(10, TimeUnit.SECONDS); assertTrue("foo should not be found", result.get()); }
healthService.register("foo", 1, TimeUnit.DAYS); healthService.checkInOk("foo"); healthService.register("bar", 1, TimeUnit.DAYS); healthService.checkInOk("bar"); healthService.clientProxyFlush();
@Test public void testCheckInOkUsingCheckIn() throws Exception { final CountDownLatch countDownLatch = new CountDownLatch(1); final AtomicBoolean result = new AtomicBoolean(); healthService.register("foo", 1, TimeUnit.SECONDS); healthService.checkIn("foo", HealthStatus.PASS); healthService.findHealthyNodes(names -> { result.set(names.stream().anyMatch(s -> s.equals("foo"))); countDownLatch.countDown(); } ); ServiceProxyUtils.flushServiceProxy(healthService); countDownLatch.await(10, TimeUnit.SECONDS); assertTrue("foo is found among the healthy ", result.get()); }
@Test public void testRegister() throws Exception { final AtomicBoolean result = new AtomicBoolean(); healthService.register("foo", 1, TimeUnit.SECONDS); final CountDownLatch countDownLatch = new CountDownLatch(1); healthService.findAllNodes(names -> { result.set(names.stream().anyMatch(s -> s.equals("foo"))); countDownLatch.countDown(); } ); countDownLatch.await(10, TimeUnit.SECONDS); assertTrue("foo is found", result.get()); healthService.unregister("foo"); final CountDownLatch countDownLatch2 = new CountDownLatch(1); result.set(false); healthService.findAllNodes(names -> { result.set(!names.stream().anyMatch(s -> s.equals("foo"))); countDownLatch2.countDown(); } ); countDownLatch2.await(10, TimeUnit.SECONDS); }
private void handleDiscoveryCheckInWithHealth(final AtomicLong lastCheckIn, final long checkInDuration) { final AtomicBoolean ok = new AtomicBoolean(true); httpServer.setHttpRequestsIdleConsumer(aVoid -> { httpRequestServerHandler.httpRequestQueueIdle(null); long now = Timer.clockTime(); if (now > lastCheckIn.get() + checkInDuration) { lastCheckIn.set(now); if (ok.get()) { serviceDiscovery.checkInOk(endpoint.getId()); } else { serviceDiscovery.checkIn(endpoint.getId(), HealthStatus.FAIL); } healthServiceAsync.ok(ok::set); ServiceProxyUtils.flushServiceProxy(healthServiceAsync); } }); }
@Test public void testFindAllHealthyNodes() throws Exception { adminBuilder.getHealthService().checkIn("foo", HealthStatus.FAIL); adminBuilder.getHealthService().clientProxyFlush(); Sys.sleep(1000); HttpTextResponse httpResponse = httpClient.get("/__admin/all-nodes/"); assertTrue(httpResponse.code() == 200); assertEquals("[\"bar\",\"foo\",\"baz\"]", httpResponse.body()); } }
@Before public void setUp() throws Exception { adminBuilder = AdminBuilder.adminBuilder(); int port = adminBuilder.getPort(); port = PortUtils.findOpenPortStartAt(port); adminBuilder.setPort(port); adminBuilder.getHealthService().register("foo", 1, TimeUnit.SECONDS); adminBuilder.getHealthService().register("bar", 1, TimeUnit.SECONDS); adminBuilder.getHealthService().register("baz", 1, TimeUnit.SECONDS); adminBuilder.getHealthService().checkInOk("foo"); adminBuilder.getHealthService().checkInOk("bar"); adminBuilder.getHealthService().checkInOk("baz"); serviceEndpointServer = adminBuilder.build().startServer(); httpClient = HttpClientBuilder.httpClientBuilder().setPort(adminBuilder.getPort()).build().startClient(); }
/** * Read annotation. * * @param responseCallback responseCallback */ @RequestMapping(value = "/ok", summary = "simple health check", description = "Health check. This returns true if all nodes (service actors) are healthy", returnDescription = "true if all nodes are healthy, false if all nodes are not healthy") public void ok(final Callback<HttpTextResponse> responseCallback) { healthService.ok(callbackBuilder().withBooleanCallback(ok -> { if (ok) { responseCallback.resolve(httpResponseBuilder().setBody("true").setCode(200).buildTextResponse()); } else { responseCallback.resolve(httpResponseBuilder().setBody("false").setCode(503).buildTextResponse()); } }).build()); healthService.clientProxyFlush(); }
@Override public void queueInit() { healthServiceAsync.register(serviceName, ttlMS, TimeUnit.MILLISECONDS); }
@Test public void fail() { /* This is why we need to log it. */ healthService.checkIn("no exist", HealthStatus.PASS); }
@Test public void transitionFromPassToFail() throws Exception { final CountDownLatch countDownLatch = new CountDownLatch(1); final AtomicBoolean result = new AtomicBoolean(); healthService.register("foo", 1, TimeUnit.SECONDS); healthService.checkIn("foo", HealthStatus.PASS); healthService.findHealthyNodes(names -> { result.set(names.stream().anyMatch(s -> s.equals("foo"))); countDownLatch.countDown(); } ); countDownLatch.await(10, TimeUnit.SECONDS); healthService.checkIn("foo", HealthStatus.FAIL); final CountDownLatch countDownLatch2 = new CountDownLatch(1); result.set(false); healthService.findHealthyNodes(names -> { result.set(!names.stream().anyMatch(s -> s.equals("foo"))); countDownLatch2.countDown(); } ); countDownLatch2.await(10, TimeUnit.SECONDS); assertTrue("foo is NOT found among the healthy ", result.get()); }
public void handleCheckIn() { if (healthServiceAsync == null) { if (Timer.clockTime() - lastCheckIn.get() > checkInEveryMiliDuration) { lastCheckIn.set(Timer.clockTime()); serviceDiscovery.checkInOk(endpointDefinition.getId()); } } else { if (Timer.clockTime() - lastCheckIn.get() > checkInEveryMiliDuration) { lastCheckIn.set(Timer.clockTime()); healthServiceAsync.ok(ok::set); ServiceProxyUtils.flushServiceProxy(healthServiceAsync); if (ok.get()) { serviceDiscovery.checkInOk(endpointDefinition.getId()); } else { serviceDiscovery.checkIn(endpointDefinition.getId(), HealthStatus.FAIL); } ServiceProxyUtils.flushServiceProxy(serviceDiscovery); } } }
@Override public void queueInit() { healthServiceAsync.register(serviceName, ttlMS, TimeUnit.MILLISECONDS); }
private void check() { now = timer.now(); long duration = now - lastCheckTime; if (duration > checkInIntervalMS) { lastCheckTime = now; final boolean failing = healthManager.isFailing(); if (!failing) { healthServiceAsync.checkInOk(serviceName); } else { healthServiceAsync.checkIn(serviceName, HealthStatus.FAIL); } healthServiceAsync.clientProxyFlush(); } }