private void initHM() { healthMonitor = new HealthMonitor(conf, localTarget); healthMonitor.addCallback(new HealthCallbacks()); healthMonitor.addServiceStateCallback(new ServiceStateCallBacks()); healthMonitor.start(); }
private void tryConnect() { Preconditions.checkState(proxy == null); try { synchronized (this) { proxy = createProxy(); } } catch (IOException e) { LOG.warn("Could not connect to local service at " + targetToMonitor + ": " + e.getMessage()); proxy = null; enterState(State.SERVICE_NOT_RESPONDING); } }
healthy = true; } catch (Throwable t) { if (isHealthCheckFailedException(t)) { LOG.warn("Service health check failed for {}", targetToMonitor, t); enterState(State.SERVICE_UNHEALTHY); } else { LOG.warn("Transport-level exception trying to monitor health of {}", RPC.stopProxy(proxy); proxy = null; enterState(State.SERVICE_NOT_RESPONDING); Thread.sleep(sleepAfterDisconnectMillis); return; setLastServiceStatus(status); enterState(State.SERVICE_HEALTHY);
/** * Test that the proper state is propagated when the health monitor * sees an uncaught exception in its thread. */ @Test(timeout=15000) public void testHealthMonitorDies() throws Exception { LOG.info("Mocking RTE in health monitor, waiting for FAILED"); throwOOMEOnCreate = true; svc.actUnreachable = true; waitForState(hm, HealthMonitor.State.HEALTH_MONITOR_FAILED); hm.shutdown(); hm.join(); assertFalse(hm.isAlive()); }
@Override protected HAServiceProtocol createProxy() throws IOException { createProxyCount.incrementAndGet(); if (throwOOMEOnCreate) { throw new OutOfMemoryError("oome"); } return super.createProxy(); } };
/** * Test that, if the callback throws an RTE, this will terminate the * health monitor and thus change its state to FAILED * @throws Exception */ @Test(timeout=15000) public void testCallbackThrowsRTE() throws Exception { hm.addCallback(new Callback() { @Override public void enteredState(State newState) { throw new RuntimeException("Injected RTE"); } }); LOG.info("Mocking bad health check, waiting for UNHEALTHY"); svc.isHealthy = false; waitForState(hm, HealthMonitor.State.HEALTH_MONITOR_FAILED); }
@Before public void setupHM() throws InterruptedException, IOException { Configuration conf = new Configuration(); conf.setInt(CommonConfigurationKeys.IPC_CLIENT_CONNECT_MAX_RETRIES_KEY, 1); conf.setInt(CommonConfigurationKeys.HA_HM_CHECK_INTERVAL_KEY, 50); conf.setInt(CommonConfigurationKeys.HA_HM_CONNECT_RETRY_INTERVAL_KEY, 50); conf.setInt(CommonConfigurationKeys.HA_HM_SLEEP_AFTER_DISCONNECT_KEY, 50); svc = new DummyHAService(HAServiceState.ACTIVE, new InetSocketAddress("0.0.0.0", 0), true); hm = new HealthMonitor(conf, svc) { @Override protected HAServiceProtocol createProxy() throws IOException { createProxyCount.incrementAndGet(); if (throwOOMEOnCreate) { throw new OutOfMemoryError("oome"); } return super.createProxy(); } }; LOG.info("Starting health monitor"); hm.start(); LOG.info("Waiting for HEALTHY signal"); waitForState(hm, HealthMonitor.State.SERVICE_HEALTHY); }
healthy = true; } catch (Throwable t) { if (isHealthCheckFailedException(t)) { LOG.warn("Service health check failed for " + targetToMonitor + ": " + t.getMessage()); enterState(State.SERVICE_UNHEALTHY); } else { LOG.warn("Transport-level exception trying to monitor health of " + RPC.stopProxy(proxy); proxy = null; enterState(State.SERVICE_NOT_RESPONDING); Thread.sleep(sleepAfterDisconnectMillis); return; setLastServiceStatus(status); enterState(State.SERVICE_HEALTHY);
/** * Test that the proper state is propagated when the health monitor * sees an uncaught exception in its thread. */ @Test(timeout=15000) public void testHealthMonitorDies() throws Exception { LOG.info("Mocking RTE in health monitor, waiting for FAILED"); throwOOMEOnCreate = true; svc.actUnreachable = true; waitForState(hm, HealthMonitor.State.HEALTH_MONITOR_FAILED); hm.shutdown(); hm.join(); assertFalse(hm.isAlive()); }
@Override protected HAServiceProtocol createProxy() throws IOException { createProxyCount.incrementAndGet(); if (throwOOMEOnCreate) { throw new OutOfMemoryError("oome"); } return super.createProxy(); } };
/** * Test that, if the callback throws an RTE, this will terminate the * health monitor and thus change its state to FAILED * @throws Exception */ @Test(timeout=15000) public void testCallbackThrowsRTE() throws Exception { hm.addCallback(new Callback() { @Override public void enteredState(State newState) { throw new RuntimeException("Injected RTE"); } }); LOG.info("Mocking bad health check, waiting for UNHEALTHY"); svc.isHealthy = false; waitForState(hm, HealthMonitor.State.HEALTH_MONITOR_FAILED); }
@Before public void setupHM() throws InterruptedException, IOException { Configuration conf = new Configuration(); conf.setInt(CommonConfigurationKeys.IPC_CLIENT_CONNECT_MAX_RETRIES_KEY, 1); conf.setInt(CommonConfigurationKeys.HA_HM_CHECK_INTERVAL_KEY, 50); conf.setInt(CommonConfigurationKeys.HA_HM_CONNECT_RETRY_INTERVAL_KEY, 50); conf.setInt(CommonConfigurationKeys.HA_HM_SLEEP_AFTER_DISCONNECT_KEY, 50); svc = new DummyHAService(HAServiceState.ACTIVE, new InetSocketAddress("0.0.0.0", 0), true); hm = new HealthMonitor(conf, svc) { @Override protected HAServiceProtocol createProxy() throws IOException { createProxyCount.incrementAndGet(); if (throwOOMEOnCreate) { throw new OutOfMemoryError("oome"); } return super.createProxy(); } }; LOG.info("Starting health monitor"); hm.start(); LOG.info("Waiting for HEALTHY signal"); waitForState(hm, HealthMonitor.State.SERVICE_HEALTHY); }
private void initHM() { healthMonitor = new HealthMonitor(conf, localTarget); healthMonitor.addCallback(new HealthCallbacks()); healthMonitor.addServiceStateCallback(new ServiceStateCallBacks()); healthMonitor.start(); }
healthy = true; } catch (Throwable t) { if (isHealthCheckFailedException(t)) { LOG.warn("Service health check failed for " + targetToMonitor + ": " + t.getMessage()); enterState(State.SERVICE_UNHEALTHY); } else { LOG.warn("Transport-level exception trying to monitor health of " + RPC.stopProxy(proxy); proxy = null; enterState(State.SERVICE_NOT_RESPONDING); Thread.sleep(sleepAfterDisconnectMillis); return; setLastServiceStatus(status); enterState(State.SERVICE_HEALTHY);
@Test(timeout=15000) public void testMonitor() throws Exception { LOG.info("Mocking bad health check, waiting for UNHEALTHY"); svc.isHealthy = false; waitForState(hm, HealthMonitor.State.SERVICE_UNHEALTHY); LOG.info("Returning to healthy state, waiting for HEALTHY"); svc.isHealthy = true; waitForState(hm, HealthMonitor.State.SERVICE_HEALTHY); LOG.info("Returning an IOException, as if node went down"); // should expect many rapid retries int countBefore = createProxyCount.get(); svc.actUnreachable = true; waitForState(hm, HealthMonitor.State.SERVICE_NOT_RESPONDING); // Should retry several times while (createProxyCount.get() < countBefore + 3) { Thread.sleep(10); } LOG.info("Returning to healthy state, waiting for HEALTHY"); svc.actUnreachable = false; waitForState(hm, HealthMonitor.State.SERVICE_HEALTHY); hm.shutdown(); hm.join(); assertFalse(hm.isAlive()); }
private void tryConnect() { Preconditions.checkState(proxy == null); try { synchronized (this) { proxy = createProxy(); } } catch (IOException e) { LOG.warn("Could not connect to local service at " + targetToMonitor + ": " + e.getMessage()); proxy = null; enterState(State.SERVICE_NOT_RESPONDING); } }
private void initHM() { healthMonitor = new HealthMonitor(conf, localTarget); healthMonitor.addCallback(new HealthCallbacks()); healthMonitor.addServiceStateCallback(new ServiceStateCallBacks()); healthMonitor.start(); }
healthy = true; } catch (Throwable t) { if (isHealthCheckFailedException(t)) { LOG.warn("Service health check failed for " + targetToMonitor + ": " + t.getMessage()); enterState(State.SERVICE_UNHEALTHY); } else { LOG.warn("Transport-level exception trying to monitor health of " + RPC.stopProxy(proxy); proxy = null; enterState(State.SERVICE_NOT_RESPONDING); Thread.sleep(sleepAfterDisconnectMillis); return; setLastServiceStatus(status); enterState(State.SERVICE_HEALTHY);