/** * Ask the remote zkfc to cede its active status and wait for the specified * timeout before attempting to claim leader status. * @param remote node to ask * @param timeout amount of time to cede * @return the {@link ZKFCProtocol} used to talk to the ndoe * @throws IOException */ private ZKFCProtocol cedeRemoteActive(HAServiceTarget remote, int timeout) throws IOException { LOG.info("Asking " + remote + " to cede its active state for " + timeout + "ms"); ZKFCProtocol oldZkfc = remote.getZKFCProxy(conf, timeout); oldZkfc.cedeActive(timeout); return oldZkfc; }
/** * Initiate a graceful failover by talking to the target node's ZKFC. * This sends an RPC to the ZKFC, which coordinates the failover. * * @param toNode the node to fail to * @return status code (0 for success) * @throws IOException if failover does not succeed */ private int gracefulFailoverThroughZKFCs(HAServiceTarget toNode) throws IOException { int timeout = FailoverController.getRpcTimeoutToNewActive(getConf()); ZKFCProtocol proxy = toNode.getZKFCProxy(getConf(), timeout); try { proxy.gracefulFailover(); out.println("Failover to " + toNode + " successful"); } catch (ServiceFailedException sfe) { errOut.println("Failover failed: " + sfe.getLocalizedMessage()); return -1; } return 0; }
@Override protected HAServiceTarget resolveTarget(String nnId) { HAServiceTarget target = super.resolveTarget(nnId); HAServiceTarget spy = Mockito.spy(target); // OVerride the target to return our mock protocol try { Mockito.doReturn(mockProtocol).when(spy).getProxy( Mockito.<Configuration>any(), Mockito.anyInt()); Mockito.doReturn(mockZkfcProtocol).when(spy).getZKFCProxy( Mockito.<Configuration>any(), Mockito.anyInt()); } catch (IOException e) { throw new AssertionError(e); // mock setup doesn't really throw } return spy; } };
/** * Initiate a graceful failover by talking to the target node's ZKFC. * This sends an RPC to the ZKFC, which coordinates the failover. * * @param toNode the node to fail to * @return status code (0 for success) * @throws IOException if failover does not succeed */ private int gracefulFailoverThroughZKFCs(HAServiceTarget toNode) throws IOException { int timeout = FailoverController.getRpcTimeoutToNewActive(getConf()); ZKFCProtocol proxy = toNode.getZKFCProxy(getConf(), timeout); try { proxy.gracefulFailover(); out.println("Failover to " + toNode + " successful"); } catch (ServiceFailedException sfe) { errOut.println("Failover failed: " + sfe.getLocalizedMessage()); return -1; } return 0; }
/** * Initiate a graceful failover by talking to the target node's ZKFC. * This sends an RPC to the ZKFC, which coordinates the failover. * * @param toNode the node to fail to * @return status code (0 for success) * @throws IOException if failover does not succeed */ private int gracefulFailoverThroughZKFCs(HAServiceTarget toNode) throws IOException { int timeout = FailoverController.getRpcTimeoutToNewActive(getConf()); ZKFCProtocol proxy = toNode.getZKFCProxy(getConf(), timeout); try { proxy.gracefulFailover(); out.println("Failover to " + toNode + " successful"); } catch (ServiceFailedException sfe) { errOut.println("Failover failed: " + sfe.getLocalizedMessage()); return -1; } return 0; }
/** * Initiate a graceful failover by talking to the target node's ZKFC. * This sends an RPC to the ZKFC, which coordinates the failover. * * @param toNode the node to fail to * @return status code (0 for success) * @throws IOException if failover does not succeed */ private int gracefulFailoverThroughZKFCs(HAServiceTarget toNode) throws IOException { int timeout = FailoverController.getRpcTimeoutToNewActive(getConf()); ZKFCProtocol proxy = toNode.getZKFCProxy(getConf(), timeout); try { proxy.gracefulFailover(); out.println("Failover to " + toNode + " successful"); } catch (ServiceFailedException sfe) { errOut.println("Failover failed: " + sfe.getLocalizedMessage()); return -1; } return 0; }
/** * Initiate a graceful failover by talking to the target node's ZKFC. * This sends an RPC to the ZKFC, which coordinates the failover. * * @param toNode the node to fail to * @return status code (0 for success) * @throws IOException if failover does not succeed */ private int gracefulFailoverThroughZKFCs(HAServiceTarget toNode) throws IOException { int timeout = FailoverController.getRpcTimeoutToNewActive(getConf()); ZKFCProtocol proxy = toNode.getZKFCProxy(getConf(), timeout); try { proxy.gracefulFailover(); out.println("Failover to " + toNode + " successful"); } catch (ServiceFailedException sfe) { errOut.println("Failover failed: " + sfe.getLocalizedMessage()); return -1; } return 0; }
ZKFCProtocol oldZkfc = oldActive.getZKFCProxy(conf, timeout); oldZkfc.cedeActive(timeout);
ZKFCProtocol oldZkfc = oldActive.getZKFCProxy(conf, timeout); oldZkfc.cedeActive(timeout);
ZKFCProtocol oldZkfc = oldActive.getZKFCProxy(conf, timeout); oldZkfc.cedeActive(timeout);
@Test(timeout=30000) public void testManualFailover() throws Exception { thr2.zkfc.getLocalTarget().getZKFCProxy(conf, 15000).gracefulFailover(); waitForHAState(0, HAServiceState.STANDBY); waitForHAState(1, HAServiceState.ACTIVE); thr1.zkfc.getLocalTarget().getZKFCProxy(conf, 15000).gracefulFailover(); waitForHAState(0, HAServiceState.ACTIVE); waitForHAState(1, HAServiceState.STANDBY); }
ZKFCProtocol oldZkfc = oldActive.getZKFCProxy(conf, timeout); oldZkfc.cedeActive(timeout);
/** * Test that the ZKFC can gracefully cede its active status. */ @Test public void testCedeActive() throws Exception { cluster.start(); DummyZKFC zkfc = cluster.getZkfc(0); // It should be in active to start. assertEquals(ActiveStandbyElector.State.ACTIVE, zkfc.getElectorForTests().getStateForTests()); // Ask it to cede active for 3 seconds. It should respond promptly // (i.e. the RPC itself should not take 3 seconds!) ZKFCProtocol proxy = zkfc.getLocalTarget().getZKFCProxy(conf, 5000); long st = Time.now(); proxy.cedeActive(3000); long et = Time.now(); assertTrue("RPC to cedeActive took " + (et - st) + " ms", et - st < 1000); // Should be in "INIT" state since it's not in the election // at this point. assertEquals(ActiveStandbyElector.State.INIT, zkfc.getElectorForTests().getStateForTests()); // After the prescribed 3 seconds, should go into STANDBY state, // since the other node in the cluster would have taken ACTIVE. cluster.waitForElectorState(0, ActiveStandbyElector.State.STANDBY); long et2 = Time.now(); assertTrue("Should take ~3 seconds to rejoin. Only took " + (et2 - et) + "ms before rejoining.", et2 - et > 2800); }
ZKFCProtocol proxy = zkfc.getLocalTarget().getZKFCProxy(conf, 5000); long st = Time.now(); proxy.cedeActive(3000);