public static void testRecoveryAndDoubleExecution(final HBaseTestingUtility testUtil, final long procId, final int lastStepBeforeFailover) throws Exception { ProcedureExecutor<MasterProcedureEnv> procExec = testUtil.getHBaseCluster().getMaster().getMasterProcedureExecutor(); ProcedureTestingUtility.waitProcedure(procExec, procId); final Procedure proc = procExec.getProcedure(procId); for (int i = 0; i < lastStepBeforeFailover; ++i) { LOG.info("Restart "+ i +" exec state: " + proc); ProcedureTestingUtility.assertProcNotYetCompleted(procExec, procId); MasterProcedureTestingUtility.restartMasterProcedureExecutor(procExec); ProcedureTestingUtility.waitProcedure(procExec, procId); } ProcedureTestingUtility.assertProcNotYetCompleted(procExec, procId); LOG.info("Trigger master failover"); MasterProcedureTestingUtility.masterFailover(testUtil); procExec = testUtil.getHBaseCluster().getMaster().getMasterProcedureExecutor(); ProcedureTestingUtility.waitProcedure(procExec, procId); ProcedureTestingUtility.assertProcNotFailed(procExec, procId); }
/** * Restart the ProcedureExecutor and inject an abort to the specified procedure. * If the procedure implement abort() this should result in rollback being triggered. * At the end of this call the procedure should be finished and rolledback, if abort is implemnted */ public static void testRestartWithAbort(ProcedureExecutor<MasterProcedureEnv> procExec, long procId) throws Exception { ProcedureTestingUtility.setKillAndToggleBeforeStoreUpdate(procExec, false); InjectAbortOnLoadListener abortListener = new InjectAbortOnLoadListener(procExec); abortListener.addProcId(procId); procExec.registerListener(abortListener); try { ProcedureTestingUtility.assertProcNotYetCompleted(procExec, procId); LOG.info("Restart and rollback procId=" + procId); restartMasterProcedureExecutor(procExec); ProcedureTestingUtility.waitProcedure(procExec, procId); } finally { assertTrue(procExec.unregisterListener(abortListener)); } }
ProcedureTestingUtility.assertProcNotYetCompleted(procExecutor, procId); assertFalse(procExecutor.isRunning()); ProcedureTestingUtility.assertProcNotYetCompleted(procExecutor, procId); assertFalse(procExecutor.isRunning()); ProcedureTestingUtility.assertProcNotYetCompleted(procExecutor, procId); assertFalse(procExecutor.isRunning()); assertTrue(procExecutor.abort(procId)); waitProcedure(procId); ProcedureTestingUtility.assertProcNotYetCompleted(procExecutor, procId); assertFalse(procExecutor.isRunning()); waitProcedure(procId); assertFalse(procExecutor.isRunning()); ProcedureTestingUtility.assertProcNotYetCompleted(procExecutor, procId); waitProcedure(procId); assertFalse(procExecutor.isRunning()); ProcedureTestingUtility.assertProcNotYetCompleted(procExecutor, procId);
ProcedureTestingUtility.assertProcNotYetCompleted(procExecutor, procId); assertFalse(procExecutor.isRunning()); ProcedureTestingUtility.assertProcNotYetCompleted(procExecutor, procId); assertFalse(procExecutor.isRunning()); ProcedureTestingUtility.assertProcNotYetCompleted(procExecutor, procId); assertFalse(procExecutor.isRunning()); ProcedureTestingUtility.assertProcNotYetCompleted(procExecutor, procId); assertFalse(procExecutor.isRunning());
@Test public void testMultiStepProcRecovery() throws Exception { // Step 0 - kill Procedure proc = new TestMultiStepProcedure(); long procId = ProcedureTestingUtility.submitAndWait(procExecutor, proc); assertFalse(procExecutor.isRunning()); // Step 0 exec && Step 1 - kill restart(); waitProcedure(procId); ProcedureTestingUtility.assertProcNotYetCompleted(procExecutor, procId); assertFalse(procExecutor.isRunning()); // Step 1 exec && step 2 - kill restart(); waitProcedure(procId); ProcedureTestingUtility.assertProcNotYetCompleted(procExecutor, procId); assertFalse(procExecutor.isRunning()); // Step 2 exec restart(); waitProcedure(procId); assertTrue(procExecutor.isRunning()); // The procedure is completed Procedure<?> result = procExecutor.getResult(procId); ProcedureTestingUtility.assertProcNotFailed(result); }
ProcedureTestingUtility.assertProcNotYetCompleted(procExec, procId); restartMasterProcedureExecutor(procExec); ProcedureTestingUtility.waitProcedure(procExec, procId);
for (int i = 0; !procExec.isFinished(procId); ++i) { LOG.info("Restart " + i + " rollback state: " + procExec.getProcedure(procId)); ProcedureTestingUtility.assertProcNotYetCompleted(procExec, procId); restartMasterProcedureExecutor(procExec); ProcedureTestingUtility.waitProcedure(procExec, procId);
ProcedureTestingUtility.assertProcNotYetCompleted(procExecutor, procId); assertFalse(procExecutor.isRunning()); ProcedureTestingUtility.assertProcNotYetCompleted(procExecutor, procId); assertFalse(procExecutor.isRunning()); ProcedureTestingUtility.assertProcNotYetCompleted(procExecutor, procId); assertFalse(procExecutor.isRunning());
@Test public void testAbortProcedureFailure() throws Exception { final TableName tableName = TableName.valueOf(name.getMethodName()); final ProcedureExecutor<MasterProcedureEnv> procExec = getMasterProcedureExecutor(); RegionInfo[] regions = MasterProcedureTestingUtility.createTable(procExec, tableName, null, "f"); UTIL.getAdmin().disableTable(tableName); ProcedureTestingUtility.waitNoProcedureRunning(procExec); ProcedureTestingUtility.setKillAndToggleBeforeStoreUpdate(procExec, true); // Submit an un-abortable procedure long procId = procExec.submitProcedure( new DeleteTableProcedure(procExec.getEnvironment(), tableName)); // Wait for a couple of steps to complete (first step "prepare" is abortable) ProcedureTestingUtility.waitProcedure(procExec, procId); for (int i = 0; i < 2; ++i) { ProcedureTestingUtility.assertProcNotYetCompleted(procExec, procId); ProcedureTestingUtility.restart(procExec); ProcedureTestingUtility.waitProcedure(procExec, procId); } boolean abortResult = procExec.abort(procId, true); assertFalse(abortResult); MasterProcedureTestingUtility.testRestartWithAbort(procExec, procId); ProcedureTestingUtility.waitNoProcedureRunning(procExec); ProcedureTestingUtility.assertProcNotFailed(procExec, procId); // Validate the delete table procedure was not aborted MasterProcedureTestingUtility.validateTableDeletion( UTIL.getHBaseCluster().getMaster(), tableName); }
public static void testRecoveryAndDoubleExecution(final HBaseTestingUtility testUtil, final long procId, final int lastStepBeforeFailover) throws Exception { ProcedureExecutor<MasterProcedureEnv> procExec = testUtil.getHBaseCluster().getMaster().getMasterProcedureExecutor(); ProcedureTestingUtility.waitProcedure(procExec, procId); final Procedure proc = procExec.getProcedure(procId); for (int i = 0; i < lastStepBeforeFailover; ++i) { LOG.info("Restart "+ i +" exec state: " + proc); ProcedureTestingUtility.assertProcNotYetCompleted(procExec, procId); MasterProcedureTestingUtility.restartMasterProcedureExecutor(procExec); ProcedureTestingUtility.waitProcedure(procExec, procId); } ProcedureTestingUtility.assertProcNotYetCompleted(procExec, procId); LOG.info("Trigger master failover"); MasterProcedureTestingUtility.masterFailover(testUtil); procExec = testUtil.getHBaseCluster().getMaster().getMasterProcedureExecutor(); ProcedureTestingUtility.waitProcedure(procExec, procId); ProcedureTestingUtility.assertProcNotFailed(procExec, procId); }
/** * Restart the ProcedureExecutor and inject an abort to the specified procedure. * If the procedure implement abort() this should result in rollback being triggered. * At the end of this call the procedure should be finished and rolledback, if abort is implemnted */ public static void testRestartWithAbort(ProcedureExecutor<MasterProcedureEnv> procExec, long procId) throws Exception { ProcedureTestingUtility.setKillAndToggleBeforeStoreUpdate(procExec, false); InjectAbortOnLoadListener abortListener = new InjectAbortOnLoadListener(procExec); abortListener.addProcId(procId); procExec.registerListener(abortListener); try { ProcedureTestingUtility.assertProcNotYetCompleted(procExec, procId); LOG.info("Restart and rollback procId=" + procId); restartMasterProcedureExecutor(procExec); ProcedureTestingUtility.waitProcedure(procExec, procId); } finally { assertTrue(procExec.unregisterListener(abortListener)); } }
@Test public void testMultiStepProcRecovery() throws Exception { // Step 0 - kill Procedure proc = new TestMultiStepProcedure(); long procId = ProcedureTestingUtility.submitAndWait(procExecutor, proc); assertFalse(procExecutor.isRunning()); // Step 0 exec && Step 1 - kill restart(); waitProcedure(procId); ProcedureTestingUtility.assertProcNotYetCompleted(procExecutor, procId); assertFalse(procExecutor.isRunning()); // Step 1 exec && step 2 - kill restart(); waitProcedure(procId); ProcedureTestingUtility.assertProcNotYetCompleted(procExecutor, procId); assertFalse(procExecutor.isRunning()); // Step 2 exec restart(); waitProcedure(procId); assertTrue(procExecutor.isRunning()); // The procedure is completed Procedure<?> result = procExecutor.getResult(procId); ProcedureTestingUtility.assertProcNotFailed(result); }
ProcedureTestingUtility.assertProcNotYetCompleted(procExec, procId); restartMasterProcedureExecutor(procExec); ProcedureTestingUtility.waitProcedure(procExec, procId);
ProcedureTestingUtility.assertProcNotYetCompleted(procExecutor, procId); assertFalse(procExecutor.isRunning()); ProcedureTestingUtility.assertProcNotYetCompleted(procExecutor, procId); assertFalse(procExecutor.isRunning()); ProcedureTestingUtility.assertProcNotYetCompleted(procExecutor, procId); assertFalse(procExecutor.isRunning()); ProcedureTestingUtility.assertProcNotYetCompleted(procExecutor, procId); assertFalse(procExecutor.isRunning());
ProcedureTestingUtility.assertProcNotYetCompleted(procExecutor, procId); assertFalse(procExecutor.isRunning()); ProcedureTestingUtility.assertProcNotYetCompleted(procExecutor, procId); assertFalse(procExecutor.isRunning()); ProcedureTestingUtility.assertProcNotYetCompleted(procExecutor, procId); assertFalse(procExecutor.isRunning()); ProcedureTestingUtility.assertProcNotYetCompleted(procExecutor, procId); assertFalse(procExecutor.isRunning());
@Test public void testMultiStepProcRecovery() throws Exception { // Step 0 - kill Procedure proc = new TestMultiStepProcedure(); long procId = ProcedureTestingUtility.submitAndWait(procExecutor, proc); assertFalse(procExecutor.isRunning()); // Step 0 exec && Step 1 - kill restart(); waitProcedure(procId); ProcedureTestingUtility.assertProcNotYetCompleted(procExecutor, procId); assertFalse(procExecutor.isRunning()); // Step 1 exec && step 2 - kill restart(); waitProcedure(procId); ProcedureTestingUtility.assertProcNotYetCompleted(procExecutor, procId); assertFalse(procExecutor.isRunning()); // Step 2 exec restart(); waitProcedure(procId); assertTrue(procExecutor.isRunning()); // The procedure is completed Procedure<?> result = procExecutor.getResult(procId); ProcedureTestingUtility.assertProcNotFailed(result); }
for (int i = 0; !procExec.isFinished(procId); ++i) { LOG.info("Restart " + i + " rollback state: " + procExec.getProcedure(procId)); ProcedureTestingUtility.assertProcNotYetCompleted(procExec, procId); restartMasterProcedureExecutor(procExec); ProcedureTestingUtility.waitProcedure(procExec, procId);
ProcedureTestingUtility.assertProcNotYetCompleted(procExecutor, procId); assertFalse(procExecutor.isRunning()); ProcedureTestingUtility.assertProcNotYetCompleted(procExecutor, procId); assertFalse(procExecutor.isRunning()); ProcedureTestingUtility.assertProcNotYetCompleted(procExecutor, procId); assertFalse(procExecutor.isRunning());
ProcedureTestingUtility.assertProcNotYetCompleted(procExecutor, procId); assertFalse(procExecutor.isRunning()); ProcedureTestingUtility.assertProcNotYetCompleted(procExecutor, procId); assertFalse(procExecutor.isRunning()); ProcedureTestingUtility.assertProcNotYetCompleted(procExecutor, procId); assertFalse(procExecutor.isRunning());
@Test public void testAbortProcedureFailure() throws Exception { final TableName tableName = TableName.valueOf(name.getMethodName()); final ProcedureExecutor<MasterProcedureEnv> procExec = getMasterProcedureExecutor(); RegionInfo[] regions = MasterProcedureTestingUtility.createTable(procExec, tableName, null, "f"); UTIL.getAdmin().disableTable(tableName); ProcedureTestingUtility.waitNoProcedureRunning(procExec); ProcedureTestingUtility.setKillAndToggleBeforeStoreUpdate(procExec, true); // Submit an un-abortable procedure long procId = procExec.submitProcedure( new DeleteTableProcedure(procExec.getEnvironment(), tableName)); // Wait for a couple of steps to complete (first step "prepare" is abortable) ProcedureTestingUtility.waitProcedure(procExec, procId); for (int i = 0; i < 2; ++i) { ProcedureTestingUtility.assertProcNotYetCompleted(procExec, procId); ProcedureTestingUtility.restart(procExec); ProcedureTestingUtility.waitProcedure(procExec, procId); } boolean abortResult = procExec.abort(procId, true); assertFalse(abortResult); MasterProcedureTestingUtility.testRestartWithAbort(procExec, procId); ProcedureTestingUtility.waitNoProcedureRunning(procExec); ProcedureTestingUtility.assertProcNotFailed(procExec, procId); // Validate the delete table procedure was not aborted MasterProcedureTestingUtility.validateTableDeletion( UTIL.getHBaseCluster().getMaster(), tableName); }