private void testTimeoutEventProcedureDoubleExecution(final boolean killIfSuspended) throws Exception { TestTimeoutEventProcedure proc = new TestTimeoutEventProcedure(1000, 3); ProcedureTestingUtility.setKillAndToggleBeforeStoreUpdate(procExecutor, true); ProcedureTestingUtility.setKillIfSuspended(procExecutor, killIfSuspended); long procId = procExecutor.submitProcedure(proc); ProcedureTestingUtility.testRecoveryAndDoubleExecution(procExecutor, procId, true); ProcedureTestingUtility.assertIsAbortException(procExecutor.getResult(proc.getProcId())); }
@Test public void testAbortProcedureFailure() throws Exception { final TableName tableName = TableName.valueOf(name.getMethodName()); final ProcedureExecutor<MasterProcedureEnv> procExec = getMasterProcedureExecutor(); RegionInfo[] regions = MasterProcedureTestingUtility.createTable(procExec, tableName, null, "f"); UTIL.getAdmin().disableTable(tableName); ProcedureTestingUtility.waitNoProcedureRunning(procExec); ProcedureTestingUtility.setKillAndToggleBeforeStoreUpdate(procExec, true); // Submit an un-abortable procedure long procId = procExec.submitProcedure( new DeleteTableProcedure(procExec.getEnvironment(), tableName)); // Wait for a couple of steps to complete (first step "prepare" is abortable) ProcedureTestingUtility.waitProcedure(procExec, procId); for (int i = 0; i < 2; ++i) { ProcedureTestingUtility.assertProcNotYetCompleted(procExec, procId); ProcedureTestingUtility.restart(procExec); ProcedureTestingUtility.waitProcedure(procExec, procId); } boolean abortResult = procExec.abort(procId, true); assertFalse(abortResult); MasterProcedureTestingUtility.testRestartWithAbort(procExec, procId); ProcedureTestingUtility.waitNoProcedureRunning(procExec); ProcedureTestingUtility.assertProcNotFailed(procExec, procId); // Validate the delete table procedure was not aborted MasterProcedureTestingUtility.validateTableDeletion( UTIL.getHBaseCluster().getMaster(), tableName); }
private void createProcExecutor() throws Exception { logDir = new Path(testDir, name.getMethodName()); procStore = ProcedureTestingUtility.createWalStore(htu.getConfiguration(), logDir); procExecutor = new ProcedureExecutor<>(htu.getConfiguration(), null, procStore); procStore.start(PROCEDURE_EXECUTOR_SLOTS); ProcedureTestingUtility.initAndStartWorkers(procExecutor, PROCEDURE_EXECUTOR_SLOTS, true, true); }
public static <TEnv> void setKillAndToggleBeforeStoreUpdate(ProcedureExecutor<TEnv> procExecutor, boolean value) { ProcedureTestingUtility.setKillBeforeStoreUpdate(procExecutor, value); ProcedureTestingUtility.setToggleKillBeforeStoreUpdate(procExecutor, value); assertSingleExecutorForKillTests(procExecutor); }
@Test public void testChildRollbackLoad() throws Exception { procEnv.toggleKillBeforeStoreUpdate = false; procEnv.triggerRollbackOnChild = true; TestRootProcedure proc = new TestRootProcedure(); long procId = ProcedureTestingUtility.submitAndWait(procExecutor, proc); ProcedureTestingUtility.restart(procExecutor); ProcedureTestingUtility.waitProcedure(procExecutor, proc); assertProcFailed(procId); }
for (int i = 0; !procExec.isFinished(procId); ++i) { LOG.info("Restart " + i + " rollback state: " + procExec.getProcedure(procId)); ProcedureTestingUtility.assertProcNotYetCompleted(procExec, procId); restartMasterProcedureExecutor(procExec); ProcedureTestingUtility.waitProcedure(procExec, procId); ProcedureTestingUtility.setKillAndToggleBeforeStoreUpdate(procExec, false); ProcedureTestingUtility.waitNoProcedureRunning(procExec); ProcedureTestingUtility.assertIsAbortException(procExec.getResult(procId));
@Test public void testStateMachineRecovery() throws Exception { ProcedureTestingUtility.setToggleKillBeforeStoreUpdate(procExecutor, true); ProcedureTestingUtility.setKillBeforeStoreUpdate(procExecutor, true); long procId = ProcedureTestingUtility.submitAndWait(procExecutor, proc); assertFalse(procExecutor.isRunning()); ProcedureTestingUtility.assertProcNotYetCompleted(procExecutor, procId); assertFalse(procExecutor.isRunning()); ProcedureTestingUtility.assertProcNotYetCompleted(procExecutor, procId); assertFalse(procExecutor.isRunning()); ProcedureTestingUtility.assertProcNotYetCompleted(procExecutor, procId); assertFalse(procExecutor.isRunning()); ProcedureTestingUtility.assertProcNotFailed(result); assertEquals(26, Bytes.toInt(result.getResult()));
@Test public void testStateMachineRollbackRecovery() throws Exception { ProcedureTestingUtility.setToggleKillBeforeStoreUpdate(procExecutor, true); ProcedureTestingUtility.setKillBeforeStoreUpdate(procExecutor, true); long procId = ProcedureTestingUtility.submitAndWait(procExecutor, proc); ProcedureTestingUtility.assertProcNotYetCompleted(procExecutor, procId); assertFalse(procExecutor.isRunning()); ProcedureTestingUtility.assertProcNotYetCompleted(procExecutor, procId); assertFalse(procExecutor.isRunning()); ProcedureTestingUtility.assertProcNotYetCompleted(procExecutor, procId); assertFalse(procExecutor.isRunning()); assertTrue(procExecutor.abort(procId)); waitProcedure(procId); ProcedureTestingUtility.assertProcNotYetCompleted(procExecutor, procId); assertFalse(procExecutor.isRunning()); waitProcedure(procId); assertFalse(procExecutor.isRunning()); ProcedureTestingUtility.assertProcNotYetCompleted(procExecutor, procId); waitProcedure(procId); assertFalse(procExecutor.isRunning()); ProcedureTestingUtility.assertProcNotYetCompleted(procExecutor, procId); ProcedureTestingUtility.assertIsAbortException(result);
@Test public void testChildLoad() throws Exception { procEnv.toggleKillBeforeStoreUpdate = false; TestRootProcedure proc = new TestRootProcedure(); long procId = ProcedureTestingUtility.submitAndWait(procExecutor, proc); ProcedureTestingUtility.restart(procExecutor); ProcedureTestingUtility.waitProcedure(procExecutor, proc); assertTrue("expected completed proc", procExecutor.isFinished(procId)); ProcedureTestingUtility.assertProcNotFailed(procExecutor, procId); }
@Test public void testDeleteSystemNamespace() throws Exception { final String namespaceName = NamespaceDescriptor.SYSTEM_NAMESPACE.getName(); final ProcedureExecutor<MasterProcedureEnv> procExec = getMasterProcedureExecutor(); long procId = procExec.submitProcedure( new DeleteNamespaceProcedure(procExec.getEnvironment(), namespaceName)); // Wait the completion ProcedureTestingUtility.waitProcedure(procExec, procId); Procedure<?> result = procExec.getResult(procId); assertTrue(result.isFailed()); LOG.debug("Delete namespace failed with exception: " + result.getException()); assertTrue(ProcedureTestingUtility.getExceptionCause(result) instanceof ConstraintException); }
private void createNamespaceForTesting(NamespaceDescriptor nsDescriptor) throws Exception { final ProcedureExecutor<MasterProcedureEnv> procExec = getMasterProcedureExecutor(); long procId = procExec.submitProcedure( new CreateNamespaceProcedure(procExec.getEnvironment(), nsDescriptor)); // Wait the completion ProcedureTestingUtility.waitProcedure(procExec, procId); ProcedureTestingUtility.assertProcNotFailed(procExec, procId); } }
@Test public void testMultiStepProcRecovery() throws Exception { // Step 0 - kill Procedure proc = new TestMultiStepProcedure(); long procId = ProcedureTestingUtility.submitAndWait(procExecutor, proc); assertFalse(procExecutor.isRunning()); // Step 0 exec && Step 1 - kill restart(); waitProcedure(procId); ProcedureTestingUtility.assertProcNotYetCompleted(procExecutor, procId); assertFalse(procExecutor.isRunning()); // Step 1 exec && step 2 - kill restart(); waitProcedure(procId); ProcedureTestingUtility.assertProcNotYetCompleted(procExecutor, procId); assertFalse(procExecutor.isRunning()); // Step 2 exec restart(); waitProcedure(procId); assertTrue(procExecutor.isRunning()); // The procedure is completed Procedure<?> result = procExecutor.getResult(procId); ProcedureTestingUtility.assertProcNotFailed(result); }
@Before public void setup() throws Exception { setupConf(UTIL.getConfiguration()); // Set master number and use default values for other options. StartMiniClusterOption option = StartMiniClusterOption.builder().numMasters(2).build(); UTIL.startMiniCluster(option); final ProcedureExecutor<MasterProcedureEnv> procExec = getMasterProcedureExecutor(); ProcedureTestingUtility.setToggleKillBeforeStoreUpdate(procExec, false); ProcedureTestingUtility.setKillBeforeStoreUpdate(procExec, false); }
public static void testRecoveryAndDoubleExecution(final HBaseTestingUtility testUtil, final long procId, final int lastStepBeforeFailover) throws Exception { ProcedureExecutor<MasterProcedureEnv> procExec = testUtil.getHBaseCluster().getMaster().getMasterProcedureExecutor(); ProcedureTestingUtility.waitProcedure(procExec, procId); final Procedure proc = procExec.getProcedure(procId); for (int i = 0; i < lastStepBeforeFailover; ++i) { LOG.info("Restart "+ i +" exec state: " + proc); ProcedureTestingUtility.assertProcNotYetCompleted(procExec, procId); MasterProcedureTestingUtility.restartMasterProcedureExecutor(procExec); ProcedureTestingUtility.waitProcedure(procExec, procId); } ProcedureTestingUtility.assertProcNotYetCompleted(procExec, procId); LOG.info("Trigger master failover"); MasterProcedureTestingUtility.masterFailover(testUtil); procExec = testUtil.getHBaseCluster().getMaster().getMasterProcedureExecutor(); ProcedureTestingUtility.waitProcedure(procExec, procId); ProcedureTestingUtility.assertProcNotFailed(procExec, procId); }
/** * Restart the ProcedureExecutor and inject an abort to the specified procedure. * If the procedure implement abort() this should result in rollback being triggered. * At the end of this call the procedure should be finished and rolledback, if abort is implemnted */ public static void testRestartWithAbort(ProcedureExecutor<MasterProcedureEnv> procExec, long procId) throws Exception { ProcedureTestingUtility.setKillAndToggleBeforeStoreUpdate(procExec, false); InjectAbortOnLoadListener abortListener = new InjectAbortOnLoadListener(procExec); abortListener.addProcId(procId); procExec.registerListener(abortListener); try { ProcedureTestingUtility.assertProcNotYetCompleted(procExec, procId); LOG.info("Restart and rollback procId=" + procId); restartMasterProcedureExecutor(procExec); ProcedureTestingUtility.waitProcedure(procExec, procId); } finally { assertTrue(procExec.unregisterListener(abortListener)); } }
@Test public void testRollbackRetriableFailure() { long procId = ProcedureTestingUtility.submitAndWait(procExecutor, new TestFaultyRollback()); Procedure<?> result = procExecutor.getResult(procId); assertTrue("expected a failure", result.isFailed()); LOG.info(result.getException().getMessage()); Throwable cause = ProcedureTestingUtility.getExceptionCause(result); assertTrue("expected TestProcedureException, got " + cause, cause instanceof TestProcedureException); }
@Test public void testSingleSequentialProc() { List<String> state = new ArrayList<>(); Procedure<Void> subProc2 = new TestSequentialProcedure("subProc2", state); Procedure<Void> subProc1 = new TestSequentialProcedure("subProc1", state, subProc2); Procedure<Void> rootProc = new TestSequentialProcedure("rootProc", state, subProc1); long rootId = ProcedureTestingUtility.submitAndWait(procExecutor, rootProc); // successful state, with 3 execute LOG.info(Objects.toString(state)); Procedure<?> result = procExecutor.getResult(rootId); ProcedureTestingUtility.assertProcNotFailed(result); assertEquals(state.toString(), 3, state.size()); }
@Test public void testChildOnLastStepDoubleExecution() throws Exception { ProcedureTestingUtility.setKillAndToggleBeforeStoreUpdate(procExecutor, true); long procId = procExecutor.submitProcedure(new TestSMProcedure()); ProcedureTestingUtility.testRecoveryAndDoubleExecution(procExecutor, procId); assertEquals(6, procExecutor.getEnvironment().execCount.get()); assertEquals(0, procExecutor.getEnvironment().rollbackCount.get()); ProcedureTestingUtility.assertProcNotFailed(procExecutor, procId); }
/** * Tests being able to suspend a Procedure for N timeouts and then failing.s * Resets the timeout after each elapses. See {@link TestTimeoutEventProcedure} for example * of how to do this sort of trickery with the ProcedureExecutor; i.e. suspend for a while, * check for a condition and if not set, suspend again, etc., ultimately failing or succeeding * eventually. */ @Test public void testTimeoutEventProcedure() throws Exception { final int NTIMEOUTS = 5; TestTimeoutEventProcedure proc = new TestTimeoutEventProcedure(500, NTIMEOUTS); procExecutor.submitProcedure(proc); ProcedureTestingUtility.waitProcedure(procExecutor, proc.getProcId()); ProcedureTestingUtility.assertIsAbortException(procExecutor.getResult(proc.getProcId())); assertEquals(NTIMEOUTS + 1, proc.getTimeoutsCount()); }
long procId = ProcedureTestingUtility.submitAndWait(procExecutor, proc); assertFalse(procExecutor.isRunning()); ProcedureTestingUtility.assertProcNotYetCompleted(procExecutor, procId); assertFalse(procExecutor.isRunning()); ProcedureTestingUtility.assertProcNotYetCompleted(procExecutor, procId); assertFalse(procExecutor.isRunning()); ProcedureTestingUtility.assertProcNotYetCompleted(procExecutor, procId); assertFalse(procExecutor.isRunning()); ProcedureTestingUtility.assertProcNotYetCompleted(procExecutor, procId); assertFalse(procExecutor.isRunning()); ProcedureTestingUtility.assertIsAbortException(result);