/** * Remove the procedures that are marked as finished */ private synchronized void cleanupCompletedRestoreInMap() { ProcedureExecutor<MasterProcedureEnv> procExec = master.getMasterProcedureExecutor(); Iterator<Map.Entry<TableName, Long>> it = restoreTableToProcIdMap.entrySet().iterator(); while (it.hasNext()) { Map.Entry<TableName, Long> entry = it.next(); Long procId = entry.getValue(); if (procExec.isRunning() && procExec.isFinished(procId)) { it.remove(); } } }
@Test public void testAbortNonExistProcedure() throws Exception { final ProcedureExecutor<MasterProcedureEnv> procExec = getMasterProcedureExecutor(); Random randomGenerator = new Random(); long procId; // Generate a non-existing procedure do { procId = randomGenerator.nextLong(); } while (procExec.getResult(procId) != null); boolean abortResult = procExec.abort(procId, true); assertFalse(abortResult); }
/** * Release the lock. * No-op if the lock was never acquired. */ public void release() { if (proc != null) { proc.unlock(master.getMasterProcedureExecutor().getEnvironment()); } proc = null; }
private int waitThreadCount(final int expectedThreads) { while (procExecutor.isRunning()) { if (procExecutor.getWorkerThreadCount() == expectedThreads) { break; } LOG.debug("waiting for thread count=" + expectedThreads + " current=" + procExecutor.getWorkerThreadCount()); Threads.sleepWithoutInterrupt(250); } return procExecutor.getWorkerThreadCount(); }
@Test public void testSetFailureResultForNonce() throws IOException { final long nonceGroup = 234; final long nonce = 55555; // check and register the request nonce final NonceKey nonceKey = procExecutor.createNonceKey(nonceGroup, nonce); assertFalse(procExecutor.registerNonce(nonceKey) >= 0); procExecutor.setFailureResultForNonce(nonceKey, "testProc", User.getCurrent(), new IOException("test failure")); final long procId = procExecutor.registerNonce(nonceKey); Procedure<?> result = procExecutor.getResult(procId); ProcedureTestingUtility.assertProcFailed(result); }
@Test public void testChildOnLastStepWithRollback() { procExecutor.getEnvironment().triggerChildRollback = true; long procId = procExecutor.submitProcedure(new TestSMProcedure()); ProcedureTestingUtility.waitProcedure(procExecutor, procId); assertEquals(3, procExecutor.getEnvironment().execCount.get()); assertEquals(3, procExecutor.getEnvironment().rollbackCount.get()); Throwable cause = ProcedureTestingUtility.assertProcFailed(procExecutor, procId); assertEquals(TEST_FAILURE_EXCEPTION, cause); }
@Test public void testTableProcedureSubProcedureDeadLock() throws Exception { // the shared procedure will also schedule a shared procedure, but after the exclusive procedure long procId1 = procExec.submitProcedure(new TableShardParentProcedure()); long procId2 = procExec.submitProcedure(new TableExclusiveProcedure()); procExec.startWorkers(); UTIL.waitFor(10000, () -> procExec.getProcedures().stream().anyMatch(p -> p instanceof TableSharedProcedure)); procExec.getProcedures().stream().filter(p -> p instanceof TableSharedProcedure) .map(p -> (TableSharedProcedure) p).forEach(p -> p.latch.release()); ((TableExclusiveProcedure) procExec.getProcedure(procId2)).latch.release(); UTIL.waitFor(10000, () -> procExec.isFinished(procId1)); UTIL.waitFor(10000, () -> procExec.isFinished(procId2)); } }
@Test public void testAbortProcedureSuccess() throws Exception { final TableName tableName = TableName.valueOf(name.getMethodName()); final ProcedureExecutor<MasterProcedureEnv> procExec = getMasterProcedureExecutor(); MasterProcedureTestingUtility.createTable(procExec, tableName, null, "f"); ProcedureTestingUtility.waitNoProcedureRunning(procExec); ProcedureTestingUtility.setKillAndToggleBeforeStoreUpdate(procExec, true); // Submit an abortable procedure long procId = procExec.submitProcedure( new DisableTableProcedure(procExec.getEnvironment(), tableName, false)); // Wait for one step to complete ProcedureTestingUtility.waitProcedure(procExec, procId); boolean abortResult = procExec.abort(procId, true); assertTrue(abortResult); MasterProcedureTestingUtility.testRestartWithAbort(procExec, procId); ProcedureTestingUtility.waitNoProcedureRunning(procExec); // Validate the disable table procedure was aborted successfully MasterProcedureTestingUtility.validateTableIsEnabled( UTIL.getHBaseCluster().getMaster(), tableName); }
assertFalse(procExecutor.isRunning()); waitProcedure(procId); ProcedureTestingUtility.assertProcNotYetCompleted(procExecutor, procId); assertFalse(procExecutor.isRunning()); waitProcedure(procId); ProcedureTestingUtility.assertProcNotYetCompleted(procExecutor, procId); assertFalse(procExecutor.isRunning()); assertTrue(procExecutor.abort(procId)); waitProcedure(procId); assertFalse(procExecutor.isRunning()); waitProcedure(procId); ProcedureTestingUtility.assertProcNotYetCompleted(procExecutor, procId); assertFalse(procExecutor.isRunning()); waitProcedure(procId); ProcedureTestingUtility.assertProcNotYetCompleted(procExecutor, procId); assertFalse(procExecutor.isRunning()); Procedure<?> result = procExecutor.getResult(procId); ProcedureTestingUtility.assertIsAbortException(result);
@Test public void test() throws Exception { TestProcedure proc = new TestProcedure(); long procId = procExecutor.submitProcedure(proc); htu.waitFor(30000, () -> proc.isWaiting()); ProcedureTestingUtility.restart(procExecutor); htu.waitFor(30000, () -> { Procedure<?> p = procExecutor.getProcedure(procId); return p.isWaiting() || p.isFinished(); }); assertFalse(procExecutor.isFinished(procId)); ProcedureTestingUtility.restart(procExecutor); htu.waitFor(30000, () -> procExecutor.isFinished(procId)); Procedure<ProcEnv> p = procExecutor.getResult(procId); assertTrue(p.isSuccess()); } }
public static <TEnv> void testRecoveryAndDoubleExecution(final ProcedureExecutor<TEnv> procExec, final long procId, final boolean expectFailure, final Runnable customRestart) throws Exception { Procedure proc = procExec.getProcedure(procId); waitProcedure(procExec, procId); assertEquals(false, procExec.isRunning()); for (int i = 0; !procExec.isFinished(procId); ++i) { proc = procExec.getProcedure(procId); LOG.info("Restart " + i + " exec state: " + proc); if (customRestart != null) { customRestart.run(); } else { restart(procExec); } waitProcedure(procExec, procId); } assertEquals(true, procExec.isRunning()); if (expectFailure) { assertProcFailed(procExec, procId); } else { assertProcNotFailed(procExec, procId); } }
@After public void tearDown() throws Exception { ProcedureTestingUtility.setKillAndToggleBeforeStoreUpdate(procExec, false); // Kill all running procedures. for (Procedure<?> proc : procExec.getProcedures()) { procExec.abort(proc.getProcId()); ProcedureTestingUtility.waitProcedure(procExec, proc); } assertEquals(0, procExec.getEnvironment().getProcedureScheduler().size()); }
public boolean isRunning() { if (this.master == null || this.master.getMasterProcedureExecutor() == null) return false; return master.getMasterProcedureExecutor().isRunning(); }
private void kill(String msg) { LOG.debug(msg); stop(); throw new RuntimeException(msg); }
@Test public void testDeleteSystemNamespace() throws Exception { final String namespaceName = NamespaceDescriptor.SYSTEM_NAMESPACE.getName(); final ProcedureExecutor<MasterProcedureEnv> procExec = getMasterProcedureExecutor(); long procId = procExec.submitProcedure( new DeleteNamespaceProcedure(procExec.getEnvironment(), namespaceName)); // Wait the completion ProcedureTestingUtility.waitProcedure(procExec, procId); Procedure<?> result = procExec.getResult(procId); assertTrue(result.isFailed()); LOG.debug("Delete namespace failed with exception: " + result.getException()); assertTrue(ProcedureTestingUtility.getExceptionCause(result) instanceof ConstraintException); }
public static RegionInfo[] createTable(final ProcedureExecutor<MasterProcedureEnv> procExec, final TableName tableName, final byte[][] splitKeys, String... family) throws IOException { TableDescriptor htd = createHTD(tableName, family); RegionInfo[] regions = ModifyRegionUtils.createRegionInfos(htd, splitKeys); long procId = ProcedureTestingUtility.submitAndWait(procExec, new CreateTableProcedure(procExec.getEnvironment(), htd, regions)); ProcedureTestingUtility.assertProcNotFailed(procExec.getResult(procId)); return regions; }
private void createNewExecutor(final Configuration conf, final int numThreads) throws Exception { procExecutor = new ProcedureExecutor<>(conf, procEnv, procStore); ProcedureTestingUtility.initAndStartWorkers(procExecutor, numThreads, true); }
public static <TEnv> void assertProcNotYetCompleted(ProcedureExecutor<TEnv> procExecutor, long procId) { assertFalse("expected a running proc", procExecutor.isFinished(procId)); assertEquals(null, procExecutor.getResult(procId)); }
/** * Execute the procedure up to "lastStep" and then the ProcedureExecutor * is restarted and an abort() is injected. * If the procedure implement abort() this should result in rollback being triggered. * At the end of this call the procedure should be finished and rolledback. * This method assert on the procedure being terminated with an AbortException. */ public static void testRollbackRetriableFailure( final ProcedureExecutor<MasterProcedureEnv> procExec, final long procId, final int lastStep) throws Exception { // Execute up to last step testRecoveryAndDoubleExecution(procExec, procId, lastStep, false); // execute the rollback testRestartWithAbort(procExec, procId); assertEquals(true, procExec.isRunning()); ProcedureTestingUtility.assertIsAbortException(procExec.getResult(procId)); }