public static <TEnv> void restart(ProcedureExecutor<TEnv> procExecutor, boolean avoidTestKillDuringRestart, boolean failOnCorrupted, Callable<Void> stopAction, Callable<Void> actionBeforeStartWorker, Callable<Void> startAction) throws Exception { restart(procExecutor, avoidTestKillDuringRestart, failOnCorrupted, stopAction, actionBeforeStartWorker, startAction, false, true); }
public static <TEnv> void restart(final ProcedureExecutor<TEnv> procExecutor, boolean abort) throws Exception { restart(procExecutor, false, true, null, null, null, abort, true); }
public static <TEnv> void restart(final ProcedureExecutor<TEnv> procExecutor, boolean abort, boolean startWorkers) throws Exception { restart(procExecutor, false, true, null, null, null, abort, startWorkers); }
public static <TEnv> void restart(final ProcedureExecutor<TEnv> procExecutor) throws Exception { restart(procExecutor, false, true, null, null, null, false, true); }
private void restart() throws Exception { dumpLogDirState(); ProcedureTestingUtility.restart(procExecutor); dumpLogDirState(); }
@Test public void testResetDeleteWhenBuildingHoldingCleanupTracker() throws Exception { createProcExecutor(); ExchangeProcedure proc1 = new ExchangeProcedure(); ExchangeProcedure proc2 = new ExchangeProcedure(); procExecutor.submitProcedure(proc1); long procId2 = procExecutor.submitProcedure(proc2); Thread.sleep(500); procStore.rollWriterForTesting(); proc1.exchanger.exchange(Boolean.TRUE); Thread.sleep(500); FileStatus[] walFiles = fs.listStatus(logDir); Arrays.sort(walFiles, (f1, f2) -> f1.getPath().getName().compareTo(f2.getPath().getName())); // corrupt the first proc wal file, so we will have a partial tracker for it after restarting corrupt(walFiles[0]); ProcedureTestingUtility.restart(procExecutor, false, true); // also update proc2, which means that all the procedures in the first proc wal have been // updated and it should be deleted. proc2 = (ExchangeProcedure) procExecutor.getProcedure(procId2); proc2.exchanger.exchange(Boolean.TRUE); htu.waitFor(10000, () -> !fs.exists(walFiles[0].getPath())); }
@Test public void testSingleStepReplayOrder() throws Exception { final int NUM_PROC_XTHREAD = 32; final int NUM_PROCS = NUM_THREADS * NUM_PROC_XTHREAD; // submit the procedures submitProcedures(NUM_THREADS, NUM_PROC_XTHREAD, TestSingleStepProcedure.class); while (procEnv.getExecId() < NUM_PROCS) { Thread.sleep(100); } // restart the executor and allow the procedures to run ProcedureTestingUtility.restart(procExecutor); // wait the execution of all the procedures and // assert that the execution order was sorted by procId ProcedureTestingUtility.waitNoProcedureRunning(procExecutor); procEnv.assertSortedExecList(NUM_PROCS); }
@Test public void testMultiStepReplayOrder() throws Exception { final int NUM_PROC_XTHREAD = 24; final int NUM_PROCS = NUM_THREADS * (NUM_PROC_XTHREAD * 2); // submit the procedures submitProcedures(NUM_THREADS, NUM_PROC_XTHREAD, TestTwoStepProcedure.class); while (procEnv.getExecId() < NUM_PROCS) { Thread.sleep(100); } // restart the executor and allow the procedures to run ProcedureTestingUtility.restart(procExecutor); // wait the execution of all the procedures and // assert that the execution order was sorted by procId ProcedureTestingUtility.waitNoProcedureRunning(procExecutor); procEnv.assertSortedExecList(NUM_PROCS); }
@Test public void testStuckProcedure() throws Exception { final StuckProcedure proc = new StuckProcedure(); long id = procExecutor.submitProcedure(proc); Thread.sleep(500); //bypass the procedure assertTrue(procExecutor.bypassProcedure(id, 1000, true, false)); //Since the procedure is stuck there, we need to restart the executor to recovery. ProcedureTestingUtility.restart(procExecutor); htu.waitFor(5000, () -> proc.isSuccess() && proc.isBypass()); LOG.info("{} finished", proc); }
@Test public void testChildRollbackLoad() throws Exception { procEnv.toggleKillBeforeStoreUpdate = false; procEnv.triggerRollbackOnChild = true; TestRootProcedure proc = new TestRootProcedure(); long procId = ProcedureTestingUtility.submitAndWait(procExecutor, proc); ProcedureTestingUtility.restart(procExecutor); ProcedureTestingUtility.waitProcedure(procExecutor, proc); assertProcFailed(procId); }
@Test public void testChildLoad() throws Exception { procEnv.toggleKillBeforeStoreUpdate = false; TestRootProcedure proc = new TestRootProcedure(); long procId = ProcedureTestingUtility.submitAndWait(procExecutor, proc); ProcedureTestingUtility.restart(procExecutor); ProcedureTestingUtility.waitProcedure(procExecutor, proc); assertTrue("expected completed proc", procExecutor.isFinished(procId)); ProcedureTestingUtility.assertProcNotFailed(procExecutor, procId); }
@Test public void testChildRollbackLoadWithSteppedRestart() throws Exception { procEnv.toggleKillBeforeStoreUpdate = true; procEnv.triggerRollbackOnChild = true; TestRootProcedure proc = new TestRootProcedure(); long procId = ProcedureTestingUtility.submitAndWait(procExecutor, proc); int restartCount = 0; while (!procExecutor.isFinished(procId)) { ProcedureTestingUtility.restart(procExecutor); ProcedureTestingUtility.waitProcedure(procExecutor, proc); restartCount++; } assertEquals(2, restartCount); assertProcFailed(procId); }
public static <TEnv> void testRecoveryAndDoubleExecution(final ProcedureExecutor<TEnv> procExec, final long procId, final boolean expectFailure, final Runnable customRestart) throws Exception { Procedure proc = procExec.getProcedure(procId); waitProcedure(procExec, procId); assertEquals(false, procExec.isRunning()); for (int i = 0; !procExec.isFinished(procId); ++i) { proc = procExec.getProcedure(procId); LOG.info("Restart " + i + " exec state: " + proc); if (customRestart != null) { customRestart.run(); } else { restart(procExec); } waitProcedure(procExec, procId); } assertEquals(true, procExec.isRunning()); if (expectFailure) { assertProcFailed(procExec, procId); } else { assertProcNotFailed(procExec, procId); } }
@Test public void testTableProcedureDeadLockAfterRestarting() throws Exception { // let the shared procedure run first, but let it have a greater procId so when loading it will // be loaded at last. long procId1 = procExec.submitProcedure(new TableSharedProcedureWithId()); long procId2 = procExec.submitProcedure(new TableExclusiveProcedureWithId()); procExec.startWorkers(); UTIL.waitFor(10000, () -> ((TableSharedProcedure) procExec.getProcedure(procId1)).latch.hasQueuedThreads()); ProcedureTestingUtility.restart(procExec); ((TableSharedProcedure) procExec.getProcedure(procId1)).latch.release(); ((TableExclusiveProcedure) procExec.getProcedure(procId2)).latch.release(); UTIL.waitFor(10000, () -> procExec.isFinished(procId1)); UTIL.waitFor(10000, () -> procExec.isFinished(procId2)); }
@Test public void test() throws Exception { TestProcedure proc = new TestProcedure(); long procId = procExecutor.submitProcedure(proc); htu.waitFor(30000, () -> proc.isWaiting()); ProcedureTestingUtility.restart(procExecutor); htu.waitFor(30000, () -> { Procedure<?> p = procExecutor.getProcedure(procId); return p.isWaiting() || p.isFinished(); }); assertFalse(procExecutor.isFinished(procId)); ProcedureTestingUtility.restart(procExecutor); htu.waitFor(30000, () -> procExecutor.isFinished(procId)); Procedure<ProcEnv> p = procExecutor.getResult(procId); assertTrue(p.isSuccess()); } }
@Test public void testChildLoadWithSteppedRestart() throws Exception { procEnv.toggleKillBeforeStoreUpdate = true; TestRootProcedure proc = new TestRootProcedure(); long procId = ProcedureTestingUtility.submitAndWait(procExecutor, proc); int restartCount = 0; while (!procExecutor.isFinished(procId)) { ProcedureTestingUtility.restart(procExecutor); ProcedureTestingUtility.waitProcedure(procExecutor, proc); restartCount++; } assertEquals(3, restartCount); assertTrue("expected completed proc", procExecutor.isFinished(procId)); ProcedureTestingUtility.assertProcNotFailed(procExecutor, procId); }
@Test public void testLocalMasterLockRecovery() throws Exception { ProcedureTestingUtility.setKillAndToggleBeforeStoreUpdate(procExec, true); CountDownLatch latch = new CountDownLatch(1); LockProcedure lockProc = new LockProcedure(UTIL.getConfiguration(), TableName.valueOf("table"), LockType.EXCLUSIVE, "desc", latch); procExec.submitProcedure(lockProc); assertTrue(latch.await(2000, TimeUnit.MILLISECONDS)); // wait for proc Executor to die, then restart it and wait for Lock Procedure to get started. ProcedureTestingUtility.waitProcedure(procExec, lockProc.getProcId()); assertEquals(false, procExec.isRunning()); ProcedureTestingUtility.setKillAndToggleBeforeStoreUpdate(procExec, false); // remove zk lock node otherwise recovered lock will keep waiting on it. ProcedureTestingUtility.restart(procExec); while (!procExec.isStarted(lockProc.getProcId())) { Thread.sleep(250); } assertEquals(true, procExec.isRunning()); ProcedureTestingUtility.waitProcedure(procExec, lockProc.getProcId()); Procedure<?> result = procExec.getResultOrProcedure(lockProc.getProcId()); assertTrue(result != null && !result.isFailed()); ProcedureTestingUtility.assertProcNotFailed(procExec, lockProc.getProcId()); } }
/** * Test the state setting that happens after store to WAL; in particular the bit where we * set the parent runnable again after its children have all completed successfully. * See HBASE-20978. */ @Test public void testChildLoadWithRestartAfterChildSuccess() throws Exception { procEnv.toggleKillAfterStoreUpdate = true; TestRootProcedure proc = new TestRootProcedure(); long procId = ProcedureTestingUtility.submitAndWait(procExecutor, proc); int restartCount = 0; while (!procExecutor.isFinished(procId)) { ProcedureTestingUtility.restart(procExecutor); ProcedureTestingUtility.waitProcedure(procExecutor, proc); restartCount++; } assertEquals(4, restartCount); assertTrue("expected completed proc", procExecutor.isFinished(procId)); ProcedureTestingUtility.assertProcNotFailed(procExecutor, procId); }
@Test public void testCompletedProcWithSameNonce() throws Exception { final long nonceGroup = 123; final long nonce = 2222; // register the nonce final NonceKey nonceKey = procExecutor.createNonceKey(nonceGroup, nonce); assertFalse(procExecutor.registerNonce(nonceKey) >= 0); // Submit a proc and wait for its completion Procedure proc = new TestSingleStepProcedure(); long procId = procExecutor.submitProcedure(proc, nonceKey); ProcedureTestingUtility.waitProcedure(procExecutor, procId); // Restart ProcedureTestingUtility.restart(procExecutor); ProcedureTestingUtility.waitProcedure(procExecutor, procId); // try to register a procedure with the same nonce // we should get back the old procId assertEquals(procId, procExecutor.registerNonce(nonceKey)); Procedure<?> result = procExecutor.getResult(procId); ProcedureTestingUtility.assertProcNotFailed(result); }
@Test public void testAbortProcedureInterruptedNotAllowed() throws Exception { final TableName tableName = TableName.valueOf(name.getMethodName()); final ProcedureExecutor<MasterProcedureEnv> procExec = getMasterProcedureExecutor(); RegionInfo[] regions = MasterProcedureTestingUtility.createTable(procExec, tableName, null, "f"); ProcedureTestingUtility.waitNoProcedureRunning(procExec); ProcedureTestingUtility.setKillAndToggleBeforeStoreUpdate(procExec, true); // Submit a procedure long procId = procExec.submitProcedure( new DisableTableProcedure(procExec.getEnvironment(), tableName, true)); // Wait for one step to complete ProcedureTestingUtility.waitProcedure(procExec, procId); // Set the mayInterruptIfRunning flag to false boolean abortResult = procExec.abort(procId, false); assertFalse(abortResult); ProcedureTestingUtility.setKillAndToggleBeforeStoreUpdate(procExec, false); ProcedureTestingUtility.restart(procExec); ProcedureTestingUtility.waitNoProcedureRunning(procExec); ProcedureTestingUtility.assertProcNotFailed(procExec, procId); // Validate the delete table procedure was not aborted MasterProcedureTestingUtility.validateTableIsDisabled( UTIL.getHBaseCluster().getMaster(), tableName); }