/** * Remove the procedures that are marked as finished */ private synchronized void cleanupCompletedRestoreInMap() { ProcedureExecutor<MasterProcedureEnv> procExec = master.getMasterProcedureExecutor(); Iterator<Map.Entry<TableName, Long>> it = restoreTableToProcIdMap.entrySet().iterator(); while (it.hasNext()) { Map.Entry<TableName, Long> entry = it.next(); Long procId = entry.getValue(); if (procExec.isRunning() && procExec.isFinished(procId)) { it.remove(); } } }
/** * Verify if the restore of the specified table is in progress. * * @param tableName table under restore * @return <tt>true</tt> if there is a restore in progress of the specified table. */ private synchronized boolean isRestoringTable(final TableName tableName) { Long procId = this.restoreTableToProcIdMap.get(tableName); if (procId == null) { return false; } ProcedureExecutor<MasterProcedureEnv> procExec = master.getMasterProcedureExecutor(); if (procExec.isRunning() && !procExec.isFinished(procId)) { return true; } else { this.restoreTableToProcIdMap.remove(tableName); return false; } }
public boolean isRunning() { if (this.master == null || this.master.getMasterProcedureExecutor() == null) return false; return master.getMasterProcedureExecutor().isRunning(); }
@Override public void run() { while (executor.isRunning()) { final DelayedWithTimeout task = DelayedUtil.takeWithoutInterrupt(queue); if (task == null || task == DelayedUtil.DELAYED_POISON) { // the executor may be shutting down, // and the task is just the shutdown request continue; } LOG.trace("Executing {}", task); // execute the task if (task instanceof InlineChore) { execInlineChore((InlineChore) task); } else if (task instanceof DelayedProcedure) { execDelayedProcedure((DelayedProcedure<TEnvironment>) task); } else { LOG.error("CODE-BUG unknown timeout task type {}", task); } } }
@Override public Boolean evaluate() throws IOException { if (!procExec.isRunning()) { return true; } ProcedureState state = proc.getState(); if (state == ProcedureState.INITIALIZING || state == ProcedureState.RUNNABLE) { // under these states the procedure may have not been added to procExec yet, so do not // use isFinished to test whether it is finished, as this method will just check if the // procedure is in the running procedure list return false; } return procExec.isFinished(proc.getProcId()); } });
public static <TEnv> void waitProcedure(ProcedureExecutor<TEnv> procExecutor, long procId) { while (!procExecutor.isFinished(procId) && procExecutor.isRunning()) { Threads.sleepWithoutInterrupt(250); } }
private void resetProcExecutorTestingKillFlag() { final ProcedureExecutor<MasterProcedureEnv> procExec = getMasterProcedureExecutor(); ProcedureTestingUtility.setKillAndToggleBeforeStoreUpdate(procExec, false); assertTrue("expected executor to be running", procExec.isRunning()); }
private void resetProcExecutorTestingKillFlag() { final ProcedureExecutor<MasterProcedureEnv> procExec = getMasterProcedureExecutor(); ProcedureTestingUtility.setKillAndToggleBeforeStoreUpdate(procExec, false); assertTrue("expected executor to be running", procExec.isRunning()); }
private int waitThreadCount(final int expectedThreads) { while (procExecutor.isRunning()) { if (procExecutor.getWorkerThreadCount() == expectedThreads) { break; } LOG.debug("waiting for thread count=" + expectedThreads + " current=" + procExecutor.getWorkerThreadCount()); Threads.sleepWithoutInterrupt(250); } return procExecutor.getWorkerThreadCount(); }
protected void resetProcExecutorTestingKillFlag() { final ProcedureExecutor<MasterProcedureEnv> procExec = getMasterProcedureExecutor(); ProcedureTestingUtility.setKillAndToggleBeforeStoreUpdate(procExec, false); assertTrue("expected executor to be running", procExec.isRunning()); }
@After public void tearDown() throws IOException { ProcedureTestingUtility.setKillAndToggleBeforeStoreUpdate(procExecutor, false); assertTrue("expected executor to be running", procExecutor.isRunning()); procExecutor.stop(); procStore.stop(false); fs.delete(logDir, true); }
@Before public void setup() throws Exception { final ProcedureExecutor<MasterProcedureEnv> procExec = getMasterProcedureExecutor(); ProcedureTestingUtility.setKillAndToggleBeforeStoreUpdate(procExec, false); assertTrue("expected executor to be running", procExec.isRunning()); }
private void resetProcExecutorTestingKillFlag() { ProcedureExecutor<MasterProcedureEnv> procExec = UTIL.getHBaseCluster().getMaster().getMasterProcedureExecutor(); ProcedureTestingUtility.setKillAndToggleBeforeStoreUpdate(procExec, false); assertTrue("expected executor to be running", procExec.isRunning()); }
public static <TEnv> void testRecoveryAndDoubleExecution(final ProcedureExecutor<TEnv> procExec, final long procId, final boolean expectFailure, final Runnable customRestart) throws Exception { Procedure proc = procExec.getProcedure(procId); waitProcedure(procExec, procId); assertEquals(false, procExec.isRunning()); for (int i = 0; !procExec.isFinished(procId); ++i) { proc = procExec.getProcedure(procId); LOG.info("Restart " + i + " exec state: " + proc); if (customRestart != null) { customRestart.run(); } else { restart(procExec); } waitProcedure(procExec, procId); } assertEquals(true, procExec.isRunning()); if (expectFailure) { assertProcFailed(procExec, procId); } else { assertProcNotFailed(procExec, procId); } }
/** * Execute the procedure up to "lastStep" and then the ProcedureExecutor * is restarted and an abort() is injected. * If the procedure implement abort() this should result in rollback being triggered. * At the end of this call the procedure should be finished and rolledback. * This method assert on the procedure being terminated with an AbortException. */ public static void testRollbackRetriableFailure( final ProcedureExecutor<MasterProcedureEnv> procExec, final long procId, final int lastStep) throws Exception { // Execute up to last step testRecoveryAndDoubleExecution(procExec, procId, lastStep, false); // execute the rollback testRestartWithAbort(procExec, procId); assertEquals(true, procExec.isRunning()); ProcedureTestingUtility.assertIsAbortException(procExec.getResult(procId)); }
@Test public void testMultiStepProcRecovery() throws Exception { // Step 0 - kill Procedure proc = new TestMultiStepProcedure(); long procId = ProcedureTestingUtility.submitAndWait(procExecutor, proc); assertFalse(procExecutor.isRunning()); // Step 0 exec && Step 1 - kill restart(); waitProcedure(procId); ProcedureTestingUtility.assertProcNotYetCompleted(procExecutor, procId); assertFalse(procExecutor.isRunning()); // Step 1 exec && step 2 - kill restart(); waitProcedure(procId); ProcedureTestingUtility.assertProcNotYetCompleted(procExecutor, procId); assertFalse(procExecutor.isRunning()); // Step 2 exec restart(); waitProcedure(procId); assertTrue(procExecutor.isRunning()); // The procedure is completed Procedure<?> result = procExecutor.getResult(procId); ProcedureTestingUtility.assertProcNotFailed(result); }
@After public void tearDown() throws Exception { assertTrue("expected executor to be running", getMasterProcedureExecutor().isRunning()); ProcedureTestingUtility.setKillAndToggleBeforeStoreUpdate(getMasterProcedureExecutor(), false); for (HTableDescriptor htd: UTIL.getAdmin().listTables()) { LOG.info("Tear down, remove table=" + htd.getTableName()); UTIL.deleteTable(htd.getTableName()); } }
private void testRemoteLockRecovery(LockRequest lock) throws Exception { ProcedureTestingUtility.setKillAndToggleBeforeStoreUpdate(procExec, true); final long procId = queueLock(lock); assertTrue(awaitForLocked(procId, 2000)); // wait for proc Executor to die, then restart it and wait for Lock Procedure to get started. ProcedureTestingUtility.waitProcedure(procExec, procId); assertEquals(false, procExec.isRunning()); ProcedureTestingUtility.setKillAndToggleBeforeStoreUpdate(procExec, false); ProcedureTestingUtility.restart(procExec); while (!procExec.isStarted(procId)) { Thread.sleep(250); } assertEquals(true, procExec.isRunning()); // After recovery, remote locks should reacquire locks and function normally. assertTrue(awaitForLocked(procId, 2000)); Thread.sleep(HEARTBEAT_TIMEOUT/2); sendHeartbeatAndCheckLocked(procId, true); Thread.sleep(HEARTBEAT_TIMEOUT/2); sendHeartbeatAndCheckLocked(procId, true); Thread.sleep(2 * HEARTBEAT_TIMEOUT + HEARTBEAT_TIMEOUT/2); sendHeartbeatAndCheckLocked(procId, false); ProcedureTestingUtility.waitProcedure(procExec, procId); ProcedureTestingUtility.assertProcNotFailed(procExec, procId); }
@Test public void testLocalMasterLockRecovery() throws Exception { ProcedureTestingUtility.setKillAndToggleBeforeStoreUpdate(procExec, true); CountDownLatch latch = new CountDownLatch(1); LockProcedure lockProc = new LockProcedure(UTIL.getConfiguration(), TableName.valueOf("table"), LockType.EXCLUSIVE, "desc", latch); procExec.submitProcedure(lockProc); assertTrue(latch.await(2000, TimeUnit.MILLISECONDS)); // wait for proc Executor to die, then restart it and wait for Lock Procedure to get started. ProcedureTestingUtility.waitProcedure(procExec, lockProc.getProcId()); assertEquals(false, procExec.isRunning()); ProcedureTestingUtility.setKillAndToggleBeforeStoreUpdate(procExec, false); // remove zk lock node otherwise recovered lock will keep waiting on it. ProcedureTestingUtility.restart(procExec); while (!procExec.isStarted(lockProc.getProcId())) { Thread.sleep(250); } assertEquals(true, procExec.isRunning()); ProcedureTestingUtility.waitProcedure(procExec, lockProc.getProcId()); Procedure<?> result = procExec.getResultOrProcedure(lockProc.getProcId()); assertTrue(result != null && !result.isFailed()); ProcedureTestingUtility.assertProcNotFailed(procExec, lockProc.getProcId()); } }
@Test public void testSingleStepProcRecovery() throws Exception { Procedure proc = new TestSingleStepProcedure(); procExecutor.testing.killBeforeStoreUpdate = true; long procId = ProcedureTestingUtility.submitAndWait(procExecutor, proc); assertFalse(procExecutor.isRunning()); procExecutor.testing.killBeforeStoreUpdate = false; // Restart and verify that the procedures restart long restartTs = EnvironmentEdgeManager.currentTime(); restart(); waitProcedure(procId); Procedure<?> result = procExecutor.getResult(procId); assertTrue(result.getLastUpdate() > restartTs); ProcedureTestingUtility.assertProcNotFailed(result); assertEquals(1, Bytes.toInt(result.getResult())); long resultTs = result.getLastUpdate(); // Verify that after another restart the result is still there restart(); result = procExecutor.getResult(procId); ProcedureTestingUtility.assertProcNotFailed(result); assertEquals(resultTs, result.getLastUpdate()); assertEquals(1, Bytes.toInt(result.getResult())); }