@SuppressWarnings("unchecked") private RemoteProcedure<MasterProcedureEnv, ?> getRemoteProcedure(long procId) { Procedure<?> procedure = procedureExecutor.getProcedure(procId); if (procedure == null) { return null; } assert procedure instanceof RemoteProcedure; return (RemoteProcedure<MasterProcedureEnv, ?>) procedure; }
@Test public void testTableProcedureSubProcedureDeadLock() throws Exception { // the shared procedure will also schedule a shared procedure, but after the exclusive procedure long procId1 = procExec.submitProcedure(new TableShardParentProcedure()); long procId2 = procExec.submitProcedure(new TableExclusiveProcedure()); procExec.startWorkers(); UTIL.waitFor(10000, () -> procExec.getProcedures().stream().anyMatch(p -> p instanceof TableSharedProcedure)); procExec.getProcedures().stream().filter(p -> p instanceof TableSharedProcedure) .map(p -> (TableSharedProcedure) p).forEach(p -> p.latch.release()); ((TableExclusiveProcedure) procExec.getProcedure(procId2)).latch.release(); UTIL.waitFor(10000, () -> procExec.isFinished(procId1)); UTIL.waitFor(10000, () -> procExec.isFinished(procId2)); } }
public <T extends Procedure<TEnvironment>> T getProcedure(Class<T> clazz, long procId) { Procedure<TEnvironment> proc = getProcedure(procId); if (clazz.isInstance(proc)) { return clazz.cast(proc); } return null; }
@Test public void testProcedureShouldNotCleanOnLoad() throws Exception { createProcExecutor(); final RootProcedure proc = new RootProcedure(); long rootProc = procExecutor.submitProcedure(proc); LOG.info("Begin to execute " + rootProc); // wait until the child procedure arrival htu.waitFor(10000, () -> procExecutor.getProcedures().size() >= 2); SuspendProcedure suspendProcedure = (SuspendProcedure) procExecutor .getProcedures().get(1); // wait until the suspendProcedure executed suspendProcedure.latch.countDown(); Thread.sleep(100); // roll the procedure log LOG.info("Begin to roll log "); procStore.rollWriterForTesting(); LOG.info("finish to roll log "); Thread.sleep(500); LOG.info("begin to restart1 "); ProcedureTestingUtility.restart(procExecutor, true); LOG.info("finish to restart1 "); assertTrue(procExecutor.getProcedure(rootProc) != null); Thread.sleep(500); LOG.info("begin to restart2 "); ProcedureTestingUtility.restart(procExecutor, true); LOG.info("finish to restart2 "); assertTrue(procExecutor.getProcedure(rootProc) != null); }
public MasterProcedureEnv(final MasterServices master, final RSProcedureDispatcher remoteDispatcher) { this.master = master; this.procSched = new MasterProcedureScheduler( procId -> master.getMasterProcedureExecutor().getProcedure(procId)); this.remoteDispatcher = remoteDispatcher; }
public static void updateProgress(MasterProcedureEnv env, long parentId) { if (parentId == NO_PROC_ID) { return; } Procedure parentProcedure = env.getMasterServices().getMasterProcedureExecutor().getProcedure(parentId); if (parentProcedure != null && parentProcedure instanceof ServerCrashProcedure) { ((ServerCrashProcedure) parentProcedure).updateProgress(false); } } }
@Override public void remoteProcedureCompleted(long procId) { if (FAIL && getMasterProcedureExecutor() .getProcedure(procId) instanceof SyncReplicationReplayWALRemoteProcedure) { throw new RuntimeException("Inject error"); } super.remoteProcedureCompleted(procId); } }
private boolean isAborted() { boolean aborted = abort.get(); BaseTestStepProcedure proc = this; while (proc.hasParent() && !aborted) { proc = (BaseTestStepProcedure)procExecutor.getProcedure(proc.getParentProcId()); aborted = proc.isAborted(); } return aborted; } }
@Test public void testResetDeleteWhenBuildingHoldingCleanupTracker() throws Exception { createProcExecutor(); ExchangeProcedure proc1 = new ExchangeProcedure(); ExchangeProcedure proc2 = new ExchangeProcedure(); procExecutor.submitProcedure(proc1); long procId2 = procExecutor.submitProcedure(proc2); Thread.sleep(500); procStore.rollWriterForTesting(); proc1.exchanger.exchange(Boolean.TRUE); Thread.sleep(500); FileStatus[] walFiles = fs.listStatus(logDir); Arrays.sort(walFiles, (f1, f2) -> f1.getPath().getName().compareTo(f2.getPath().getName())); // corrupt the first proc wal file, so we will have a partial tracker for it after restarting corrupt(walFiles[0]); ProcedureTestingUtility.restart(procExecutor, false, true); // also update proc2, which means that all the procedures in the first proc wal have been // updated and it should be deleted. proc2 = (ExchangeProcedure) procExecutor.getProcedure(procId2); proc2.exchanger.exchange(Boolean.TRUE); htu.waitFor(10000, () -> !fs.exists(walFiles[0].getPath())); }
@Test public void testProcedureUpdatedShouldClean() throws Exception { createProcExecutor(); SuspendProcedure suspendProcedure = new SuspendProcedure(); long suspendProc = procExecutor.submitProcedure(suspendProcedure); LOG.info("Begin to execute " + suspendProc); suspendProcedure.latch.countDown(); Thread.sleep(500); LOG.info("begin to restart1 "); ProcedureTestingUtility.restart(procExecutor, true); LOG.info("finish to restart1 "); htu.waitFor(10000, () -> procExecutor.getProcedure(suspendProc) != null); // Wait until the suspendProc executed after restart suspendProcedure = (SuspendProcedure) procExecutor.getProcedure(suspendProc); suspendProcedure.latch.countDown(); Thread.sleep(500); // Should be 1 log since the suspendProcedure is updated in the new log assertTrue(procStore.getActiveLogs().size() == 1); // restart procExecutor LOG.info("begin to restart2"); // Restart the executor but do not start the workers. // Otherwise, the suspendProcedure will soon be executed and the oldest log // will be cleaned, leaving only the newest log. ProcedureTestingUtility.restart(procExecutor, true, false); LOG.info("finish to restart2"); // There should be two active logs assertTrue(procStore.getActiveLogs().size() == 2); procExecutor.startWorkers(); }
@Test public void testTableProcedureDeadLockAfterRestarting() throws Exception { // let the shared procedure run first, but let it have a greater procId so when loading it will // be loaded at last. long procId1 = procExec.submitProcedure(new TableSharedProcedureWithId()); long procId2 = procExec.submitProcedure(new TableExclusiveProcedureWithId()); procExec.startWorkers(); UTIL.waitFor(10000, () -> ((TableSharedProcedure) procExec.getProcedure(procId1)).latch.hasQueuedThreads()); ProcedureTestingUtility.restart(procExec); ((TableSharedProcedure) procExec.getProcedure(procId1)).latch.release(); ((TableExclusiveProcedure) procExec.getProcedure(procId2)).latch.release(); UTIL.waitFor(10000, () -> procExec.isFinished(procId1)); UTIL.waitFor(10000, () -> procExec.isFinished(procId2)); }
@Test public void testProcedureDeletedShouldClean() throws Exception { createProcExecutor(); WaitProcedure waitProcedure = new WaitProcedure(); long waitProce = procExecutor.submitProcedure(waitProcedure); LOG.info("Begin to execute " + waitProce); Thread.sleep(500); LOG.info("begin to restart1 "); ProcedureTestingUtility.restart(procExecutor, true); LOG.info("finish to restart1 "); htu.waitFor(10000, () -> procExecutor.getProcedure(waitProce) != null); // Wait until the suspendProc executed after restart waitProcedure = (WaitProcedure) procExecutor.getProcedure(waitProce); waitProcedure.latch.countDown(); Thread.sleep(500); // Should be 1 log since the suspendProcedure is updated in the new log assertTrue(procStore.getActiveLogs().size() == 1); // restart procExecutor LOG.info("begin to restart2"); // Restart the executor but do not start the workers. // Otherwise, the suspendProcedure will soon be executed and the oldest log // will be cleaned, leaving only the newest log. ProcedureTestingUtility.restart(procExecutor, true, false); LOG.info("finish to restart2"); // There should be two active logs assertTrue(procStore.getActiveLogs().size() == 2); procExecutor.startWorkers(); }
/** * @param keepAlive if false, release the lock. * @return true, if procedure is found and it has the lock; else false. */ public boolean lockHeartbeat(final long procId, final boolean keepAlive) throws IOException { final LockProcedure proc = master.getMasterProcedureExecutor() .getProcedure(LockProcedure.class, procId); if (proc == null) return false; master.getMasterCoprocessorHost().preLockHeartbeat(proc, keepAlive); proc.updateHeartBeat(); if (!keepAlive) { proc.unlock(master.getMasterProcedureExecutor().getEnvironment()); } master.getMasterCoprocessorHost().postLockHeartbeat(proc, keepAlive); return proc.isLocked(); } }
public static <TEnv> void testRecoveryAndDoubleExecution(final ProcedureExecutor<TEnv> procExec, final long procId, final boolean expectFailure, final Runnable customRestart) throws Exception { Procedure proc = procExec.getProcedure(procId); waitProcedure(procExec, procId); assertEquals(false, procExec.isRunning()); for (int i = 0; !procExec.isFinished(procId); ++i) { proc = procExec.getProcedure(procId); LOG.info("Restart " + i + " exec state: " + proc); if (customRestart != null) { customRestart.run(); } else { restart(procExec); } waitProcedure(procExec, procId); } assertEquals(true, procExec.isRunning()); if (expectFailure) { assertProcFailed(procExec, procId); } else { assertProcNotFailed(procExec, procId); } }
Procedure<?> proc = procExec.getProcedure(procId); int stepNum = proc instanceof StateMachineProcedure ? ((StateMachineProcedure) proc).getCurrentStateId() : 0; ProcedureTestingUtility.waitProcedure(procExec, procId); proc = procExec.getProcedure(procId); stepNum = proc instanceof StateMachineProcedure ? ((StateMachineProcedure) proc).getCurrentStateId() : stepNum + 1;
public static void testRecoveryAndDoubleExecution(final HBaseTestingUtility testUtil, final long procId, final int lastStepBeforeFailover) throws Exception { ProcedureExecutor<MasterProcedureEnv> procExec = testUtil.getHBaseCluster().getMaster().getMasterProcedureExecutor(); ProcedureTestingUtility.waitProcedure(procExec, procId); final Procedure proc = procExec.getProcedure(procId); for (int i = 0; i < lastStepBeforeFailover; ++i) { LOG.info("Restart "+ i +" exec state: " + proc); ProcedureTestingUtility.assertProcNotYetCompleted(procExec, procId); MasterProcedureTestingUtility.restartMasterProcedureExecutor(procExec); ProcedureTestingUtility.waitProcedure(procExec, procId); } ProcedureTestingUtility.assertProcNotYetCompleted(procExec, procId); LOG.info("Trigger master failover"); MasterProcedureTestingUtility.masterFailover(testUtil); procExec = testUtil.getHBaseCluster().getMaster().getMasterProcedureExecutor(); ProcedureTestingUtility.waitProcedure(procExec, procId); ProcedureTestingUtility.assertProcNotFailed(procExec, procId); }
assertEquals(false, procExec.isRunning()); for (int i = 0; !procExec.isFinished(procId); ++i) { LOG.info("Restart " + i + " exec state=" + procExec.getProcedure(procId)); if (hook != null) { assertTrue(hook.execute(i));
AssignmentTestingUtil.killRs(util, rsToKill); long procId = getSCPProcId(pExecutor); Procedure procA = pExecutor.getProcedure(procId); LOG.info("submit SCP procedureA"); util.waitFor(5000, () -> procA.hasLock());
@Test public void test() throws Exception { TestProcedure proc = new TestProcedure(); long procId = procExecutor.submitProcedure(proc); htu.waitFor(30000, () -> proc.isWaiting()); ProcedureTestingUtility.restart(procExecutor); htu.waitFor(30000, () -> { Procedure<?> p = procExecutor.getProcedure(procId); return p.isWaiting() || p.isFinished(); }); assertFalse(procExecutor.isFinished(procId)); ProcedureTestingUtility.restart(procExecutor); htu.waitFor(30000, () -> procExecutor.isFinished(procId)); Procedure<ProcEnv> p = procExecutor.getResult(procId); assertTrue(p.isSuccess()); } }
UTIL.waitFor(30000, () -> master.isActiveMaster() && master.isInitialized()); TestProcedure proc = (TestProcedure) master.getMasterProcedureExecutor().getProcedure(procId); assertFalse(proc.isFinished()); FINISH_PROC = true;