/** * Remove the procedures that are marked as finished */ private synchronized void cleanupCompletedRestoreInMap() { ProcedureExecutor<MasterProcedureEnv> procExec = master.getMasterProcedureExecutor(); Iterator<Map.Entry<TableName, Long>> it = restoreTableToProcIdMap.entrySet().iterator(); while (it.hasNext()) { Map.Entry<TableName, Long> entry = it.next(); Long procId = entry.getValue(); if (procExec.isRunning() && procExec.isFinished(procId)) { it.remove(); } } }
/** * Verify if the restore of the specified table is in progress. * * @param tableName table under restore * @return <tt>true</tt> if there is a restore in progress of the specified table. */ private synchronized boolean isRestoringTable(final TableName tableName) { Long procId = this.restoreTableToProcIdMap.get(tableName); if (procId == null) { return false; } ProcedureExecutor<MasterProcedureEnv> procExec = master.getMasterProcedureExecutor(); if (procExec.isRunning() && !procExec.isFinished(procId)) { return true; } else { this.restoreTableToProcIdMap.remove(tableName); return false; } }
@Override public Boolean evaluate() throws IOException { if (!procExec.isRunning()) { return true; } ProcedureState state = proc.getState(); if (state == ProcedureState.INITIALIZING || state == ProcedureState.RUNNABLE) { // under these states the procedure may have not been added to procExec yet, so do not // use isFinished to test whether it is finished, as this method will just check if the // procedure is in the running procedure list return false; } return procExec.isFinished(proc.getProcId()); } });
public static <TEnv> void waitProcedure(ProcedureExecutor<TEnv> procExecutor, long procId) { while (!procExecutor.isFinished(procId) && procExecutor.isRunning()) { Threads.sleepWithoutInterrupt(250); } }
@Test public void testTableProcedureSubProcedureDeadLock() throws Exception { // the shared procedure will also schedule a shared procedure, but after the exclusive procedure long procId1 = procExec.submitProcedure(new TableShardParentProcedure()); long procId2 = procExec.submitProcedure(new TableExclusiveProcedure()); procExec.startWorkers(); UTIL.waitFor(10000, () -> procExec.getProcedures().stream().anyMatch(p -> p instanceof TableSharedProcedure)); procExec.getProcedures().stream().filter(p -> p instanceof TableSharedProcedure) .map(p -> (TableSharedProcedure) p).forEach(p -> p.latch.release()); ((TableExclusiveProcedure) procExec.getProcedure(procId2)).latch.release(); UTIL.waitFor(10000, () -> procExec.isFinished(procId1)); UTIL.waitFor(10000, () -> procExec.isFinished(procId2)); } }
public static <TEnv> void assertProcNotYetCompleted(ProcedureExecutor<TEnv> procExecutor, long procId) { assertFalse("expected a running proc", procExecutor.isFinished(procId)); assertEquals(null, procExecutor.getResult(procId)); }
@Test public void testArrayIndexOutOfBounds() { ParentProcedure proc = new ParentProcedure(); long procId = procExec.submitProcedure(proc); long noopProcId = -1L; // make sure that the sub procedure will have a new BitSetNode for (int i = 0; i < Long.SIZE - 2; i++) { noopProcId = procExec.submitProcedure(new NoopProcedure<>()); } final long lastNoopProcId = noopProcId; UTIL.waitFor(30000, () -> procExec.isFinished(lastNoopProcId)); proc.latch.countDown(); UTIL.waitFor(10000, () -> procExec.isFinished(procId)); } }
private void waitOnPids(List<Long> pids) { for (Long pid: pids) { while (!TEST_UTIL.getHBaseCluster().getMaster().getMasterProcedureExecutor(). isFinished(pid)) { Threads.sleep(100); } } } }
private void assertProcFailed(long procId) { assertTrue("expected completed proc", procExecutor.isFinished(procId)); Procedure<?> result = procExecutor.getResult(procId); assertEquals(true, result.isFailed()); LOG.info(result.getException().getMessage()); }
@Test public void testChildLoadWithSteppedRestart() throws Exception { procEnv.toggleKillBeforeStoreUpdate = true; TestRootProcedure proc = new TestRootProcedure(); long procId = ProcedureTestingUtility.submitAndWait(procExecutor, proc); int restartCount = 0; while (!procExecutor.isFinished(procId)) { ProcedureTestingUtility.restart(procExecutor); ProcedureTestingUtility.waitProcedure(procExecutor, proc); restartCount++; } assertEquals(3, restartCount); assertTrue("expected completed proc", procExecutor.isFinished(procId)); ProcedureTestingUtility.assertProcNotFailed(procExecutor, procId); }
/** * Test the state setting that happens after store to WAL; in particular the bit where we * set the parent runnable again after its children have all completed successfully. * See HBASE-20978. */ @Test public void testChildLoadWithRestartAfterChildSuccess() throws Exception { procEnv.toggleKillAfterStoreUpdate = true; TestRootProcedure proc = new TestRootProcedure(); long procId = ProcedureTestingUtility.submitAndWait(procExecutor, proc); int restartCount = 0; while (!procExecutor.isFinished(procId)) { ProcedureTestingUtility.restart(procExecutor); ProcedureTestingUtility.waitProcedure(procExecutor, proc); restartCount++; } assertEquals(4, restartCount); assertTrue("expected completed proc", procExecutor.isFinished(procId)); ProcedureTestingUtility.assertProcNotFailed(procExecutor, procId); }
@Test public void testTableProcedureDeadLockAfterRestarting() throws Exception { // let the shared procedure run first, but let it have a greater procId so when loading it will // be loaded at last. long procId1 = procExec.submitProcedure(new TableSharedProcedureWithId()); long procId2 = procExec.submitProcedure(new TableExclusiveProcedureWithId()); procExec.startWorkers(); UTIL.waitFor(10000, () -> ((TableSharedProcedure) procExec.getProcedure(procId1)).latch.hasQueuedThreads()); ProcedureTestingUtility.restart(procExec); ((TableSharedProcedure) procExec.getProcedure(procId1)).latch.release(); ((TableExclusiveProcedure) procExec.getProcedure(procId2)).latch.release(); UTIL.waitFor(10000, () -> procExec.isFinished(procId1)); UTIL.waitFor(10000, () -> procExec.isFinished(procId2)); }
@Test public void testChildLoad() throws Exception { procEnv.toggleKillBeforeStoreUpdate = false; TestRootProcedure proc = new TestRootProcedure(); long procId = ProcedureTestingUtility.submitAndWait(procExecutor, proc); ProcedureTestingUtility.restart(procExecutor); ProcedureTestingUtility.waitProcedure(procExecutor, proc); assertTrue("expected completed proc", procExecutor.isFinished(procId)); ProcedureTestingUtility.assertProcNotFailed(procExecutor, procId); }
@Test public void testCompletedProcedure() throws InterruptedException, IOException { long procId = EXEC.submitProcedure(new ExchangeProcedure()); EXCHANGER.exchange(Boolean.FALSE); UTIL.waitFor(10000, () -> EXEC.isFinished(procId)); for (int i = 0; i < WAL_COUNT - 1; i++) { assertTrue(STORE.rollWriterForTesting()); // The exchange procedure is completed but still not deleted yet so we can not delete the // oldest wal file long pid = EXEC.submitProcedure(new NoopNoAckProcedure()); assertEquals(2 + i, STORE.getActiveLogs().size()); UTIL.waitFor(10000, () -> EXEC.isFinished(pid)); } // Only the exchange procedure can not be deleted UTIL.waitFor(10000, () -> EXEC.getCompletedSize() == 1); STORE.rollWriterForTesting(); UTIL.waitFor(10000, () -> STORE.getActiveLogs().size() <= 1); } }
@Test public void testChildRollbackLoadWithSteppedRestart() throws Exception { procEnv.toggleKillBeforeStoreUpdate = true; procEnv.triggerRollbackOnChild = true; TestRootProcedure proc = new TestRootProcedure(); long procId = ProcedureTestingUtility.submitAndWait(procExecutor, proc); int restartCount = 0; while (!procExecutor.isFinished(procId)) { ProcedureTestingUtility.restart(procExecutor); ProcedureTestingUtility.waitProcedure(procExecutor, proc); restartCount++; } assertEquals(2, restartCount); assertProcFailed(procId); }
@Test public void test() throws Exception { TableDescriptor tableDescriptor = client.getDescriptor(); ProcedureExecutor<MasterProcedureEnv> executor = UTIL.getMiniHBaseCluster().getMaster() .getMasterProcedureExecutor(); MasterProcedureEnv env = executor.getEnvironment(); List<RegionInfo> regionInfos = admin.getRegions(TABLE_NAME); MergeTableRegionsProcedure mergeTableRegionsProcedure = new MergeTableRegionsProcedure( UTIL.getMiniHBaseCluster().getMaster().getMasterProcedureExecutor() .getEnvironment(), regionInfos.get(0), regionInfos.get(1)); ModifyTableProcedure modifyTableProcedure = new ModifyTableProcedure(env, tableDescriptor); long procModify = executor.submitProcedure(modifyTableProcedure); UTIL.waitFor(30000, () -> executor.getProcedures().stream() .filter(p -> p instanceof ModifyTableProcedure) .map(p -> (ModifyTableProcedure) p) .anyMatch(p -> TABLE_NAME.equals(p.getTableName()))); long proc = executor.submitProcedure(mergeTableRegionsProcedure); UTIL.waitFor(3000000, () -> UTIL.getMiniHBaseCluster().getMaster() .getMasterProcedureExecutor().isFinished(procModify)); Assert.assertEquals("Modify Table procedure should success!", ProcedureProtos.ProcedureState.SUCCESS, modifyTableProcedure.getState()); }
@Test public void test() throws Exception { TestProcedure proc = new TestProcedure(); long procId = procExecutor.submitProcedure(proc); htu.waitFor(30000, () -> proc.isWaiting()); ProcedureTestingUtility.restart(procExecutor); htu.waitFor(30000, () -> { Procedure<?> p = procExecutor.getProcedure(procId); return p.isWaiting() || p.isFinished(); }); assertFalse(procExecutor.isFinished(procId)); ProcedureTestingUtility.restart(procExecutor); htu.waitFor(30000, () -> procExecutor.isFinished(procId)); Procedure<ProcEnv> p = procExecutor.getResult(procId); assertTrue(p.isSuccess()); } }
@Test public void testDowngradeActiveToActive() throws IOException, InterruptedException { ProcedureExecutor<MasterProcedureEnv> procExec = UTIL.getMiniHBaseCluster().getMaster().getMasterProcedureExecutor(); // Test procedure: DOWNGRADE_ACTIVE ==> ACTIVE long procId = procExec.submitProcedure( new TestTransitPeerSyncReplicationStateProcedure("1", SyncReplicationState.ACTIVE)); // No retry for PRE_PEER_SYNC_REPLICATION_STATE_TRANSITION // SET_PEER_NEW_SYNC_REPLICATION_STATE assertBackoffIncrease(); // No retry for REFRESH_PEER_SYNC_REPLICATION_STATE_ON_RS_BEGIN // No retry for REOPEN_ALL_REGIONS_IN_PEER // TRANSIT_PEER_NEW_SYNC_REPLICATION_STATE assertBackoffIncrease(); // No retry for REFRESH_PEER_SYNC_REPLICATION_STATE_ON_RS_END // No retry for POST_PEER_SYNC_REPLICATION_STATE_TRANSITION UTIL.waitFor(30000, () -> procExec.isFinished(procId)); }
@Test public void test() throws Exception { List<RegionInfo> regionInfos = admin.getRegions(TABLE_NAME); MergeTableRegionsProcedure mergeTableRegionsProcedure = new MergeTableRegionsProcedure( UTIL.getMiniHBaseCluster().getMaster().getMasterProcedureExecutor() .getEnvironment(), regionInfos.get(0), regionInfos.get(1)); long procID = UTIL.getMiniHBaseCluster().getMaster().getMasterProcedureExecutor() .submitProcedure(mergeTableRegionsProcedure); mergeCommitArrive.await(); UTIL.getMiniHBaseCluster().stopMaster(0); UTIL.getMiniHBaseCluster().startMaster(); //wait until master initialized UTIL.waitFor(30000, () -> UTIL.getMiniHBaseCluster().getMaster() != null && UTIL .getMiniHBaseCluster().getMaster().isInitialized()); UTIL.waitFor(30000, () -> UTIL.getMiniHBaseCluster().getMaster() .getMasterProcedureExecutor().isFinished(procID)); Assert.assertTrue("Found region RIT, that's impossible!", UTIL.getMiniHBaseCluster().getMaster().getAssignmentManager() .getRegionsInTransition().size() == 0); }
@Test public void testDowngradeActiveToStandby() throws IOException, InterruptedException { ProcedureExecutor<MasterProcedureEnv> procExec = UTIL.getMiniHBaseCluster().getMaster().getMasterProcedureExecutor(); // Test procedure: DOWNGRADE_ACTIVE ==> ACTIVE long procId = procExec.submitProcedure( new TestTransitPeerSyncReplicationStateProcedure("2", SyncReplicationState.STANDBY)); // No retry for PRE_PEER_SYNC_REPLICATION_STATE_TRANSITION // SET_PEER_NEW_SYNC_REPLICATION_STATE assertBackoffIncrease(); // No retry for REFRESH_PEER_SYNC_REPLICATION_STATE_ON_RS_BEGIN // REMOVE_ALL_REPLICATION_QUEUES_IN_PEER assertBackoffIncrease(); // TRANSIT_PEER_NEW_SYNC_REPLICATION_STATE assertBackoffIncrease(); // No retry for REFRESH_PEER_SYNC_REPLICATION_STATE_ON_RS_END // CREATE_DIR_FOR_REMOTE_WAL assertBackoffIncrease(); // No retry for POST_PEER_SYNC_REPLICATION_STATE_TRANSITION UTIL.waitFor(30000, () -> procExec.isFinished(procId)); } }