@Test public void testSingleStepReplayOrder() throws Exception { final int NUM_PROC_XTHREAD = 32; final int NUM_PROCS = NUM_THREADS * NUM_PROC_XTHREAD; // submit the procedures submitProcedures(NUM_THREADS, NUM_PROC_XTHREAD, TestSingleStepProcedure.class); while (procEnv.getExecId() < NUM_PROCS) { Thread.sleep(100); } // restart the executor and allow the procedures to run ProcedureTestingUtility.restart(procExecutor); // wait the execution of all the procedures and // assert that the execution order was sorted by procId ProcedureTestingUtility.waitNoProcedureRunning(procExecutor); procEnv.assertSortedExecList(NUM_PROCS); }
@Test public void testMultiStepReplayOrder() throws Exception { final int NUM_PROC_XTHREAD = 24; final int NUM_PROCS = NUM_THREADS * (NUM_PROC_XTHREAD * 2); // submit the procedures submitProcedures(NUM_THREADS, NUM_PROC_XTHREAD, TestTwoStepProcedure.class); while (procEnv.getExecId() < NUM_PROCS) { Thread.sleep(100); } // restart the executor and allow the procedures to run ProcedureTestingUtility.restart(procExecutor); // wait the execution of all the procedures and // assert that the execution order was sorted by procId ProcedureTestingUtility.waitNoProcedureRunning(procExecutor); procEnv.assertSortedExecList(NUM_PROCS); }
@Test public void testRollbackAndDoubleExecution() throws Exception { final TableName tableName = TableName.valueOf(name.getMethodName()); final ProcedureExecutor<MasterProcedureEnv> procExec = getMasterProcedureExecutor(); final byte[][] splitKeys = new byte[][] { Bytes.toBytes("a"), Bytes.toBytes("b"), Bytes.toBytes("c") }; MasterProcedureTestingUtility.createTable(procExec, tableName, splitKeys, "f1", "f2"); UTIL.getAdmin().disableTable(tableName); ProcedureTestingUtility.waitNoProcedureRunning(procExec); ProcedureTestingUtility.setKillAndToggleBeforeStoreUpdate(procExec, true); // Start the Enable procedure && kill the executor long procId = procExec.submitProcedure(new EnableTableProcedure(procExec.getEnvironment(), tableName)); int lastStep = 3; // fail before ENABLE_TABLE_SET_ENABLING_TABLE_STATE MasterProcedureTestingUtility.testRollbackAndDoubleExecution(procExec, procId, lastStep); MasterProcedureTestingUtility.validateTableIsDisabled(getMaster(), tableName); } }
@Test public void testRecoveryAndDoubleExecution() throws Exception { final TableName tableName = TableName.valueOf("testRecoveryAndDoubleExecution"); final ProcedureExecutor<MasterProcedureEnv> procExec = getMasterProcedureExecutor(); List<RegionInfo> tableRegions = createTable(tableName); ProcedureTestingUtility.waitNoProcedureRunning(procExec); ProcedureTestingUtility.setKillIfHasParent(procExec, false); ProcedureTestingUtility.setKillAndToggleBeforeStoreUpdate(procExec, true); RegionInfo[] regionsToMerge = new RegionInfo[2]; regionsToMerge[0] = tableRegions.get(0); regionsToMerge[1] = tableRegions.get(1); long procId = procExec.submitProcedure( new MergeTableRegionsProcedure(procExec.getEnvironment(), regionsToMerge, true)); // Restart the executor and execute the step twice MasterProcedureTestingUtility.testRecoveryAndDoubleExecution(procExec, procId); ProcedureTestingUtility.assertProcNotFailed(procExec, procId); assertRegionCount(tableName, initialRegionCount - 1); }
@Test public void testRollbackAndDoubleExecution() throws Exception { final TableName tableName = TableName.valueOf("testRollbackAndDoubleExecution"); final ProcedureExecutor<MasterProcedureEnv> procExec = getMasterProcedureExecutor(); List<RegionInfo> tableRegions = createTable(tableName); ProcedureTestingUtility.waitNoProcedureRunning(procExec); ProcedureTestingUtility.setKillAndToggleBeforeStoreUpdate(procExec, true); RegionInfo[] regionsToMerge = new RegionInfo[2]; regionsToMerge[0] = tableRegions.get(0); regionsToMerge[1] = tableRegions.get(1); long procId = procExec.submitProcedure( new MergeTableRegionsProcedure(procExec.getEnvironment(), regionsToMerge, true)); // Failing before MERGE_TABLE_REGIONS_UPDATE_META we should trigger the rollback // NOTE: the 8 (number of MERGE_TABLE_REGIONS_UPDATE_META step) is // hardcoded, so you have to look at this test at least once when you add a new step. int lastStep = 8; MasterProcedureTestingUtility.testRollbackAndDoubleExecution(procExec, procId, lastStep, true); assertEquals(initialRegionCount, UTIL.getAdmin().getRegions(tableName).size()); UTIL.waitUntilAllRegionsAssigned(tableName); List<HRegion> regions = UTIL.getMiniHBaseCluster().getRegions(tableName); assertEquals(initialRegionCount, regions.size()); }
@Test public void testRecoveryAndDoubleExecution() throws Exception { final TableName tableName = TableName.valueOf(name.getMethodName()); final ProcedureExecutor<MasterProcedureEnv> procExec = getMasterProcedureExecutor(); final byte[][] splitKeys = new byte[][] { Bytes.toBytes("a"), Bytes.toBytes("b"), Bytes.toBytes("c") }; MasterProcedureTestingUtility.createTable(procExec, tableName, splitKeys, "f1", "f2"); UTIL.getAdmin().disableTable(tableName); ProcedureTestingUtility.waitNoProcedureRunning(procExec); ProcedureTestingUtility.setKillIfHasParent(procExec, false); ProcedureTestingUtility.setKillAndToggleBeforeStoreUpdate(procExec, true); // Start the Enable procedure && kill the executor long procId = procExec.submitProcedure(new EnableTableProcedure(procExec.getEnvironment(), tableName)); // Restart the executor and execute the step twice MasterProcedureTestingUtility.testRecoveryAndDoubleExecution(procExec, procId); MasterProcedureTestingUtility.validateTableIsEnabled(getMaster(), tableName); }
@Test public void testMergeWithoutPONR() throws Exception { final TableName tableName = TableName.valueOf("testMergeWithoutPONR"); final ProcedureExecutor<MasterProcedureEnv> procExec = getMasterProcedureExecutor(); List<RegionInfo> tableRegions = createTable(tableName); ProcedureTestingUtility.waitNoProcedureRunning(procExec); ProcedureTestingUtility.setKillAndToggleBeforeStoreUpdate(procExec, true); RegionInfo[] regionsToMerge = new RegionInfo[2]; regionsToMerge[0] = tableRegions.get(0); regionsToMerge[1] = tableRegions.get(1); long procId = procExec.submitProcedure( new MergeTableRegionsProcedure(procExec.getEnvironment(), regionsToMerge, true)); // Execute until step 9 of split procedure // NOTE: step 9 is after step MERGE_TABLE_REGIONS_UPDATE_META MasterProcedureTestingUtility.testRecoveryAndDoubleExecution(procExec, procId, 9, false); // Unset Toggle Kill and make ProcExec work correctly ProcedureTestingUtility.setKillAndToggleBeforeStoreUpdate(procExec, false); MasterProcedureTestingUtility.restartMasterProcedureExecutor(procExec); ProcedureTestingUtility.waitProcedure(procExec, procId); assertRegionCount(tableName, initialRegionCount - 1); }
@Test public void testAbortProcedureSuccess() throws Exception { final TableName tableName = TableName.valueOf(name.getMethodName()); final ProcedureExecutor<MasterProcedureEnv> procExec = getMasterProcedureExecutor(); MasterProcedureTestingUtility.createTable(procExec, tableName, null, "f"); ProcedureTestingUtility.waitNoProcedureRunning(procExec); ProcedureTestingUtility.setKillAndToggleBeforeStoreUpdate(procExec, true); // Submit an abortable procedure long procId = procExec.submitProcedure( new DisableTableProcedure(procExec.getEnvironment(), tableName, false)); // Wait for one step to complete ProcedureTestingUtility.waitProcedure(procExec, procId); boolean abortResult = procExec.abort(procId, true); assertTrue(abortResult); MasterProcedureTestingUtility.testRestartWithAbort(procExec, procId); ProcedureTestingUtility.waitNoProcedureRunning(procExec); // Validate the disable table procedure was aborted successfully MasterProcedureTestingUtility.validateTableIsEnabled( UTIL.getHBaseCluster().getMaster(), tableName); }
@Test public void testAbortProcedureInterruptedNotAllowed() throws Exception { final TableName tableName = TableName.valueOf(name.getMethodName()); final ProcedureExecutor<MasterProcedureEnv> procExec = getMasterProcedureExecutor(); RegionInfo[] regions = MasterProcedureTestingUtility.createTable(procExec, tableName, null, "f"); ProcedureTestingUtility.waitNoProcedureRunning(procExec); ProcedureTestingUtility.setKillAndToggleBeforeStoreUpdate(procExec, true); // Submit a procedure long procId = procExec.submitProcedure( new DisableTableProcedure(procExec.getEnvironment(), tableName, true)); // Wait for one step to complete ProcedureTestingUtility.waitProcedure(procExec, procId); // Set the mayInterruptIfRunning flag to false boolean abortResult = procExec.abort(procId, false); assertFalse(abortResult); ProcedureTestingUtility.setKillAndToggleBeforeStoreUpdate(procExec, false); ProcedureTestingUtility.restart(procExec); ProcedureTestingUtility.waitNoProcedureRunning(procExec); ProcedureTestingUtility.assertProcNotFailed(procExec, procId); // Validate the delete table procedure was not aborted MasterProcedureTestingUtility.validateTableIsDisabled( UTIL.getHBaseCluster().getMaster(), tableName); }
@Test public void testRollbackAndDoubleExecution() throws Exception { final ProcedureExecutor<MasterProcedureEnv> procExec = getMasterProcedureExecutor(); final TableName clonedTableName = TableName.valueOf("testRollbackAndDoubleExecution"); final TableDescriptor htd = createTableDescriptor(clonedTableName, CF); // take the snapshot SnapshotProtos.SnapshotDescription snapshotDesc = getSnapshot(); ProcedureTestingUtility.waitNoProcedureRunning(procExec); ProcedureTestingUtility.setKillAndToggleBeforeStoreUpdate(procExec, true); // Start the Clone snapshot procedure && kill the executor long procId = procExec.submitProcedure( new CloneSnapshotProcedure(procExec.getEnvironment(), htd, snapshotDesc)); int lastStep = 2; // failing before CLONE_SNAPSHOT_WRITE_FS_LAYOUT MasterProcedureTestingUtility.testRollbackAndDoubleExecution(procExec, procId, lastStep); MasterProcedureTestingUtility.validateTableDeletion( UTIL.getHBaseCluster().getMaster(), clonedTableName); } }
@Test public void testRecoveryAndDoubleExecution() throws Exception { final String namespaceName = "testRecoveryAndDoubleExecution"; final ProcedureExecutor<MasterProcedureEnv> procExec = getMasterProcedureExecutor(); createNamespaceForTesting(namespaceName); ProcedureTestingUtility.waitNoProcedureRunning(procExec); ProcedureTestingUtility.setKillAndToggleBeforeStoreUpdate(procExec, true); // Start the DeleteNamespace procedure && kill the executor long procId = procExec.submitProcedure( new DeleteNamespaceProcedure(procExec.getEnvironment(), namespaceName)); // Restart the executor and execute the step twice MasterProcedureTestingUtility.testRecoveryAndDoubleExecution(procExec, procId); // Validate the deletion of namespace ProcedureTestingUtility.assertProcNotFailed(procExec, procId); validateNamespaceNotExist(namespaceName); }
@Test public void testRecoveryAndDoubleExecution() throws Exception { final TableName tableName = TableName.valueOf(name.getMethodName()); // create the table byte[][] splitKeys = null; RegionInfo[] regions = MasterProcedureTestingUtility.createTable( getMasterProcedureExecutor(), tableName, splitKeys, "f1", "f2"); UTIL.getAdmin().disableTable(tableName); final ProcedureExecutor<MasterProcedureEnv> procExec = getMasterProcedureExecutor(); ProcedureTestingUtility.waitNoProcedureRunning(procExec); ProcedureTestingUtility.setKillAndToggleBeforeStoreUpdate(procExec, true); // Start the Delete procedure && kill the executor long procId = procExec.submitProcedure( new DeleteTableProcedure(procExec.getEnvironment(), tableName)); // Restart the executor and execute the step twice MasterProcedureTestingUtility.testRecoveryAndDoubleExecution(procExec, procId); MasterProcedureTestingUtility.validateTableDeletion(getMaster(), tableName); } }
@Test public void testAbortProcedureFailure() throws Exception { final TableName tableName = TableName.valueOf(name.getMethodName()); final ProcedureExecutor<MasterProcedureEnv> procExec = getMasterProcedureExecutor(); RegionInfo[] regions = MasterProcedureTestingUtility.createTable(procExec, tableName, null, "f"); UTIL.getAdmin().disableTable(tableName); ProcedureTestingUtility.waitNoProcedureRunning(procExec); ProcedureTestingUtility.setKillAndToggleBeforeStoreUpdate(procExec, true); // Submit an un-abortable procedure long procId = procExec.submitProcedure( new DeleteTableProcedure(procExec.getEnvironment(), tableName)); // Wait for a couple of steps to complete (first step "prepare" is abortable) ProcedureTestingUtility.waitProcedure(procExec, procId); for (int i = 0; i < 2; ++i) { ProcedureTestingUtility.assertProcNotYetCompleted(procExec, procId); ProcedureTestingUtility.restart(procExec); ProcedureTestingUtility.waitProcedure(procExec, procId); } boolean abortResult = procExec.abort(procId, true); assertFalse(abortResult); MasterProcedureTestingUtility.testRestartWithAbort(procExec, procId); ProcedureTestingUtility.waitNoProcedureRunning(procExec); ProcedureTestingUtility.assertProcNotFailed(procExec, procId); // Validate the delete table procedure was not aborted MasterProcedureTestingUtility.validateTableDeletion( UTIL.getHBaseCluster().getMaster(), tableName); }
@Test public void testQueueLockAndLockHeartbeatOperations() throws Exception { HMaster master = UTIL.getMiniHBaseCluster().getMaster(); CPMasterObserver cp = master.getMasterCoprocessorHost().findCoprocessor(CPMasterObserver.class); cp.resetStates(); final TableName tableName = TableName.valueOf("testLockedTable"); long procId = master.getLockManager().remoteLocks().requestTableLock(tableName, LockType.EXCLUSIVE, "desc", null); master.getLockManager().remoteLocks().lockHeartbeat(procId, false); assertTrue(cp.preAndPostForQueueLockAndHeartbeatLockCalled()); ProcedureTestingUtility.waitNoProcedureRunning(master.getMasterProcedureExecutor()); ProcedureTestingUtility.assertProcNotFailed(master.getMasterProcedureExecutor(), procId); } }
@Test public void testSplitWithoutPONR() throws Exception { final TableName tableName = TableName.valueOf(name.getMethodName()); final ProcedureExecutor<MasterProcedureEnv> procExec = getMasterProcedureExecutor(); RegionInfo [] regions = MasterProcedureTestingUtility.createTable( procExec, tableName, null, ColumnFamilyName1, ColumnFamilyName2); insertData(tableName); int splitRowNum = startRowNum + rowCount / 2; byte[] splitKey = Bytes.toBytes("" + splitRowNum); assertTrue("not able to find a splittable region", regions != null); assertTrue("not able to find a splittable region", regions.length == 1); ProcedureTestingUtility.waitNoProcedureRunning(procExec); ProcedureTestingUtility.setKillAndToggleBeforeStoreUpdate(procExec, true); // Split region of the table long procId = procExec.submitProcedure( new SplitTableRegionProcedure(procExec.getEnvironment(), regions[0], splitKey)); // Execute until step 7 of split procedure // NOTE: the 7 (number after SPLIT_TABLE_REGION_UPDATE_META step) MasterProcedureTestingUtility.testRecoveryAndDoubleExecution(procExec, procId, 7, false); // Unset Toggle Kill and make ProcExec work correctly ProcedureTestingUtility.setKillAndToggleBeforeStoreUpdate(procExec, false); MasterProcedureTestingUtility.restartMasterProcedureExecutor(procExec); ProcedureTestingUtility.waitProcedure(procExec, procId); // Even split failed after step 4, it should still works fine verify(tableName, splitRowNum); }
@Test public void testRollbackAndDoubleExecution() throws Exception { final String namespaceName = "testRollbackAndDoubleExecution"; final ProcedureExecutor<MasterProcedureEnv> procExec = getMasterProcedureExecutor(); createNamespaceForTesting(namespaceName); ProcedureTestingUtility.waitNoProcedureRunning(procExec); ProcedureTestingUtility.setKillAndToggleBeforeStoreUpdate(procExec, true); // Start the DeleteNamespace procedure && kill the executor long procId = procExec.submitProcedure( new DeleteNamespaceProcedure(procExec.getEnvironment(), namespaceName)); int lastStep = 2; // failing before DELETE_NAMESPACE_DELETE_FROM_NS_TABLE MasterProcedureTestingUtility.testRollbackAndDoubleExecution(procExec, procId, lastStep); // Validate the namespace still exists NamespaceDescriptor createdNsDescriptor= UTIL.getAdmin().getNamespaceDescriptor(namespaceName); assertNotNull(createdNsDescriptor); }
@Test public void testRollbackAndDoubleExecution() throws Exception { final NamespaceDescriptor nsd = NamespaceDescriptor.create("testRollbackAndDoubleExecution").build(); final ProcedureExecutor<MasterProcedureEnv> procExec = getMasterProcedureExecutor(); ProcedureTestingUtility.waitNoProcedureRunning(procExec); ProcedureTestingUtility.setKillAndToggleBeforeStoreUpdate(procExec, true); // Start the CreateNamespace procedure && kill the executor long procId = procExec.submitProcedure( new CreateNamespaceProcedure(procExec.getEnvironment(), nsd)); int lastStep = 2; // failing before CREATE_NAMESPACE_CREATE_DIRECTORY MasterProcedureTestingUtility.testRollbackAndDoubleExecution(procExec, procId, lastStep); // Validate the non-existence of namespace try { NamespaceDescriptor nsDescriptor = UTIL.getAdmin().getNamespaceDescriptor(nsd.getName()); assertNull(nsDescriptor); } catch (NamespaceNotFoundException nsnfe) { // Expected LOG.info("The namespace " + nsd.getName() + " is not created."); } }
@Test public void testRecoveryAndDoubleExecution() throws Exception { final NamespaceDescriptor nsd = NamespaceDescriptor.create("testRecoveryAndDoubleExecution").build(); final ProcedureExecutor<MasterProcedureEnv> procExec = getMasterProcedureExecutor(); ProcedureTestingUtility.waitNoProcedureRunning(procExec); ProcedureTestingUtility.setKillAndToggleBeforeStoreUpdate(procExec, true); // Start the CreateNamespace procedure && kill the executor long procId = procExec.submitProcedure( new CreateNamespaceProcedure(procExec.getEnvironment(), nsd)); // Restart the executor and execute the step twice MasterProcedureTestingUtility.testRecoveryAndDoubleExecution(procExec, procId); // Validate the creation of namespace ProcedureTestingUtility.assertProcNotFailed(procExec, procId); validateNamespaceCreated(nsd); }
@Test public void testRecoveryAndDoubleExecution() throws Exception { final NamespaceDescriptor nsd = NamespaceDescriptor.create("testRecoveryAndDoubleExecution").build(); final String nsKey = "foo"; final String nsValue = "bar"; final ProcedureExecutor<MasterProcedureEnv> procExec = getMasterProcedureExecutor(); createNamespaceForTesting(nsd); ProcedureTestingUtility.waitNoProcedureRunning(procExec); ProcedureTestingUtility.setKillAndToggleBeforeStoreUpdate(procExec, true); // Modify nsd.setConfiguration(nsKey, nsValue); // Start the Modify procedure && kill the executor long procId = procExec.submitProcedure( new ModifyNamespaceProcedure(procExec.getEnvironment(), nsd)); // Restart the executor and execute the step twice MasterProcedureTestingUtility.testRecoveryAndDoubleExecution(procExec, procId); ProcedureTestingUtility.assertProcNotFailed(procExec, procId); // Validate NamespaceDescriptor currentNsDescriptor = UTIL.getAdmin().getNamespaceDescriptor(nsd.getName()); assertEquals(nsValue, currentNsDescriptor.getConfigurationValue(nsKey)); }
@Test public void testRollbackAndDoubleExecution() throws Exception { final NamespaceDescriptor nsd = NamespaceDescriptor.create("testRollbackAndDoubleExecution").build(); final String nsKey = "foo"; final String nsValue = "bar"; final ProcedureExecutor<MasterProcedureEnv> procExec = getMasterProcedureExecutor(); createNamespaceForTesting(nsd); ProcedureTestingUtility.waitNoProcedureRunning(procExec); ProcedureTestingUtility.setKillAndToggleBeforeStoreUpdate(procExec, true); // Modify // Start the Modify procedure && kill the executor long procId = procExec.submitProcedure(new ModifyNamespaceProcedure(procExec.getEnvironment(), NamespaceDescriptor.create(nsd).addConfiguration(nsKey, nsValue).build())); int lastStep = 2; // failing before MODIFY_NAMESPACE_UPDATE_NS_TABLE MasterProcedureTestingUtility.testRollbackAndDoubleExecution(procExec, procId, lastStep); // Validate NamespaceDescriptor currentNsDescriptor = UTIL.getAdmin().getNamespaceDescriptor(nsd.getName()); assertNull(currentNsDescriptor.getConfigurationValue(nsKey)); }