public SplitTableRegionProcedure createSplitProcedure(final RegionInfo regionToSplit, final byte[] splitKey) throws IOException { return new SplitTableRegionProcedure(getProcedureEnvironment(), regionToSplit, splitKey); }
switch (state) { case SPLIT_TABLE_REGION_PREPARE: if (prepareSplitRegion(env)) { setNextState(SplitTableRegionState.SPLIT_TABLE_REGION_PRE_OPERATION); break; } else { preSplitRegion(env); setNextState(SplitTableRegionState.SPLIT_TABLE_REGION_CLOSE_PARENT_REGION); break; case SPLIT_TABLE_REGION_CLOSE_PARENT_REGION: addChildProcedure(createUnassignProcedures(env)); setNextState(SplitTableRegionState.SPLIT_TABLE_REGIONS_CHECK_CLOSED_REGIONS); break; case SPLIT_TABLE_REGIONS_CHECK_CLOSED_REGIONS: checkClosedRegions(env); setNextState(SplitTableRegionState.SPLIT_TABLE_REGION_CREATE_DAUGHTER_REGIONS); break; case SPLIT_TABLE_REGION_CREATE_DAUGHTER_REGIONS: removeNonDefaultReplicas(env); createDaughterRegions(env); setNextState(SplitTableRegionState.SPLIT_TABLE_REGION_WRITE_MAX_SEQUENCE_ID_FILE); break; case SPLIT_TABLE_REGION_WRITE_MAX_SEQUENCE_ID_FILE: writeMaxSequenceIdFile(env); setNextState(SplitTableRegionState.SPLIT_TABLE_REGION_PRE_OPERATION_BEFORE_META); break; case SPLIT_TABLE_REGION_PRE_OPERATION_BEFORE_META: preSplitRegionBeforeMETA(env);
/** * Create daughter regions */ @VisibleForTesting public void createDaughterRegions(final MasterProcedureEnv env) throws IOException { final MasterFileSystem mfs = env.getMasterServices().getMasterFileSystem(); final Path tabledir = FSUtils.getTableDir(mfs.getRootDir(), getTableName()); final FileSystem fs = mfs.getFileSystem(); HRegionFileSystem regionFs = HRegionFileSystem.openRegionFromFileSystem( env.getMasterConfiguration(), fs, tabledir, getParentRegion(), false); regionFs.createSplitsDir(); Pair<Integer, Integer> expectedReferences = splitStoreFiles(env, regionFs); assertReferenceFileCount(fs, expectedReferences.getFirst(), regionFs.getSplitsDir(daughter_1_RI)); //Move the files from the temporary .splits to the final /table/region directory regionFs.commitDaughterRegion(daughter_1_RI); assertReferenceFileCount(fs, expectedReferences.getFirst(), new Path(tabledir, daughter_1_RI.getEncodedName())); assertReferenceFileCount(fs, expectedReferences.getSecond(), regionFs.getSplitsDir(daughter_2_RI)); regionFs.commitDaughterRegion(daughter_2_RI); assertReferenceFileCount(fs, expectedReferences.getSecond(), new Path(tabledir, daughter_2_RI.getEncodedName())); }
@Override protected boolean abort(MasterProcedureEnv env) { // Abort means rollback. We can't rollback all steps. HBASE-18018 added abort to all // Procedures. Here is a Procedure that has a PONR and cannot be aborted wants it enters this // range of steps; what do we do for these should an operator want to cancel them? HBASE-20022. return isRollbackSupported(getCurrentState())? super.abort(env): false; } }
public SplitTableRegionProcedure(final MasterProcedureEnv env, final RegionInfo regionToSplit, final byte[] splitRow) throws IOException { super(env, regionToSplit); preflightChecks(env, true); checkOnline(env, regionToSplit); this.bestSplitRow = splitRow; checkSplittable(env, regionToSplit, bestSplitRow); final TableName table = regionToSplit.getTable(); final long rid = getDaughterRegionIdTimestamp(regionToSplit); this.daughter_1_RI = RegionInfoBuilder.newBuilder(table) .setStartKey(regionToSplit.getStartKey()) .setRegionId(rid) .build(); TableDescriptor htd = env.getMasterServices().getTableDescriptors().get(getTableName()); if(htd.getRegionSplitPolicyClassName() != null) {
SplitTableRegionProcedure splitProcedure = new SplitTableRegionProcedure( env, regionInfos.get(0), Bytes.toBytes("row5"));
new SplitTableRegionProcedure(procExec.getEnvironment(), regions[0], splitKey));
@Test public void testSplitTableRegionEmptyDaughter() throws Exception { final TableName tableName = TableName.valueOf(name.getMethodName()); final ProcedureExecutor<MasterProcedureEnv> procExec = getMasterProcedureExecutor(); RegionInfo [] regions = MasterProcedureTestingUtility.createTable( procExec, tableName, null, ColumnFamilyName1, ColumnFamilyName2); insertData(tableName); // Split to two daughters with one of them only has 1 row int splitRowNum = startRowNum + rowCount; byte[] splitKey = Bytes.toBytes("" + splitRowNum); assertTrue("not able to find a splittable region", regions != null); assertTrue("not able to find a splittable region", regions.length == 1); // collect AM metrics before test collectAssignmentManagerMetrics(); // Split region of the table long procId = procExec.submitProcedure( new SplitTableRegionProcedure(procExec.getEnvironment(), regions[0], splitKey)); // Wait the completion ProcedureTestingUtility.waitProcedure(procExec, procId); ProcedureTestingUtility.assertProcNotFailed(procExec, procId); // Make sure one daughter has 0 rows. List<HRegion> daughters = UTIL.getMiniHBaseCluster().getRegions(tableName); assertTrue(daughters.size() == 2); assertTrue(UTIL.countRows(tableName) == rowCount); assertTrue(UTIL.countRows(daughters.get(0)) == 0 || UTIL.countRows(daughters.get(1)) == 0); assertEquals(splitSubmittedCount + 1, splitProcMetrics.getSubmittedCounter().getCount()); assertEquals(splitFailedCount, splitProcMetrics.getFailedCounter().getCount()); }
@Test public void testSplitWithoutPONR() throws Exception { final TableName tableName = TableName.valueOf(name.getMethodName()); final ProcedureExecutor<MasterProcedureEnv> procExec = getMasterProcedureExecutor(); RegionInfo [] regions = MasterProcedureTestingUtility.createTable( procExec, tableName, null, ColumnFamilyName1, ColumnFamilyName2); insertData(tableName); int splitRowNum = startRowNum + rowCount / 2; byte[] splitKey = Bytes.toBytes("" + splitRowNum); assertTrue("not able to find a splittable region", regions != null); assertTrue("not able to find a splittable region", regions.length == 1); ProcedureTestingUtility.waitNoProcedureRunning(procExec); ProcedureTestingUtility.setKillAndToggleBeforeStoreUpdate(procExec, true); // Split region of the table long procId = procExec.submitProcedure( new SplitTableRegionProcedure(procExec.getEnvironment(), regions[0], splitKey)); // Execute until step 7 of split procedure // NOTE: the 7 (number after SPLIT_TABLE_REGION_UPDATE_META step) MasterProcedureTestingUtility.testRecoveryAndDoubleExecution(procExec, procId, 7, false); // Unset Toggle Kill and make ProcExec work correctly ProcedureTestingUtility.setKillAndToggleBeforeStoreUpdate(procExec, false); MasterProcedureTestingUtility.restartMasterProcedureExecutor(procExec); ProcedureTestingUtility.waitProcedure(procExec, procId); // Even split failed after step 4, it should still works fine verify(tableName, splitRowNum); }
@Test public void testSplitTableRegionNoStoreFile() throws Exception { final TableName tableName = TableName.valueOf(name.getMethodName()); final ProcedureExecutor<MasterProcedureEnv> procExec = getMasterProcedureExecutor(); RegionInfo [] regions = MasterProcedureTestingUtility.createTable( procExec, tableName, null, ColumnFamilyName1, ColumnFamilyName2); int splitRowNum = startRowNum + rowCount / 2; byte[] splitKey = Bytes.toBytes("" + splitRowNum); assertTrue("not able to find a splittable region", regions != null); assertTrue("not able to find a splittable region", regions.length == 1); // collect AM metrics before test collectAssignmentManagerMetrics(); // Split region of the table long procId = procExec.submitProcedure( new SplitTableRegionProcedure(procExec.getEnvironment(), regions[0], splitKey)); // Wait the completion ProcedureTestingUtility.waitProcedure(procExec, procId); ProcedureTestingUtility.assertProcNotFailed(procExec, procId); assertTrue(UTIL.getMiniHBaseCluster().getRegions(tableName).size() == 2); assertTrue(UTIL.countRows(tableName) == 0); assertEquals(splitSubmittedCount + 1, splitProcMetrics.getSubmittedCounter().getCount()); assertEquals(splitFailedCount, splitProcMetrics.getFailedCounter().getCount()); }
@Test public void testInvalidSplitKey() throws Exception { final TableName tableName = TableName.valueOf(name.getMethodName()); final ProcedureExecutor<MasterProcedureEnv> procExec = getMasterProcedureExecutor(); RegionInfo [] regions = MasterProcedureTestingUtility.createTable( procExec, tableName, null, ColumnFamilyName1, ColumnFamilyName2); insertData(tableName); assertTrue("not able to find a splittable region", regions != null); assertTrue("not able to find a splittable region", regions.length == 1); // collect AM metrics before test collectAssignmentManagerMetrics(); // Split region of the table with null split key try { long procId1 = procExec.submitProcedure( new SplitTableRegionProcedure(procExec.getEnvironment(), regions[0], null)); ProcedureTestingUtility.waitProcedure(procExec, procId1); fail("unexpected procedure start with invalid split-key"); } catch (DoNotRetryIOException e) { LOG.debug("Expected Split procedure construction failure: " + e.getMessage()); } assertEquals(splitSubmittedCount, splitProcMetrics.getSubmittedCounter().getCount()); assertEquals(splitFailedCount, splitProcMetrics.getFailedCounter().getCount()); }
@Test public void testSplitTableRegionUnevenDaughter() throws Exception { final TableName tableName = TableName.valueOf(name.getMethodName()); final ProcedureExecutor<MasterProcedureEnv> procExec = getMasterProcedureExecutor(); RegionInfo [] regions = MasterProcedureTestingUtility.createTable( procExec, tableName, null, ColumnFamilyName1, ColumnFamilyName2); insertData(tableName); // Split to two daughters with one of them only has 1 row int splitRowNum = startRowNum + rowCount / 4; byte[] splitKey = Bytes.toBytes("" + splitRowNum); assertTrue("not able to find a splittable region", regions != null); assertTrue("not able to find a splittable region", regions.length == 1); // collect AM metrics before test collectAssignmentManagerMetrics(); // Split region of the table long procId = procExec.submitProcedure( new SplitTableRegionProcedure(procExec.getEnvironment(), regions[0], splitKey)); // Wait the completion ProcedureTestingUtility.waitProcedure(procExec, procId); ProcedureTestingUtility.assertProcNotFailed(procExec, procId); verify(tableName, splitRowNum); assertEquals(splitSubmittedCount + 1, splitProcMetrics.getSubmittedCounter().getCount()); assertEquals(splitFailedCount, splitProcMetrics.getFailedCounter().getCount()); }
new SplitTableRegionProcedure(procExec.getEnvironment(), regions[0], splitKey));
@Test public void testRecoveryAndDoubleExecution() throws Exception { final TableName tableName = TableName.valueOf(name.getMethodName()); final ProcedureExecutor<MasterProcedureEnv> procExec = getMasterProcedureExecutor(); RegionInfo [] regions = MasterProcedureTestingUtility.createTable( procExec, tableName, null, ColumnFamilyName1, ColumnFamilyName2); insertData(tableName); int splitRowNum = startRowNum + rowCount / 2; byte[] splitKey = Bytes.toBytes("" + splitRowNum); assertTrue("not able to find a splittable region", regions != null); assertTrue("not able to find a splittable region", regions.length == 1); ProcedureTestingUtility.waitNoProcedureRunning(procExec); ProcedureTestingUtility.setKillIfHasParent(procExec, false); ProcedureTestingUtility.setKillAndToggleBeforeStoreUpdate(procExec, true); // collect AM metrics before test collectAssignmentManagerMetrics(); // Split region of the table long procId = procExec.submitProcedure( new SplitTableRegionProcedure(procExec.getEnvironment(), regions[0], splitKey)); // Restart the executor and execute the step twice MasterProcedureTestingUtility.testRecoveryAndDoubleExecution(procExec, procId); ProcedureTestingUtility.assertProcNotFailed(procExec, procId); verify(tableName, splitRowNum); assertEquals(splitSubmittedCount + 1, splitProcMetrics.getSubmittedCounter().getCount()); assertEquals(splitFailedCount, splitProcMetrics.getFailedCounter().getCount()); }
new SplitTableRegionProcedure(procExec.getEnvironment(), regions[0], splitKey));
SplitTableRegionProcedure splitProcedure = new SplitTableRegionProcedure( env, regionInfos.get(0), Bytes.toBytes("row5"));
new SplitTableRegionProcedure(procExec.getEnvironment(), regions[0], splitKey));
@Test public void testSplitTableRegionEmptyDaughter() throws Exception { final TableName tableName = TableName.valueOf(name.getMethodName()); final ProcedureExecutor<MasterProcedureEnv> procExec = getMasterProcedureExecutor(); RegionInfo [] regions = MasterProcedureTestingUtility.createTable( procExec, tableName, null, ColumnFamilyName1, ColumnFamilyName2); insertData(tableName); // Split to two daughters with one of them only has 1 row int splitRowNum = startRowNum + rowCount; byte[] splitKey = Bytes.toBytes("" + splitRowNum); assertTrue("not able to find a splittable region", regions != null); assertTrue("not able to find a splittable region", regions.length == 1); // collect AM metrics before test collectAssignmentManagerMetrics(); // Split region of the table long procId = procExec.submitProcedure( new SplitTableRegionProcedure(procExec.getEnvironment(), regions[0], splitKey)); // Wait the completion ProcedureTestingUtility.waitProcedure(procExec, procId); ProcedureTestingUtility.assertProcNotFailed(procExec, procId); // Make sure one daughter has 0 rows. List<HRegion> daughters = UTIL.getMiniHBaseCluster().getRegions(tableName); assertTrue(daughters.size() == 2); assertTrue(UTIL.countRows(tableName) == rowCount); assertTrue(UTIL.countRows(daughters.get(0)) == 0 || UTIL.countRows(daughters.get(1)) == 0); assertEquals(splitSubmittedCount + 1, splitProcMetrics.getSubmittedCounter().getCount()); assertEquals(splitFailedCount, splitProcMetrics.getFailedCounter().getCount()); }
@Test public void testSplitWithoutPONR() throws Exception { final TableName tableName = TableName.valueOf(name.getMethodName()); final ProcedureExecutor<MasterProcedureEnv> procExec = getMasterProcedureExecutor(); RegionInfo [] regions = MasterProcedureTestingUtility.createTable( procExec, tableName, null, ColumnFamilyName1, ColumnFamilyName2); insertData(tableName); int splitRowNum = startRowNum + rowCount / 2; byte[] splitKey = Bytes.toBytes("" + splitRowNum); assertTrue("not able to find a splittable region", regions != null); assertTrue("not able to find a splittable region", regions.length == 1); ProcedureTestingUtility.waitNoProcedureRunning(procExec); ProcedureTestingUtility.setKillAndToggleBeforeStoreUpdate(procExec, true); // Split region of the table long procId = procExec.submitProcedure( new SplitTableRegionProcedure(procExec.getEnvironment(), regions[0], splitKey)); // Execute until step 7 of split procedure // NOTE: the 7 (number after SPLIT_TABLE_REGION_UPDATE_META step) MasterProcedureTestingUtility.testRecoveryAndDoubleExecution(procExec, procId, 7, false); // Unset Toggle Kill and make ProcExec work correctly ProcedureTestingUtility.setKillAndToggleBeforeStoreUpdate(procExec, false); MasterProcedureTestingUtility.restartMasterProcedureExecutor(procExec); ProcedureTestingUtility.waitProcedure(procExec, procId); // Even split failed after step 4, it should still works fine verify(tableName, splitRowNum); }
@Test public void testSplitTableRegionNoStoreFile() throws Exception { final TableName tableName = TableName.valueOf(name.getMethodName()); final ProcedureExecutor<MasterProcedureEnv> procExec = getMasterProcedureExecutor(); RegionInfo [] regions = MasterProcedureTestingUtility.createTable( procExec, tableName, null, ColumnFamilyName1, ColumnFamilyName2); int splitRowNum = startRowNum + rowCount / 2; byte[] splitKey = Bytes.toBytes("" + splitRowNum); assertTrue("not able to find a splittable region", regions != null); assertTrue("not able to find a splittable region", regions.length == 1); // collect AM metrics before test collectAssignmentManagerMetrics(); // Split region of the table long procId = procExec.submitProcedure( new SplitTableRegionProcedure(procExec.getEnvironment(), regions[0], splitKey)); // Wait the completion ProcedureTestingUtility.waitProcedure(procExec, procId); ProcedureTestingUtility.assertProcNotFailed(procExec, procId); assertTrue(UTIL.getMiniHBaseCluster().getRegions(tableName).size() == 2); assertTrue(UTIL.countRows(tableName) == 0); assertEquals(splitSubmittedCount + 1, splitProcMetrics.getSubmittedCounter().getCount()); assertEquals(splitFailedCount, splitProcMetrics.getFailedCounter().getCount()); }