public static void blockUntilSplitFinished(HBaseTestingUtility util, TableName tableName, int expectedRegionSize) throws Exception { for (int i = 0; i < 100; i++) { List<HRegionInfo> hRegionInfoList = util.getAdmin().getTableRegions(tableName); if (hRegionInfoList.size() >= expectedRegionSize) { break; } Thread.sleep(1000); } }
private void waitRegionsAfterMerge(final long numRegionsAfterMerge) throws IOException, InterruptedException { // Verify that there's one region less long startTime = System.currentTimeMillis(); while (admin.getTableRegions(TABLE_NAME).size() != numRegionsAfterMerge) { // This may be flaky... if after 15sec the merge is not complete give up // it will fail in the assertEquals(numRegionsAfterMerge). if ((System.currentTimeMillis() - startTime) > 15000) break; Thread.sleep(100); } SnapshotTestingUtils.waitForTableToBeOnline(UTIL, TABLE_NAME); }
private List<HRegionInfo> assertRegionCount(final TableName tableName, final int nregions) throws Exception { UTIL.waitUntilNoRegionsInTransition(); List<HRegionInfo> tableRegions = admin.getTableRegions(tableName); assertEquals(nregions, tableRegions.size()); return tableRegions; } }
/** * Do a split, and verify that this only affects one table */ private void runTestRegionOperationsIndependent() throws Exception { // Verify that region information is the same pre-split ((ClusterConnection) UTIL.getConnection()).clearRegionCache(); List<HRegionInfo> originalTableHRegions = admin.getTableRegions(originalTableName); final int originalRegionCount = originalTableHRegions.size(); final int cloneTableRegionCount = admin.getTableRegions(cloneTableName).size(); Assert.assertEquals( "The number of regions in the cloned table is different than in the original table.", originalRegionCount, cloneTableRegionCount); // Split a region on the parent table admin.splitRegion(originalTableHRegions.get(0).getRegionName()); waitOnSplit(UTIL.getConnection(), originalTable, originalRegionCount); // Verify that the cloned table region is not split final int cloneTableRegionCount2 = admin.getTableRegions(cloneTableName).size(); Assert.assertEquals( "The number of regions in the cloned table changed though none of its regions were split.", cloneTableRegionCount, cloneTableRegionCount2); }
@Override public void perform() throws Exception { HBaseTestingUtility util = context.getHBaseIntegrationTestingUtility(); Admin admin = util.getAdmin(); LOG.info("Performing action: Flush random region of table " + tableName); List<HRegionInfo> regions = admin.getTableRegions(tableName); if (regions == null || regions.isEmpty()) { LOG.info("Table " + tableName + " doesn't have regions to flush"); return; } HRegionInfo region = PolicyBasedChaosMonkey.selectRandomItem( regions.toArray(new HRegionInfo[regions.size()])); LOG.debug("Flushing region " + region.getRegionNameAsString()); try { admin.flushRegion(region.getRegionName()); } catch (Exception ex) { LOG.warn("Flush failed, might be caused by other chaos: " + ex.getMessage()); } if (sleepTime > 0) { Thread.sleep(sleepTime); } } }
@Test public void testFilteringMissingTables() throws Exception { final TableName missingTable = TableName.valueOf("doesNotExist"); // Set up Admin to return null (match the implementation) Admin admin = mock(Admin.class); when(conn.getAdmin()).thenReturn(admin); when(admin.getTableRegions(missingTable)).thenReturn(null); QuotaObserverChore chore = mock(QuotaObserverChore.class); Map<RegionInfo,Long> regionUsage = new HashMap<>(); TableQuotaSnapshotStore store = new TableQuotaSnapshotStore(conn, chore, regionUsage); // A super dirty hack to verify that, after getting no regions for our table, // we bail out and start processing the next element (which there is none). final TablesWithQuotas tables = new TablesWithQuotas(conn, conf) { @Override int getNumReportedRegions(TableName table, QuotaSnapshotStore<TableName> tableStore) { throw new RuntimeException("Should should not reach here"); } }; tables.addTableQuotaTable(missingTable); tables.filterInsufficientlyReportedTables(store); final Set<TableName> tablesWithQuotas = tables.getTableQuotaTables(); assertTrue( "Expected to find no tables, but found " + tablesWithQuotas, tablesWithQuotas.isEmpty()); } }
private static void logError(String msg, Context context) throws IOException { TableName table = getTableName(context.getConfiguration()); LOG.error("Failure in chain verification: " + msg); try (Connection connection = ConnectionFactory.createConnection(context.getConfiguration()); Admin admin = connection.getAdmin()) { LOG.error("cluster status:\n" + admin.getClusterStatus()); LOG.error("table regions:\n" + Joiner.on("\n").join(admin.getTableRegions(table))); } } }
@Test public void testRoundRobinAssignment() throws Exception { TableName tableName = TableName.valueOf("testRoundRobinAssignment"); HTableDescriptor desc = new HTableDescriptor(tableName); desc.addFamily(new HColumnDescriptor(HConstants.CATALOG_FAMILY)); admin.createTable(desc, Bytes.toBytes("aaa"), Bytes.toBytes("zzz"), REGION_NUM); TEST_UTIL.waitTableAvailable(tableName); TEST_UTIL.loadTable(admin.getConnection().getTable(tableName), HConstants.CATALOG_FAMILY); admin.flush(tableName); LoadBalancer balancer = master.getLoadBalancer(); List<RegionInfo> regions = admin.getRegions(tableName); regions.addAll(admin.getTableRegions(TableName.META_TABLE_NAME)); List<ServerName> servers = Lists.newArrayList( admin.getClusterMetrics(EnumSet.of(Option.LIVE_SERVERS)).getLiveServerMetrics().keySet()); Map<ServerName, List<RegionInfo>> map = balancer.roundRobinAssignment(regions, servers); for (List<RegionInfo> regionInfos : map.values()) { regions.removeAll(regionInfos); } assertEquals("No region should be missed by balancer", 0, regions.size()); }
@Test public void testCreateTable() throws Exception { final TableName tableName = TableName.valueOf(name.getMethodName()); TEST_UTIL.createTable(tableName, Bytes.toBytes("f"), splitKeys); TEST_UTIL.waitUntilAllRegionsAssigned(tableName); // All regions should have favored nodes checkIfFavoredNodeInformationIsCorrect(tableName); List<HRegionInfo> regions = admin.getTableRegions(tableName); TEST_UTIL.deleteTable(tableName); checkNoFNForDeletedTable(regions); }
@BeforeClass public static void setupCluster() throws Exception { setupConf(UTIL.getConfiguration()); UTIL.startMiniCluster(1); masterServices = UTIL.getMiniHBaseCluster().getMaster(); UTIL.getAdmin().createNamespace(NamespaceDescriptor.create(namespace).build()); UTIL.createTable(tableName, new byte[][]{"fam".getBytes()}, new byte[][] {"1".getBytes()}); List<HRegionInfo> regions = UTIL.getAdmin().getTableRegions(tableName); assert regions.size() > 0; tableRegions = new HRegionInfo[regions.size()]; regions.toArray(tableRegions); }
@BeforeClass public static void setUp() throws Exception { UTIL.startMiniCluster(1); try (Table table = UTIL.createTable(TABLE_NAME, CF)) { for (int i = 0; i < COUNT; i++) { table.put(new Put(Bytes.toBytes(i)).addColumn(CF, CQ, Bytes.toBytes(i))); } } HRI = UTIL.getAdmin().getTableRegions(TABLE_NAME).get(0); STUB = ((ConnectionImplementation) UTIL.getConnection()) .getClient(UTIL.getHBaseCluster().getRegionServer(0).getServerName()); }
@BeforeClass public static void setUp() throws Exception { UTIL.startMiniCluster(1); try (Table table = UTIL.createTable(TABLE_NAME, CF)) { for (int i = 0; i < COUNT; i++) { table.put(new Put(Bytes.toBytes(i)).addColumn(CF, CQ, Bytes.toBytes(i))); } } HRI = UTIL.getAdmin().getTableRegions(TABLE_NAME).get(0); STUB = ((ConnectionImplementation) UTIL.getConnection()) .getClient(UTIL.getHBaseCluster().getRegionServer(0).getServerName()); }
@Test public void testTruncateTable() throws Exception { final TableName tableName = TableName.valueOf(name.getMethodName()); TEST_UTIL.createTable(tableName, Bytes.toBytes("f"), splitKeys); TEST_UTIL.waitUntilAllRegionsAssigned(tableName); // All regions should have favored nodes checkIfFavoredNodeInformationIsCorrect(tableName); List<HRegionInfo> regions = admin.getTableRegions(tableName); TEST_UTIL.truncateTable(tableName, true); checkNoFNForDeletedTable(regions); checkIfFavoredNodeInformationIsCorrect(tableName); regions = admin.getTableRegions(tableName); TEST_UTIL.truncateTable(tableName, false); checkNoFNForDeletedTable(regions); TEST_UTIL.deleteTable(tableName); }
/** * Create a table and take a snapshot of the table used by the export test. */ @Before public void setUp() throws Exception { this.admin = TEST_UTIL.getAdmin(); tableName = TableName.valueOf("testtb-" + testName.getMethodName()); snapshotName = Bytes.toBytes("snaptb0-" + testName.getMethodName()); emptySnapshotName = Bytes.toBytes("emptySnaptb0-" + testName.getMethodName()); // create Table createTable(); // Take an empty snapshot admin.snapshot(emptySnapshotName, tableName); // Add some rows SnapshotTestingUtils.loadData(TEST_UTIL, tableName, 50, FAMILY); tableNumFiles = admin.getTableRegions(tableName).size(); // take a snapshot admin.snapshot(snapshotName, tableName); }
@Test public void testRandomAssignmentWithNoFavNodes() throws Exception { final String tableName = "testRandomAssignmentWithNoFavNodes"; HTableDescriptor desc = new HTableDescriptor(TableName.valueOf(tableName)); desc.addFamily(new HColumnDescriptor(HConstants.CATALOG_FAMILY)); admin.createTable(desc); TEST_UTIL.waitTableAvailable(desc.getTableName()); RegionInfo hri = admin.getTableRegions(TableName.valueOf(tableName)).get(0); FavoredNodesManager fnm = master.getFavoredNodesManager(); fnm.deleteFavoredNodesForRegions(Lists.newArrayList(hri)); assertNull("Favored nodes not found null after delete", fnm.getFavoredNodes(hri)); LoadBalancer balancer = master.getLoadBalancer(); ServerName destination = balancer.randomAssignment(hri, Lists.newArrayList(admin .getClusterMetrics(EnumSet.of(Option.LIVE_SERVERS)).getLiveServerMetrics() .keySet().stream().collect(Collectors.toList()))); assertNotNull(destination); List<ServerName> favoredNodes = fnm.getFavoredNodes(hri); assertNotNull(favoredNodes); boolean containsFN = false; for (ServerName sn : favoredNodes) { if (ServerName.isSameAddress(destination, sn)) { containsFN = true; } } assertTrue("Destination server does not belong to favored nodes.", containsFN); }
private void checkFavoredNodeAssignments(TableName tableName, FavoredNodesManager fnm, RegionStates regionStates) throws IOException { for (RegionInfo hri : admin.getTableRegions(tableName)) { ServerName host = regionStates.getRegionServerOfRegion(hri); assertNotNull("Region: " + hri.getEncodedName() + " not on FN, current: " + host + " FN list: " + fnm.getFavoredNodes(hri), FavoredNodesPlan.getFavoredServerPosition(fnm.getFavoredNodes(hri), host)); } }
@Test public void testSystemTables() throws Exception { final TableName tableName = TableName.valueOf(name.getMethodName()); TEST_UTIL.createTable(tableName, Bytes.toBytes("f"), splitKeys); TEST_UTIL.waitUntilAllRegionsAssigned(tableName); // All regions should have favored nodes checkIfFavoredNodeInformationIsCorrect(tableName); for (TableName sysTable : admin.listTableNamesByNamespace(NamespaceDescriptor.SYSTEM_NAMESPACE_NAME_STR)) { List<HRegionInfo> regions = admin.getTableRegions(sysTable); for (HRegionInfo region : regions) { assertNull("FN should be null for sys region", fnm.getFavoredNodes(region)); } } TEST_UTIL.deleteTable(tableName); }
private void unbalance(HMaster master, TableName tableName) throws Exception { while (master.getAssignmentManager().getRegionStates().getRegionsInTransitionCount() > 0) { Thread.sleep(100); } HRegionServer biasedServer = TEST_UTIL.getMiniHBaseCluster().getRegionServer(0); for (HRegionInfo regionInfo : TEST_UTIL.getAdmin().getTableRegions(tableName)) { master.move(regionInfo.getEncodedNameAsBytes(), Bytes.toBytes(biasedServer.getServerName().getServerName())); } while (master.getAssignmentManager().getRegionStates().getRegionsInTransitionCount() > 0) { Thread.sleep(100); } } }
@Test public void testSplitSwitch() throws Exception { final TableName tableName = TableName.valueOf(name.getMethodName()); Table t = TEST_UTIL.createTable(tableName, FAMILY); TEST_UTIL.loadTable(t, FAMILY, false); RegionLocator locator = TEST_UTIL.getConnection().getRegionLocator(t.getName()); int originalCount = locator.getAllRegionLocations().size(); Admin admin = TEST_UTIL.getAdmin(); initSwitchStatus(admin); boolean[] results = admin.setSplitOrMergeEnabled(false, false, MasterSwitchType.SPLIT); assertEquals(1, results.length); assertTrue(results[0]); admin.split(t.getName()); int count = admin.getTableRegions(tableName).size(); assertTrue(originalCount == count); results = admin.setSplitOrMergeEnabled(true, false, MasterSwitchType.SPLIT); assertEquals(1, results.length); assertFalse(results[0]); admin.split(t.getName()); while ((count = admin.getTableRegions(tableName).size()) == originalCount) { Threads.sleep(1); } count = admin.getTableRegions(tableName).size(); assertTrue(originalCount < count); admin.close(); }
public static void createTable(final HBaseTestingUtility util, final TableName tableName, int regionReplication, int nRegions, final byte[]... families) throws IOException, InterruptedException { TableDescriptorBuilder builder = TableDescriptorBuilder .newBuilder(tableName) .setRegionReplication(regionReplication); for (byte[] family : families) { builder.setColumnFamily(ColumnFamilyDescriptorBuilder.of(family)); } byte[][] splitKeys = getSplitKeys(nRegions); util.createTable(builder.build(), splitKeys); assertEquals((splitKeys.length + 1) * regionReplication, util.getAdmin().getTableRegions(tableName).size()); }