@BeforeClass public static void before() throws Exception { // Reduce the hdfs block size and prefetch to trigger the file-link reopen // when the file is moved to archive (e.g. compaction) HTU.getConfiguration().setInt(DFSConfigKeys.DFS_BLOCK_SIZE_KEY, 8192); HTU.getConfiguration().setInt(DFSConfigKeys.DFS_CLIENT_READ_PREFETCH_SIZE_KEY, 1); HTU.getConfiguration().setInt(HConstants.HREGION_MEMSTORE_FLUSH_SIZE, 128 * 1024 * 1024); HTU.startMiniCluster(NB_SERVERS); final TableName tableName = TableName.valueOf(TestRegionReplicas.class.getSimpleName()); // Create table then get the single region for our new table. table = HTU.createTable(tableName, f); try (RegionLocator locator = HTU.getConnection().getRegionLocator(tableName)) { hriPrimary = locator.getRegionLocation(row, false).getRegionInfo(); } // mock a secondary region info to open hriSecondary = new HRegionInfo(hriPrimary.getTable(), hriPrimary.getStartKey(), hriPrimary.getEndKey(), hriPrimary.isSplit(), hriPrimary.getRegionId(), 1); // No master TestRegionServerNoMaster.stopMasterAndAssignMeta(HTU); }
return false; if (metaLocation.getRegionInfo() == null) { errors.reportError(ERROR_CODE.NULL_META_REGION, "META location regionInfo is null"); return false; if (metaLocation.getHostname() == null) { errors.reportError(ERROR_CODE.NULL_META_REGION, "META location hostName is null"); return false; ServerName sn = metaLocation.getServerName(); MetaEntry m = new MetaEntry(metaLocation.getRegionInfo(), sn, EnvironmentEdgeManager.currentTime()); HbckInfo hbckInfo = regionInfoMap.get(metaLocation.getRegionInfo().getEncodedName()); if (hbckInfo == null) { regionInfoMap.put(metaLocation.getRegionInfo().getEncodedName(), new HbckInfo(m)); } else { hbckInfo.metaEntry = m;
/** * Makes some table with given region names. * */ private RegionLocator mockRegionLocator(String... regionNames) throws IOException { RegionLocator mockedTable = Mockito.mock(RegionLocator.class); when(mockedTable.getName()).thenReturn(TableName.valueOf("sizeTestTable")); List<HRegionLocation> regionLocations = new ArrayList<>(regionNames.length); when(mockedTable.getAllRegionLocations()).thenReturn(regionLocations); for (String regionName : regionNames) { HRegionInfo info = Mockito.mock(HRegionInfo.class); when(info.getRegionName()).thenReturn(regionName.getBytes()); regionLocations.add(new HRegionLocation(info, sn)); } return mockedTable; }
@Override public void notifyFinal(ReturnCode code, HRegionLocation loc, long heapSizeOfRow) { if (code == ReturnCode.INCLUDE) { regionsIncluded.add(loc.getRegionInfo()); serversIncluded.add(loc.getServerName()); } busyRegions.add(loc.getRegionInfo().getRegionName()); } }
/** * @return String made of hostname and port formatted as * per {@link Addressing#createHostAndPortStr(String, int)} */ public String getHostnamePort() { return Addressing.createHostAndPortStr(this.getHostname(), this.getPort()); }
@Override void add(Result r) { if (current == null) { return; } for (HRegionLocation loc : current.getRegionLocations()) { if (loc != null) { this.results.add(new Pair<>(loc.getRegion(), loc.getServerName())); } } } };
final TableName tableName = TableName.valueOf(name.getMethodName()); table = TEST_UTIL.createTable(tableName, FAMILIES_1, 1, 1024, CustomInnerRegionObserver.class.getName()); RegionLocator locator = TEST_UTIL.getConnection().getRegionLocator(tableName); String regionName = locator.getAllRegionLocations().get(0).getRegionInfo().getEncodedName(); HRegion region = TEST_UTIL.getRSForFirstRegionInTable(tableName).getRegion(regionName); HStore store = region.getStores().iterator().next(); CacheConfig cacheConf = store.getCacheConfig(); table.put(put); region.flush(true); byte[] QUALIFIER2 = Bytes.add(QUALIFIER, QUALIFIER); put = new Put(ROW); put.addColumn(FAMILY, QUALIFIER2, data2);
throw new IOException("A scan object did not have a table name"); TableName tableName = TableName.valueOf(tableNameBytes); Bytes.compareTo(startRow, keys.getSecond()[i]) < 0) && (stopRow.length == 0 || Bytes.compareTo(stopRow, keys.getFirst()[i]) > 0)) { byte[] splitStart = startRow.length == 0 || Bytes.compareTo(keys.getFirst()[i], startRow) >= 0 ? keys.getFirst()[i] : startRow; byte[] splitStop = (stopRow.length == 0 || keys.getSecond()[i] : stopRow; HRegionLocation hregionLocation = regionLocator.getRegionLocation( keys.getFirst()[i], false); String regionHostname = hregionLocation.getHostname(); HRegionInfo regionInfo = hregionLocation.getRegionInfo(); String encodedRegionName = regionInfo.getEncodedName(); long regionSize = sizeCalculator.getRegionSize( regionInfo.getRegionName());
final TableName tableName = TableName.valueOf(name.getMethodName()); byte [] family1 = Bytes.toBytes("f1"); byte [] family2 = Bytes.toBytes("f2"); try (Table table = TEST_UTIL.createTable(tableName, new byte[][] {family1, family2}, 10); Admin admin = TEST_UTIL.getAdmin(); RegionLocator locator = TEST_UTIL.getConnection().getRegionLocator(tableName)) { List<HRegionLocation> allRegionLocations = locator.getAllRegionLocations(); assertEquals(1, allRegionLocations.size()); HRegionInfo regionInfo = allRegionLocations.get(0).getRegionInfo(); ServerName addrBefore = allRegionLocations.get(0).getServerName(); HRegionLocation addrCache = locator.getRegionLocation(regionInfo.getStartKey(), false); HRegionLocation addrNoCache = locator.getRegionLocation(regionInfo.getStartKey(), true); assertEquals(addrBefore.getPort(), addrCache.getPort()); assertEquals(addrBefore.getPort(), addrNoCache.getPort()); ServerName addr = regionServer.getServerName(); if (addr.getPort() != addrBefore.getPort()) { admin.move(regionInfo.getEncodedNameAsBytes(), Bytes.toBytes(addr.toString())); addrNoCache = locator.getRegionLocation(regionInfo.getStartKey(), true); assertNotNull(addrAfter); assertTrue(addrAfter.getPort() != addrCache.getPort()); assertEquals(addrAfter.getPort(), addrNoCache.getPort());
@Test public void testMergeTable() throws Exception { final TableName tableName = TableName.valueOf(name.getMethodName()); TEST_UTIL.createTable(tableName, Bytes.toBytes("f"), splitKeys); TEST_UTIL.waitUntilAllRegionsAssigned(tableName); RegionLocator locator = TEST_UTIL.getConnection().getRegionLocator(tableName); HRegionInfo regionA = locator.getRegionLocation(HConstants.EMPTY_START_ROW).getRegionInfo(); HRegionInfo regionB = locator.getRegionLocation(splitKeys[0]).getRegionInfo(); LOG.info("regionA: " + regionA.getEncodedName() + " with FN: " + fnm.getFavoredNodes(regionA)); LOG.info("regionB: " + regionA.getEncodedName() + " with FN: " + fnm.getFavoredNodes(regionB)); int countOfRegions = MetaTableAccessor.getRegionCount(TEST_UTIL.getConfiguration(), tableName); admin.mergeRegionsAsync(regionA.getEncodedNameAsBytes(), regionB.getEncodedNameAsBytes(), false).get(60, TimeUnit.SECONDS); locator.getRegionLocation(HConstants.EMPTY_START_ROW).getRegionInfo(); List<ServerName> mergedFN = fnm.getFavoredNodes(mergedRegion);
@Test public void testCompactionTimestamps() throws Exception { HColumnDescriptor fam1 = new HColumnDescriptor("fam1"); final TableName tableName = TableName.valueOf(name.getMethodName()); HTableDescriptor htd = new HTableDescriptor(tableName); htd.addFamily(fam1); this.admin.createTable(htd); Table table = TEST_UTIL.getConnection().getTable(htd.getTableName()); long ts = this.admin.getLastMajorCompactionTimestamp(tableName); assertEquals(0, ts); Put p = new Put(Bytes.toBytes("row1")); p.addColumn(Bytes.toBytes("fam1"), Bytes.toBytes("fam1"), Bytes.toBytes("fam1")); table.put(p); ts = this.admin.getLastMajorCompactionTimestamp(tableName); try (RegionLocator l = TEST_UTIL.getConnection().getRegionLocator(tableName)) { regionName = l.getAllRegionLocations().get(0).getRegionInfo().getRegionName(); TEST_UTIL.compact(tableName, true); table.put(p);
@Test public void test() throws Exception { TableName tableName = TableName.valueOf("SplitMerge"); byte[] family = Bytes.toBytes("CF"); TableDescriptor td = TableDescriptorBuilder.newBuilder(tableName) .setColumnFamily(ColumnFamilyDescriptorBuilder.of(family)).build(); UTIL.getAdmin().createTable(td, new byte[][] { Bytes.toBytes(1) }); UTIL.waitTableAvailable(tableName); UTIL.getAdmin().split(tableName, Bytes.toBytes(2)); UTIL.waitFor(30000, new ExplainingPredicate<Exception>() { ServerName expected = UTIL.getMiniHBaseCluster().getRegionServer(0).getServerName(); assertEquals(expected, UTIL.getConnection().getRegionLocator(tableName) .getRegionLocation(Bytes.toBytes(1), true).getServerName()); try (AsyncConnection asyncConn = ConnectionFactory.createAsyncConnection(UTIL.getConfiguration()).get()) { assertEquals(expected, asyncConn.getRegionLocator(tableName) .getRegionLocation(Bytes.toBytes(1), true).get().getServerName());
@Test public void testMoveRegion() throws IOException, InterruptedException { String tableNameString = name.getMethodName(); TableName tableName = TableName.valueOf(tableNameString); Table t = TEST_UTIL.createTable(tableName, Bytes.toBytes("D")); TEST_UTIL.waitUntilAllRegionsAssigned(t.getName()); Admin admin = TEST_UTIL.getAdmin(); HRegionInfo regionInfo; byte[] row = Bytes.toBytes("r1"); try (RegionLocator locator = TEST_UTIL.getConnection().getRegionLocator(tableName)) { regionInfo = locator.getRegionLocation(row, true).getRegionInfo(); int currentServerIdx = cluster.getServerWith(regionInfo.getRegionName()); int destServerIdx = (currentServerIdx +1)% cluster.getLiveRegionServerThreads().size(); HRegionServer currentServer = cluster.getRegionServer(currentServerIdx); p.addColumn(Bytes.toBytes("D"), Bytes.toBytes("Zero"), Bytes.toBytes("VALUE")); t.put(p); MetricsRegionAggregateSource currentAgg = currentServer.getRegion(regionInfo.getRegionName()) .getMetrics() .getSource() "_region_" + regionInfo.getEncodedName()+ "_metric";
final TableName tableName = TableName.valueOf(name.getMethodName()); byte[] cf = "cf".getBytes(); byte[] rk = "rk1".getBytes(); JVMClusterUtil.RegionServerThread rs = TEST_UTIL.getHBaseCluster().startRegionServer(); rs.waitForServerOnline(); final ServerName sn = rs.getRegionServer().getServerName(); Table t = TEST_UTIL.createTable(tableName, cf); TEST_UTIL.waitTableAvailable(tableName); TEST_UTIL.waitUntilNoRegionsInTransition(); final ConnectionImplementation hci = (ConnectionImplementation)TEST_UTIL.getConnection(); try (RegionLocator l = TEST_UTIL.getConnection().getRegionLocator(tableName)) { while (l.getRegionLocation(rk).getPort() != sn.getPort()) { TEST_UTIL.getAdmin().move(l.getRegionLocation(rk).getRegionInfo(). getEncodedNameAsBytes(), Bytes.toBytes(sn.toString())); TEST_UTIL.waitUntilNoRegionsInTransition(); hci.clearRegionCache(tableName); TEST_UTIL.assertRegionOnServer(l.getRegionLocation(rk).getRegionInfo(), sn, 20000);
@Test public void testGetRegion() throws Exception { // We use actual HBaseAdmin instance instead of going via Admin interface in // here because makes use of an internal HBA method (TODO: Fix.). HBaseAdmin rawAdmin = TEST_UTIL.getHBaseAdmin(); final TableName tableName = TableName.valueOf(name.getMethodName()); LOG.info("Started " + tableName); Table t = TEST_UTIL.createMultiRegionTable(tableName, HConstants.CATALOG_FAMILY); try (RegionLocator locator = TEST_UTIL.getConnection().getRegionLocator(tableName)) { HRegionLocation regionLocation = locator.getRegionLocation(Bytes.toBytes("mmm")); RegionInfo region = regionLocation.getRegionInfo(); byte[] regionName = region.getRegionName(); Pair<RegionInfo, ServerName> pair = rawAdmin.getRegion(regionName); assertTrue(Bytes.equals(regionName, pair.getFirst().getRegionName())); pair = rawAdmin.getRegion(region.getEncodedNameAsBytes()); assertTrue(Bytes.equals(regionName, pair.getFirst().getRegionName())); } }
@Test public void testSplitTable() throws Exception { final TableName tableName = TableName.valueOf(name.getMethodName()); Table t = TEST_UTIL.createTable(tableName, Bytes.toBytes("f"), splitKeys); TEST_UTIL.waitUntilAllRegionsAssigned(tableName); final int numberOfRegions = admin.getTableRegions(t.getName()).size(); byte[] splitPoint = Bytes.toBytes(0); RegionLocator locator = TEST_UTIL.getConnection().getRegionLocator(tableName); HRegionInfo parent = locator.getRegionLocation(splitPoint).getRegionInfo(); List<ServerName> parentFN = fnm.getFavoredNodes(parent); assertNotNull("FN should not be null for region: " + parent, parentFN); HRegionInfo daughter1 = locator.getRegionLocation(parent.getStartKey(), true).getRegionInfo(); List<ServerName> daughter1FN = fnm.getFavoredNodes(daughter1); HRegionInfo daughter2 = locator.getRegionLocation(splitPoint, true).getRegionInfo(); List<ServerName> daughter2FN = fnm.getFavoredNodes(daughter2);
@Test public void testPreWALRestoreSkip() throws Exception { LOG.info(TestRegionObserverInterface.class.getName() + "." + name.getMethodName()); TableName tableName = TableName.valueOf(SimpleRegionObserver.TABLE_SKIPPED); Table table = util.createTable(tableName, new byte[][] { A, B, C }); try (RegionLocator locator = util.getConnection().getRegionLocator(tableName)) { JVMClusterUtil.RegionServerThread rs1 = cluster.startRegionServer(); ServerName sn2 = rs1.getRegionServer().getServerName(); String regEN = locator.getAllRegionLocations().get(0).getRegionInfo().getEncodedName(); util.getAdmin().move(regEN.getBytes(), sn2.getServerName().getBytes()); while (!sn2.equals(locator.getAllRegionLocations().get(0).getServerName())) { Thread.sleep(100); } Put put = new Put(ROW); put.addColumn(A, A, A); put.addColumn(B, B, B); put.addColumn(C, C, C); table.put(put); cluster.killRegionServer(rs1.getRegionServer().getServerName()); Threads.sleep(20000); // just to be sure that the kill has fully started. util.waitUntilAllRegionsAssigned(tableName); } verifyMethodResult(SimpleRegionObserver.class, new String[] { "getCtPreWALRestore", "getCtPostWALRestore", }, tableName, new Integer[] { 0, 0 }); util.deleteTable(tableName); table.close(); }
final int nextPort = loc.getPort() + 1; conn.updateCachedLocation(loc.getRegionInfo(), loc.getServerName(), ServerName.valueOf("127.0.0.1", nextPort, HConstants.LATEST_TIMESTAMP), HConstants.LATEST_TIMESTAMP); Assert.assertEquals(conn.getCachedLocation(TABLE_NAME, ROW) .getRegionLocation().getPort(), nextPort); byte[] regionName = toMove.getRegionInfo().getRegionName(); byte[] encodedRegionNameBytes = toMove.getRegionInfo().getEncodedNameAsBytes(); Assert.assertFalse( toMove.getPort() == destServerName.getPort()); Assert.assertNotNull(curServer.getOnlineRegion(regionName)); Assert.assertNull(destServer.getOnlineRegion(regionName)); LOG.info("Move starting region="+toMove.getRegionInfo().getRegionNameAsString()); TEST_UTIL.getAdmin().move( toMove.getRegionInfo().getEncodedNameAsBytes(), destServerName.getServerName().getBytes() ); LOG.info("Move finished for region="+toMove.getRegionInfo().getRegionNameAsString()); .getPort() == destServerName.getPort()); "Previous server was " + curServer.getServerName().getHostAndPort(), destServerName.getPort(), conn.getCachedLocation(TABLE_NAME, ROW).getRegionLocation().getPort()); LOG.info("Move starting region=" + toMove.getRegionInfo().getRegionNameAsString());
@Test public void testRegionObserverAfterRegionClosed() throws IOException { final TableName tableName = TableName.valueOf(name.getMethodName()); try (Connection connection = ConnectionFactory.createConnection(UTIL.getConfiguration()); Admin admin = connection.getAdmin()) { admin.createTable( new HTableDescriptor(tableName) try (Table table = connection.getTable(tableName)) { table.get(new Get(foo)); table.get(new Get(foo)); // 2 gets try (RegionLocator locator = connection.getRegionLocator(tableName)) { HRegionLocation loc = locator.getRegionLocation(foo); admin.unassign(loc.getRegionInfo().getEncodedNameAsBytes(), true); HRegionServer server = UTIL.getMiniHBaseCluster().getRegionServer(loc.getServerName()); UTIL.waitFor(30000, () -> server.getOnlineRegion(loc.getRegionInfo().getRegionName()) == null); assertNull(server.getOnlineRegion(loc.getRegionInfo().getRegionName()));
@Test public void testRegionObserverMultiRegion() throws IOException { final TableName tableName = TableName.valueOf(name.getMethodName()); try (Connection connection = ConnectionFactory.createConnection(UTIL.getConfiguration()); Admin admin = connection.getAdmin()) { admin.createTable( new HTableDescriptor(tableName) .addFamily(new HColumnDescriptor(foo)) // add the coprocessor for the region .addCoprocessor(CustomRegionObserver.class.getName()) , new byte[][]{foo}); // create with 2 regions try (Table table = connection.getTable(tableName); RegionLocator locator = connection.getRegionLocator(tableName)) { table.get(new Get(bar)); table.get(new Get(foo)); // 2 gets to 2 separate regions assertEquals(2, locator.getAllRegionLocations().size()); assertNotEquals(locator.getRegionLocation(bar).getRegionInfo(), locator.getRegionLocation(foo).getRegionInfo()); } } assertPreGetRequestsCounter(CustomRegionObserver.class); }