@VisibleForTesting void initializeWorkQueues() throws IOException { if (storesToCompact.isEmpty()) { connection.getTable(tableName).getDescriptor().getColumnFamilyNames() .forEach(a -> storesToCompact.add(Bytes.toString(a))); LOG.info("No family specified, will execute for all families"); } LOG.info( "Initializing compaction queues for table: " + tableName + " with cf: " + storesToCompact); List<HRegionLocation> regionLocations = connection.getRegionLocator(tableName).getAllRegionLocations(); for (HRegionLocation location : regionLocations) { Optional<MajorCompactionRequest> request = MajorCompactionRequest .newRequest(connection.getConfiguration(), location.getRegion(), storesToCompact, timestamp); request.ifPresent(majorCompactionRequest -> clusterCompactionQueues .addToCompactionQueue(location.getServerName(), majorCompactionRequest)); } }
@Override public void prepare(final boolean reload) throws IOException { // check table state if this is a retry if (reload && tableName != null && !tableName.equals(TableName.META_TABLE_NAME) && getConnection().isTableDisabled(tableName)) { throw new TableNotEnabledException(tableName.getNameAsString() + " is disabled."); } try (RegionLocator regionLocator = connection.getRegionLocator(tableName)) { this.location = regionLocator.getRegionLocation(row); } if (this.location == null) { throw new IOException("Failed to find location, tableName=" + tableName + ", row=" + Bytes.toString(row) + ", reload=" + reload); } setStubByServiceName(this.location.getServerName()); }
private void generatePartitions(Path partitionsPath) throws IOException { Connection connection = ConnectionFactory.createConnection(getConf()); Pair<byte[][], byte[][]> regionKeys = connection.getRegionLocator(TableName.valueOf(tableHash.tableName)).getStartEndKeys(); connection.close(); tableHash.selectPartitions(regionKeys); LOG.info("Writing " + tableHash.partitions.size() + " partition keys to " + partitionsPath); tableHash.writePartitionFile(getConf(), partitionsPath); }
/** * Return the start keys of all of the regions in this table, * as a list of ImmutableBytesWritable. */ private static List<ImmutableBytesWritable> getRegionStartKeys(List<RegionLocator> regionLocators, boolean writeMultipleTables) throws IOException { ArrayList<ImmutableBytesWritable> ret = new ArrayList<>(); for(RegionLocator regionLocator : regionLocators) { TableName tableName = regionLocator.getName(); LOG.info("Looking up current regions for table " + tableName); byte[][] byteKeys = regionLocator.getStartKeys(); for (byte[] byteKey : byteKeys) { byte[] fullKey = byteKey; //HFileOutputFormat2 use case if (writeMultipleTables) { //MultiTableHFileOutputFormat use case fullKey = combineTableNameSuffix(tableName.getName(), byteKey); } if (LOG.isDebugEnabled()) { LOG.debug("SplitPoint startkey for table [" + tableName + "]: [" + Bytes.toStringBinary (fullKey) + "]"); } ret.add(new ImmutableBytesWritable(fullKey)); } } return ret; }
@Override public List<THRegionLocation> getAllRegionLocations(ByteBuffer table) throws TIOError, TException { RegionLocator locator = null; try { locator = getLocator(table); return ThriftUtilities.regionLocationsFromHBase(locator.getAllRegionLocations()); } catch (IOException e) { throw getTIOError(e); } finally { if (locator != null) { try { locator.close(); } catch (IOException e) { LOG.warn("Couldn't close the locator.", e); } } } }
@Override public THRegionLocation getRegionLocation(ByteBuffer table, ByteBuffer row, boolean reload) throws TIOError, TException { RegionLocator locator = null; try { locator = getLocator(table); byte[] rowBytes = byteBufferToByteArray(row); HRegionLocation hrl = locator.getRegionLocation(rowBytes, reload); return ThriftUtilities.regionLocationFromHBase(hrl); } catch (IOException e) { throw getTIOError(e); } finally { if (locator != null) { try { locator.close(); } catch (IOException e) { LOG.warn("Couldn't close the locator.", e); } } } }
util.startMiniCluster(option); for (String tableStrSingle : tableStr) { byte[][] splitKeys = generateRandomSplitKeys(regionNum - 1); TableName tableName = TableName.valueOf(tableStrSingle); RegionLocator r = util.getConnection().getRegionLocator(tableName); assertEquals("Should start with empty table", 0, util.countRows(table)); int numRegions = r.getStartKeys().length; assertEquals("Should make " + regionNum + " regions", numRegions, regionNum); tableInfo.add(new HFileOutputFormat2.TableInfo(table.getTableDescriptor(), r)); Admin admin = util.getConnection().getAdmin(); try { Table chosenTable = allTables.values().iterator().next(); LOG.info("Changing regions in table " + chosenTable.getName().getNameAsString()); admin.disableTable(chosenTable.getName()); while (util.getConnection().getRegionLocator(chosenTable.getName()) .getAllRegionLocations().size() != 15 || tableInfoSingle.getRegionLocator().close();
Admin admin = connection.getAdmin(); RegionLocator locator = connection.getRegionLocator( TableName.valueOf("testtable")); HRegionLocation location = locator.getRegionLocation( Bytes.toBytes("row-050")); System.out.println("\nUnassigning region: " + location.getRegionInfo(). getRegionNameAsString()); admin.closeRegion(location.getRegionInfo().getRegionName(), null); // co MissingRegionExample-6-Close Close the region containing the row the reading thread is retrieving. Note that unassign() does not work here because the master would automatically reopen the region when the thread is calling the get() method. while (locator.getAllRegionLocations().size() >= 3 && count++ < 10) // co MissingRegionExample-7-Check Use the number of online regions to confirm the close. try { System.out.println("\nWaiting for region to be offline in main()..."); System.out.println("\nAssigning region: " + location.getRegionInfo(). getRegionNameAsString()); admin.assign(location.getRegionInfo().getRegionName()); // co MissingRegionExample-9-Open Open the region, which will make the blocked get() in the thread wake up and print its waiting time. locator.close(); admin.close(); table.close(); connection.close(); helper.close();
Pair<byte[][], byte[][]> keys = UTIL.getConnection() .getRegionLocator(tableName).getStartEndKeys(); assertEquals(2, keys.getFirst().length); byte[][] startKeys = keys.getFirst(); table.put(puts); get.addFamily(HConstants.CATALOG_FAMILY); get.setTimestamp(timestamp[j]); Result result = table.get(get); int cellCount = 0; for(@SuppressWarnings("unused")Cell kv : result.listCells()) { Scan scan = new Scan(); scan.addFamily(HConstants.CATALOG_FAMILY); ResultScanner s = table.getScanner(scan); try { for (Result rr = null; (rr = s.next()) != null;) {
HTU.getAdmin().createTable(htd); TableName toBeDisabledTable = TableName.valueOf( htd = HTU.createTableDescriptor(toBeDisabledTable.toString()); Table table = connection.getTable(tableName); Table tableToBeDisabled = connection.getTable(toBeDisabledTable); (ClusterConnection) connection, Executors.newSingleThreadExecutor(), Integer.MAX_VALUE, fstd); RegionLocator rl = connection.getRegionLocator(toBeDisabledTable); HRegionLocation hrl = rl.getRegionLocation(HConstants.EMPTY_BYTE_ARRAY); byte[] encodedRegionName = hrl.getRegionInfo().getEncodedNameAsBytes(); Cell cell = CellBuilderFactory.create(CellBuilderType.DEEP_COPY).setRow(Bytes.toBytes("A")) .setFamily(HTU.fam1).setValue(Bytes.toBytes("VAL")).setType(Type.Put).build(); Entry entry = new Entry( new WALKeyImpl(encodedRegionName, toBeDisabledTable, 1), table.close(); rl.close(); tableToBeDisabled.close();
final TableName tableName = TableName.valueOf(name.getMethodName()); byte[] cf = "cf".getBytes(); byte[] rk = "rk1".getBytes(); JVMClusterUtil.RegionServerThread rs = TEST_UTIL.getHBaseCluster().startRegionServer(); rs.waitForServerOnline(); final ServerName sn = rs.getRegionServer().getServerName(); Table t = TEST_UTIL.createTable(tableName, cf); TEST_UTIL.waitTableAvailable(tableName); TEST_UTIL.waitUntilNoRegionsInTransition(); final ConnectionImplementation hci = (ConnectionImplementation)TEST_UTIL.getConnection(); try (RegionLocator l = TEST_UTIL.getConnection().getRegionLocator(tableName)) { while (l.getRegionLocation(rk).getPort() != sn.getPort()) { TEST_UTIL.getAdmin().move(l.getRegionLocation(rk).getRegionInfo(). getEncodedNameAsBytes(), Bytes.toBytes(sn.toString())); TEST_UTIL.waitUntilNoRegionsInTransition(); hci.clearRegionCache(tableName); TEST_UTIL.assertRegionOnServer(l.getRegionLocation(rk).getRegionInfo(), sn, 20000); t.put(p1); t.close();
@Test public void testPreWALRestoreSkip() throws Exception { LOG.info(TestRegionObserverInterface.class.getName() + "." + name.getMethodName()); TableName tableName = TableName.valueOf(SimpleRegionObserver.TABLE_SKIPPED); Table table = util.createTable(tableName, new byte[][] { A, B, C }); try (RegionLocator locator = util.getConnection().getRegionLocator(tableName)) { JVMClusterUtil.RegionServerThread rs1 = cluster.startRegionServer(); ServerName sn2 = rs1.getRegionServer().getServerName(); String regEN = locator.getAllRegionLocations().get(0).getRegionInfo().getEncodedName(); util.getAdmin().move(regEN.getBytes(), sn2.getServerName().getBytes()); while (!sn2.equals(locator.getAllRegionLocations().get(0).getServerName())) { Thread.sleep(100); } Put put = new Put(ROW); put.addColumn(A, A, A); put.addColumn(B, B, B); put.addColumn(C, C, C); table.put(put); cluster.killRegionServer(rs1.getRegionServer().getServerName()); Threads.sleep(20000); // just to be sure that the kill has fully started. util.waitUntilAllRegionsAssigned(tableName); } verifyMethodResult(SimpleRegionObserver.class, new String[] { "getCtPreWALRestore", "getCtPostWALRestore", }, tableName, new Integer[] { 0, 0 }); util.deleteTable(tableName); table.close(); }
throw new IOException("A scan object did not have a table name"); TableName tableName = TableName.valueOf(tableNameBytes); TableName tableName = entry.getKey(); List<Scan> scanList = entry.getValue(); try (Table table = conn.getTable(tableName); RegionLocator regionLocator = conn.getRegionLocator(tableName)) { RegionSizeCalculator sizeCalculator = new RegionSizeCalculator( regionLocator, conn.getAdmin()); Pair<byte[][], byte[][]> keys = regionLocator.getStartEndKeys(); for (Scan scan : scanList) { if (keys == null || keys.getFirst() == null || keys.getFirst().length == 0) { throw new IOException("Expecting at least one region for table : " + tableName.getNameAsString()); keys.getSecond()[i] : stopRow; HRegionLocation hregionLocation = regionLocator.getRegionLocation( keys.getFirst()[i], false); String regionHostname = hregionLocation.getHostname(); HRegionInfo regionInfo = hregionLocation.getRegionInfo(); String encodedRegionName = regionInfo.getEncodedName(); long regionSize = sizeCalculator.getRegionSize( regionInfo.getRegionName());
@Test public void testMoveRegion() throws IOException, InterruptedException { String tableNameString = name.getMethodName(); TableName tableName = TableName.valueOf(tableNameString); Table t = TEST_UTIL.createTable(tableName, Bytes.toBytes("D")); TEST_UTIL.waitUntilAllRegionsAssigned(t.getName()); Admin admin = TEST_UTIL.getAdmin(); HRegionInfo regionInfo; byte[] row = Bytes.toBytes("r1"); try (RegionLocator locator = TEST_UTIL.getConnection().getRegionLocator(tableName)) { regionInfo = locator.getRegionLocation(row, true).getRegionInfo(); int currentServerIdx = cluster.getServerWith(regionInfo.getRegionName()); int destServerIdx = (currentServerIdx +1)% cluster.getLiveRegionServerThreads().size(); HRegionServer currentServer = cluster.getRegionServer(currentServerIdx); p.addColumn(Bytes.toBytes("D"), Bytes.toBytes("Zero"), Bytes.toBytes("VALUE")); t.put(p); MetricsRegionAggregateSource currentAgg = currentServer.getRegion(regionInfo.getRegionName()) .getMetrics() .getSource() "_region_" + regionInfo.getEncodedName()+ "_metric";
Admin admin = connection.getAdmin(); BufferedMutator mutator = connection.getBufferedMutator(tableName); for (int a = 'A'; a <= 'Z'; a++) for (int b = 'A'; b <= 'Z'; b++) printRegionInfo(list); System.out.println("Splitting region: " + info.getRegionNameAsString()); admin.splitRegion(info.getRegionName()); // co ClusterOperationExample-04-Split Split the last region this table has, starting at row key "TUV". Adds a new region starting with key "WEI". do { list = admin.getTableRegions(tableName); RegionLocator locator = connection.getRegionLocator(tableName); HRegionLocation location = locator.getRegionLocation(Bytes.toBytes("ZZZ")); // co ClusterOperationExample-06-Cache Retrieve region infos cached and refreshed to show the difference. System.out.println("Found cached region: " + location.getRegionInfo().getRegionNameAsString()); location = locator.getRegionLocation(Bytes.toBytes("ZZZ"), true); System.out.println("Found refreshed region: " + location.getRegionInfo().getRegionNameAsString()); admin.getOnlineRegions(location.getServerName()); online = filterTableRegions(online, tableName); int numOnline = online.size(); locator.close();
final TableName tableName = TableName.valueOf(name.getMethodName()); byte [] family1 = Bytes.toBytes("f1"); byte [] family2 = Bytes.toBytes("f2"); try (Table table = TEST_UTIL.createTable(tableName, new byte[][] {family1, family2}, 10); Admin admin = TEST_UTIL.getAdmin(); RegionLocator locator = TEST_UTIL.getConnection().getRegionLocator(tableName)) { List<HRegionLocation> allRegionLocations = locator.getAllRegionLocations(); assertEquals(1, allRegionLocations.size()); HRegionInfo regionInfo = allRegionLocations.get(0).getRegionInfo(); ServerName addrBefore = allRegionLocations.get(0).getServerName(); HRegionLocation addrCache = locator.getRegionLocation(regionInfo.getStartKey(), false); HRegionLocation addrNoCache = locator.getRegionLocation(regionInfo.getStartKey(), true); assertEquals(addrBefore.getPort(), addrCache.getPort()); assertEquals(addrBefore.getPort(), addrNoCache.getPort()); ServerName addr = regionServer.getServerName(); if (addr.getPort() != addrBefore.getPort()) { admin.move(regionInfo.getEncodedNameAsBytes(), Bytes.toBytes(addr.toString())); addrCache = locator.getRegionLocation(regionInfo.getStartKey(), false); addrNoCache = locator.getRegionLocation(regionInfo.getStartKey(), true); assertNotNull(addrAfter); assertTrue(addrAfter.getPort() != addrCache.getPort());
@BeforeClass public static void setUpBeforeClass() throws Exception { TEST_UTIL.startMiniCluster(3); REST_TEST_UTIL.startServletContainer(TEST_UTIL.getConfiguration()); client = new Client(new Cluster().add("localhost", REST_TEST_UTIL.getServletPort())); TableListModel.class, TableRegionModel.class); TEST_UTIL.createMultiRegionTable(TABLE, Bytes.toBytes(COLUMN_FAMILY), NUM_REGIONS); byte[] k = new byte[3]; byte [][] famAndQf = CellUtil.parseColumn(Bytes.toBytes(COLUMN)); List<Put> puts = new ArrayList<>(); for (byte b1 = 'a'; b1 < 'z'; b1++) { Connection connection = TEST_UTIL.getConnection(); Table table = connection.getTable(TABLE); table.put(puts); table.close(); RegionLocator regionLocator = connection.getRegionLocator(TABLE); List<HRegionLocation> m = regionLocator.getAllRegionLocations(); regionMap = m; LOG.error("regions: " + regionMap); regionLocator.close();
HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility(conf); StartMiniClusterOption option = StartMiniClusterOption.builder() .numMasters(NUM_MASTERS).build(); TEST_UTIL.startMiniCluster(option); MiniHBaseCluster cluster = TEST_UTIL.getHBaseCluster(); log("Waiting for active/ready master"); cluster.waitForActiveAndReadyMaster(); final TableName tableName = TableName.valueOf(name.getMethodName()); byte[] family = Bytes.toBytes("family"); log("Creating table with " + NUM_REGIONS_TO_CREATE + " regions"); Table ht = TEST_UTIL.createMultiRegionTable(tableName, family, NUM_REGIONS_TO_CREATE); int numRegions = -1; try (RegionLocator r = TEST_UTIL.getConnection().getRegionLocator(tableName)) { numRegions = r.getStartKeys().length; TableName.valueOf(name.getMethodName()), TableState.State.DISABLED, TableState.State.DISABLING)); log("Enabling table\n"); " switch except for the catalog table.", 5, regions.size()); assertTrue("The table should be in enabled state", cluster.getMaster().getTableStateManager() .isTableState(TableName.valueOf(name.getMethodName()), TableState.State.ENABLED)); ht.close(); TEST_UTIL.shutdownMiniCluster();
public static void main(String[] args) throws IOException, InterruptedException { Configuration conf = HBaseConfiguration.create(); HBaseHelper helper = HBaseHelper.getHelper(conf); helper.dropTable("testtable"); Connection connection = ConnectionFactory.createConnection(conf); Admin admin = connection.getAdmin(); // vv ServerAndRegionNameExample TableName tableName = TableName.valueOf("testtable"); HColumnDescriptor coldef1 = new HColumnDescriptor("colfam1"); HTableDescriptor desc = new HTableDescriptor(tableName) .addFamily(coldef1) .setValue("Description", "Chapter 5 - ServerAndRegionNameExample"); byte[][] regions = new byte[][] { Bytes.toBytes("ABC"), Bytes.toBytes("DEF"), Bytes.toBytes("GHI"), Bytes.toBytes("KLM"), Bytes.toBytes("OPQ"), Bytes.toBytes("TUV") }; admin.createTable(desc, regions); RegionLocator locator = connection.getRegionLocator(tableName); HRegionLocation location = locator.getRegionLocation(Bytes.toBytes("Foo")); HRegionInfo info = location.getRegionInfo(); System.out.println("Region Name: " + info.getRegionNameAsString()); System.out.println("Server Name: " + location.getServerName()); // ^^ ServerAndRegionNameExample locator.close(); admin.close(); connection.close(); } }
@Test public void testMergeTable() throws Exception { final TableName tableName = TableName.valueOf(name.getMethodName()); TEST_UTIL.createTable(tableName, Bytes.toBytes("f"), splitKeys); TEST_UTIL.waitUntilAllRegionsAssigned(tableName); RegionLocator locator = TEST_UTIL.getConnection().getRegionLocator(tableName); HRegionInfo regionA = locator.getRegionLocation(HConstants.EMPTY_START_ROW).getRegionInfo(); HRegionInfo regionB = locator.getRegionLocation(splitKeys[0]).getRegionInfo(); LOG.info("regionA: " + regionA.getEncodedName() + " with FN: " + fnm.getFavoredNodes(regionA)); LOG.info("regionB: " + regionA.getEncodedName() + " with FN: " + fnm.getFavoredNodes(regionB)); int countOfRegions = MetaTableAccessor.getRegionCount(TEST_UTIL.getConfiguration(), tableName); admin.mergeRegionsAsync(regionA.getEncodedNameAsBytes(), regionB.getEncodedNameAsBytes(), false).get(60, TimeUnit.SECONDS); locator.getRegionLocation(HConstants.EMPTY_START_ROW).getRegionInfo(); List<ServerName> mergedFN = fnm.getFavoredNodes(mergedRegion);