private void move() throws IOException, InterruptedException { RegionInfo region = UTIL.getHBaseCluster().getRegions(tableName).stream().findAny().get().getRegionInfo(); HRegionServer rs = UTIL.getHBaseCluster().getRegionServerThreads().stream().map(t -> t.getRegionServer()) .filter(r -> !r.getOnlineTables().contains(tableName)).findAny().get(); UTIL.getAdmin().move(region.getEncodedNameAsBytes(), Bytes.toBytes(rs.getServerName().getServerName())); while (UTIL.getRSForFirstRegionInTable(tableName) != rs) { Thread.sleep(100); } }
@Test public void testFlushedSequenceIdPersistLoad() throws Exception { Configuration conf = TEST_UTIL.getConfiguration(); int msgInterval = conf.getInt("hbase.regionserver.msginterval", 100); // insert some data into META TableName tableName = TableName.valueOf("testFlushSeqId"); HTableDescriptor desc = new HTableDescriptor(tableName); desc.addFamily(new HColumnDescriptor(Bytes.toBytes("cf"))); Table table = TEST_UTIL.createTable(desc, null); // flush META region TEST_UTIL.flush(TableName.META_TABLE_NAME); // wait for regionserver report Threads.sleep(msgInterval * 2); // record flush seqid before cluster shutdown Map<byte[], Long> regionMapBefore = TEST_UTIL.getHBaseCluster().getMaster().getServerManager() .getFlushedSequenceIdByRegion(); // restart hbase cluster which will cause flushed sequence id persist and reload TEST_UTIL.getMiniHBaseCluster().shutdown(); TEST_UTIL.restartHBaseCluster(2); TEST_UTIL.waitUntilNoRegionsInTransition(); // check equality after reloading flushed sequence id map Map<byte[], Long> regionMapAfter = TEST_UTIL.getHBaseCluster().getMaster().getServerManager() .getFlushedSequenceIdByRegion(); assertTrue(regionMapBefore.equals(regionMapAfter)); }
@BeforeClass public static void setUp() throws Exception { UTIL.startMiniCluster(1); try (Table table = UTIL.createTable(TABLE_NAME, CF)) { for (int i = 0; i < COUNT; i++) { table.put(new Put(Bytes.toBytes(i)).addColumn(CF, CQ, Bytes.toBytes(i))); } } HRI = UTIL.getAdmin().getTableRegions(TABLE_NAME).get(0); STUB = ((ConnectionImplementation) UTIL.getConnection()) .getClient(UTIL.getHBaseCluster().getRegionServer(0).getServerName()); }
private HMaster abortAndWaitForMaster() throws IOException, InterruptedException { cluster.abortMaster(0); cluster.waitOnMaster(0); HMaster master = cluster.startMaster().getMaster(); cluster.waitForActiveAndReadyMaster(); return master; }
/** * Make sure that at least the specified number of region servers * are running * @param num minimum number of region servers that should be running * @return true if we started some servers * @throws IOException */ public boolean ensureSomeRegionServersAvailable(final int num) throws IOException { boolean startedServer = false; MiniHBaseCluster hbaseCluster = getMiniHBaseCluster(); for (int i=hbaseCluster.getLiveRegionServerThreads().size(); i<num; ++i) { LOG.info("Started new server=" + hbaseCluster.startRegionServer()); startedServer = true; } return startedServer; }
private void restartMaster() throws Exception { UTIL.getHBaseCluster().getMaster(0).stop("Stopping to start again"); UTIL.getHBaseCluster().waitOnMaster(0); UTIL.getHBaseCluster().startMaster(); waitForQuotaInitialize(UTIL); }
@Before public void setup() throws Exception { fs = UTIL.getHBaseCluster().getMaster().getMasterFileSystem().getFileSystem(); rootDir = UTIL.getHBaseCluster().getMaster().getMasterFileSystem().getRootDir(); admin = UTIL.getAdmin(); originalTableName = TableName.valueOf("test" + testName.getMethodName()); cloneTableName = TableName.valueOf("test-clone-" + originalTableName); snapshotNameAsString = "snapshot_" + originalTableName; snapshotName = Bytes.toBytes(snapshotNameAsString); originalTable = createTable(originalTableName, TEST_FAM); loadData(originalTable, TEST_FAM); countOriginalTable = countRows(originalTable); System.out.println("Original table has: " + countOriginalTable + " rows"); }
@Test public void testCallQueueTooBigExceptionDoesntTriggerPffe() throws Exception { Admin admin = TEST_UTIL.getAdmin(); HTableDescriptor desc = new HTableDescriptor(TableName.valueOf(Bytes .toBytes(tableName))); desc.addFamily(new HColumnDescriptor(FAMILY)); admin.createTable(desc, Bytes.toBytes("aaaa"), Bytes.toBytes("zzzz"), 3); Configuration conf = TEST_UTIL.getConfiguration(); conf.setLong(HConstants.HBASE_CLIENT_OPERATION_TIMEOUT, 100); conf.setInt(HConstants.HBASE_CLIENT_PAUSE, 500); TEST_UTIL.getHBaseCluster().getRegionServer(0).getRpcServer().getScheduler(); Configuration newConf = HBaseConfiguration.create(TEST_UTIL.getConfiguration()); newConf.setInt("hbase.ipc.server.max.callqueue.length", 0); srs.onConfigurationChange(newConf); try (Table table = connection.getTable(TableName.valueOf(tableName))) { Get get = new Get(new byte[1]); table.get(get);
@Ignore @Test public void testAllFavoredNodesDeadMasterRestarted() throws Exception { final RegionInfo region = admin.getTableRegions(tableName).get(0); LOG.info("Region that's supposed to be in transition: " + region); FavoredNodesManager fnm = master.getFavoredNodesManager(); TEST_UTIL.waitFor(10000, new Waiter.Predicate<Exception>() { @Override public boolean evaluate() throws Exception { Configuration conf = cluster.getConf(); conf.setInt(ServerManager.WAIT_ON_REGIONSERVERS_MINTOSTART, SLAVES - FavoredNodeAssignmentHelper.FAVORED_NODES_NUM); cluster.stopMaster(master.getServerName()); cluster.waitForMasterToStop(master.getServerName(), 60000); cluster.startMaster(); cluster.waitForActiveAndReadyMaster(); master = cluster.getMaster(); fnm = master.getFavoredNodesManager(); admin.assign(regionInfo.getEncodedNameAsBytes()); TEST_UTIL.waitUntilNoRegionsInTransition(60000); assertEquals("Not all regions are online", REGION_NUM, admin.getTableRegions(tableName).size());
@Test public void testFlushedFileWithVisibilityTags() throws Exception { final byte[] qual2 = Bytes.toBytes("qual2"); TableName tableName = TableName.valueOf(TEST_NAME.getMethodName()); HTableDescriptor desc = new HTableDescriptor(tableName); HColumnDescriptor col = new HColumnDescriptor(fam); desc.addFamily(col); TEST_UTIL.getAdmin().createTable(desc); try (Table table = TEST_UTIL.getConnection().getTable(tableName)) { Put p1 = new Put(row1); p1.addColumn(fam, qual, value); p1.setCellVisibility(new CellVisibility(CONFIDENTIAL)); Put p2 = new Put(row1); p2.addColumn(fam, qual2, value); p2.setCellVisibility(new CellVisibility(SECRET)); RowMutations rm = new RowMutations(row1); rm.add(p1); rm.add(p2); table.mutateRow(rm); } TEST_UTIL.getAdmin().flush(tableName); List<HRegion> regions = TEST_UTIL.getHBaseCluster().getRegions(tableName); HStore store = regions.get(0).getStore(fam); Collection<HStoreFile> storefiles = store.getStorefiles(); assertTrue(storefiles.size() > 0); for (HStoreFile storeFile : storefiles) { assertTrue(storeFile.getReader().getHFileReader().getFileContext().isIncludesTags()); } }
@Test public void testRecoveryAndDoubleExecutionOnline() throws Exception { final TableName tableName = TableName.valueOf(name.getMethodName()); final String cf2 = "cf2"; final String cf3 = "cf3"; HTableDescriptor htd = new HTableDescriptor(UTIL.getAdmin().getTableDescriptor(tableName)); boolean newCompactionEnableOption = htd.isCompactionEnabled() ? false : true; htd.setCompactionEnabled(newCompactionEnableOption); htd.addFamily(new HColumnDescriptor(cf2)); htd.removeFamily(Bytes.toBytes(cf3)); HTableDescriptor currentHtd = UTIL.getAdmin().getTableDescriptor(tableName); assertEquals(newCompactionEnableOption, currentHtd.isCompactionEnabled()); assertEquals(2, currentHtd.getFamiliesKeys().size()); assertTrue(currentHtd.hasFamily(Bytes.toBytes(cf2))); assertFalse(currentHtd.hasFamily(Bytes.toBytes(cf3))); MasterProcedureTestingUtility.validateTableCreation(UTIL.getHBaseCluster().getMaster(), tableName, regions, "cf1", cf2);
@Test // HBASE-6308: Test CP classloader is the CoprocessorClassLoader public void testPrivateClassLoader() throws Exception { File jarFile = buildCoprocessorJar(cpName4); // create a table that references the jar HTableDescriptor htd = new HTableDescriptor(TableName.valueOf(cpName4)); htd.addFamily(new HColumnDescriptor("test")); htd.setValue("COPROCESSOR$1", getLocalPath(jarFile) + "|" + cpName4 + "|" + Coprocessor.PRIORITY_USER); Admin admin = TEST_UTIL.getAdmin(); admin.createTable(htd); waitForTable(htd.getTableName()); // verify that the coprocessor was loaded correctly boolean found = false; MiniHBaseCluster hbase = TEST_UTIL.getHBaseCluster(); for (HRegion region: hbase.getRegionServer(0).getOnlineRegionsLocalContext()) { if (region.getRegionInfo().getRegionNameAsString().startsWith(cpName4)) { Coprocessor cp = region.getCoprocessorHost().findCoprocessor(cpName4); if (cp != null) { found = true; assertEquals("Class " + cpName4 + " was not loaded by CoprocessorClassLoader", cp.getClass().getClassLoader().getClass(), CoprocessorClassLoader.class); } } } assertTrue("Class " + cpName4 + " was missing on a region", found); }
createNamespace(TEST_UTIL, desc); final TableName table = TableName.valueOf(ns, "testACLZNodeDeletionTable"); final byte[] family = Bytes.toBytes("f1"); HTableDescriptor htd = new HTableDescriptor(table); htd.addFamily(new HColumnDescriptor(family)); createTable(TEST_UTIL, htd); ZKWatcher zkw = TEST_UTIL.getMiniHBaseCluster().getMaster().getZooKeeper(); assertTrue("The acl znode for table should exist", ZKUtil.checkExists(zkw, baseAclZNode + table.getNameAsString()) != -1); assertTrue("The acl znode for namespace should exist", ZKUtil.checkExists(zkw, baseAclZNode + convertToNamespace(ns)) != -1); ZKUtil.checkExists(zkw, baseAclZNode + table.getNameAsString()) == -1); assertTrue( "The acl znode for namespace should have been deleted", ZKUtil.checkExists(zkw, baseAclZNode + convertToNamespace(ns)) == -1);
@Before public void setup() throws Exception { TEST_UTIL = new HBaseTestingUtility(); TEST_UTIL.getConfiguration().setBoolean(HBASE_SPLIT_WAL_COORDINATED_BY_ZK, false); TEST_UTIL.getConfiguration().setInt(HBASE_SPLIT_WAL_MAX_SPLITTER, 1); TEST_UTIL.startMiniCluster(3); master = TEST_UTIL.getHBaseCluster().getMaster(); splitWALManager = master.getSplitWALManager(); TABLE_NAME = TableName.valueOf(Bytes.toBytes("TestSplitWALProcedure")); FAMILY = Bytes.toBytes("test"); }
@Test public void testRegionObserverAfterRegionClosed() throws IOException { final TableName tableName = TableName.valueOf(name.getMethodName()); try (Connection connection = ConnectionFactory.createConnection(UTIL.getConfiguration()); Admin admin = connection.getAdmin()) { admin.createTable( new HTableDescriptor(tableName) .addFamily(new HColumnDescriptor(foo)) .addCoprocessor(CustomRegionObserver.class.getName()) , new byte[][]{foo}); // create with 2 regions try (Table table = connection.getTable(tableName)) { admin.unassign(loc.getRegionInfo().getEncodedNameAsBytes(), true); HRegionServer server = UTIL.getMiniHBaseCluster().getRegionServer(loc.getServerName()); UTIL.waitFor(30000, () -> server.getOnlineRegion(loc.getRegionInfo().getRegionName()) == null); assertNull(server.getOnlineRegion(loc.getRegionInfo().getRegionName())); admin.disableTable(tableName);
@Test public void testMergeTable() throws Exception { final TableName tableName = TableName.valueOf(name.getMethodName()); TEST_UTIL.createTable(tableName, Bytes.toBytes("f"), splitKeys); TEST_UTIL.waitUntilAllRegionsAssigned(tableName); RegionLocator locator = TEST_UTIL.getConnection().getRegionLocator(tableName); HRegionInfo regionA = locator.getRegionLocation(HConstants.EMPTY_START_ROW).getRegionInfo(); HRegionInfo regionB = locator.getRegionLocation(splitKeys[0]).getRegionInfo(); LOG.info("regionA: " + regionA.getEncodedName() + " with FN: " + fnm.getFavoredNodes(regionA)); LOG.info("regionB: " + regionA.getEncodedName() + " with FN: " + fnm.getFavoredNodes(regionB)); int countOfRegions = MetaTableAccessor.getRegionCount(TEST_UTIL.getConfiguration(), tableName); admin.mergeRegionsAsync(regionA.getEncodedNameAsBytes(), regionB.getEncodedNameAsBytes(), false).get(60, TimeUnit.SECONDS); TEST_UTIL.getMiniHBaseCluster().compact(tableName, true); assertEquals("Merge parents should have been cleaned", 1, admin.runCatalogScan()); TEST_UTIL.getMiniHBaseCluster().getMaster().getMasterProcedureExecutor()); assertNull("Parent FN should be null", fnm.getFavoredNodes(regionA)); assertNull("Parent FN should be null", fnm.getFavoredNodes(regionB)); List<HRegionInfo> regions = admin.getTableRegions(tableName); TEST_UTIL.deleteTable(tableName);
@Test public void testMoveThrowsPleaseHoldException() throws IOException { final TableName tableName = TableName.valueOf(name.getMethodName()); HMaster master = TEST_UTIL.getMiniHBaseCluster().getMaster(); HTableDescriptor htd = new HTableDescriptor(tableName); HColumnDescriptor hcd = new HColumnDescriptor("value"); htd.addFamily(hcd); admin.createTable(htd, null); try { List<RegionInfo> tableRegions = admin.getRegions(tableName); master.setInitialized(false); // fake it, set back later admin.move(tableRegions.get(0).getEncodedNameAsBytes(), null); fail("Region should not be moved since master is not initialized"); } catch (IOException ioe) { assertTrue(StringUtils.stringifyException(ioe).contains("PleaseHoldException")); } finally { master.setInitialized(true); TEST_UTIL.deleteTable(tableName); } }
@Test public void testKillRS() throws Exception { TableName tableName = TableName.valueOf(name.getMethodName()); UTIL.getAdmin().createTable( TableDescriptorBuilder.newBuilder(tableName).setColumnFamily(ColumnFamilyDescriptorBuilder .newBuilder(CF).setScope(HConstants.REPLICATION_SCOPE_GLOBAL).build()).build()); UTIL.waitTableAvailable(tableName); try (Table table = UTIL.getConnection().getTable(tableName)) { for (int i = 0; i < 100; i++) { table.put(new Put(Bytes.toBytes(i)).addColumn(CF, CQ, Bytes.toBytes(i))); } } RegionServerThread thread = UTIL.getMiniHBaseCluster().getRegionServerThreads().stream() .filter(t -> !t.getRegionServer().getRegions(tableName).isEmpty()).findFirst().get(); thread.getRegionServer().abort("for testing"); thread.join(); try (Table table = UTIL.getConnection().getTable(tableName)) { for (int i = 100; i < 200; i++) { table.put(new Put(Bytes.toBytes(i)).addColumn(CF, CQ, Bytes.toBytes(i))); } } enablePeerAndWaitUntilReplicationDone(200); checkOrder(200); } }
protected void addHFilesToRegions() throws IOException { MasterFileSystem mfs = HTU.getMiniHBaseCluster().getMaster().getMasterFileSystem(); Path tableDir = FSUtils.getTableDir(mfs.getRootDir(), TABLE_NAME); for (Region region : cluster.getRegions(TABLE_NAME)) { Path regionDir = new Path(tableDir, region.getRegionInfo().getEncodedName()); Path familyDir = new Path(regionDir, Bytes.toString(FAMILY)); HFileTestUtil .createHFile(HTU.getConfiguration(), HTU.getTestFileSystem(), new Path(familyDir, HFILE_NAME), FAMILY, QUALIFIER, Bytes.toBytes("50"), Bytes.toBytes("60"), NUM_ROWS); } } }
final TableName tableName = TableName.valueOf(name.getMethodName()); byte [] family1 = Bytes.toBytes("f1"); byte [] family2 = Bytes.toBytes("f2"); try (Table table = TEST_UTIL.createTable(tableName, new byte[][] {family1, family2}, 10); Admin admin = TEST_UTIL.getAdmin(); RegionLocator locator = TEST_UTIL.getConnection().getRegionLocator(tableName)) { List<HRegionLocation> allRegionLocations = locator.getAllRegionLocations(); assertEquals(1, allRegionLocations.size()); ServerName addrBefore = allRegionLocations.get(0).getServerName(); HRegionLocation addrCache = locator.getRegionLocation(regionInfo.getStartKey(), false); HRegionLocation addrNoCache = locator.getRegionLocation(regionInfo.getStartKey(), true); HRegionServer regionServer = TEST_UTIL.getHBaseCluster().getRegionServer(i); ServerName addr = regionServer.getServerName(); if (addr.getPort() != addrBefore.getPort()) { admin.move(regionInfo.getEncodedNameAsBytes(), Bytes.toBytes(addr.toString()));