@Override public Object run() throws Exception { runWALSplit(newConf); WAL wal2 = createWAL(newConf, hbaseRootDir, logName); HRegion region2 = HRegion.openHRegion(newConf, FileSystem.get(newConf), hbaseRootDir, hri, htd, wal2); long seqid2 = region2.getOpenSeqNum(); assertTrue(seqid2 > -1); assertEquals(rowsInsertedCount, getScannedCount(region2.getScanner(new Scan()))); // I can't close wal1. Its been appropriated when we split. region2.close(); wal2.close(); return null; } });
protected final void verifyThroughRegion(HBaseTestingUtility util, int start, int end) throws IOException { HRegion region = util.getMiniHBaseCluster().getRegions(TABLE_NAME).get(0); for (int i = start; i < end; i++) { assertEquals(i, Bytes.toInt(region.get(new Get(Bytes.toBytes(i))).getValue(CF, CQ))); } }
/** * Creates, flushes, and closes a new region. */ public static HRegion createHDFSRegionDir(Configuration conf, RegionInfo hri, TableDescriptor htd) throws IOException { // Create HRegion Path root = FSUtils.getRootDir(conf); HRegion region = HRegion.createHRegion(hri, root, conf, htd, null); // Close the new region to flush to disk. Close log file too. region.close(); return region; }
RegionScanner checkScannerOpen(final Scan scan) throws IOException { RegionScanner scanner; if (region.getCoprocessorHost() == null) { scanner = region.getScanner(scan); } else { region.getCoprocessorHost().preScannerOpen(scan); scanner = region.getScanner(scan); scanner = region.getCoprocessorHost().postScannerOpen(scan, scanner); } if (scanner == null) { throw new IOException("Failed to open region scanner"); } return scanner; }
/** Make sure this is a valid row for the HRegion */ void checkRow(byte[] row, String op) throws IOException { if (!rowIsInRange(getRegionInfo(), row)) { throw new WrongRegionException("Requested row out of range for " + op + " on HRegion " + this + ", startKey='" + Bytes.toStringBinary(getRegionInfo().getStartKey()) + "', getEndKey()='" + Bytes.toStringBinary(getRegionInfo().getEndKey()) + "', row='" + Bytes.toStringBinary(row) + "'"); } }
@Before public void setUp() throws Exception { HTableDescriptor htd = new HTableDescriptor( TableName.valueOf(TABLE_NAME_BYTES)); htd.addFamily(new HColumnDescriptor(FAMILY_NAME_BYTES)); HRegionInfo info = new HRegionInfo(htd.getTableName(), null, null, false); this.region = HBaseTestingUtility.createRegionAndWAL(info, TEST_UTIL.getDataTestDir(), TEST_UTIL.getConfiguration(), htd); Put put = new Put(ROW_BYTES); for (int i = 0; i < 10; i += 2) { // puts 0, 2, 4, 6 and 8 put.addColumn(FAMILY_NAME_BYTES, Bytes.toBytes(QUALIFIER_PREFIX + i), (long) i, Bytes.toBytes(VALUE_PREFIX + i)); } this.region.put(put); this.region.flush(true); }
Configuration conf = TEST_UTIL.getConfiguration(); conf.setClass(HConstants.REGION_IMPL, MockHRegion.class, HeapSize.class); HTableDescriptor htd = new HTableDescriptor(TableName.valueOf(name.getMethodName())) .addFamily(new HColumnDescriptor(family)); this.region = TEST_UTIL.createLocalHRegion(htd, null, null); Put[] puts = new Put[1]; Put put = new Put(Bytes.toBytes("r1")); put.addColumn(Bytes.toBytes(family), Bytes.toBytes("q1"), Bytes.toBytes("10")); puts[0] = put; region.batchMutate(puts, HConstants.NO_NONCE, HConstants.NO_NONCE); MultithreadedTestUtil.TestContext ctx = new MultithreadedTestUtil.TestContext(conf); Scan s = new Scan(); RegionScanner scanner = region.getScanner(s); List<Cell> results = new ArrayList<>(); ScannerContext scannerContext = ScannerContext.newBuilder().setBatchLimit(2).build();
@Test public void testAllColumnsWithBloomFilter() throws IOException { byte[] TABLE = Bytes.toBytes(name.getMethodName()); byte[] FAMILY = Bytes.toBytes("family"); HTableDescriptor htd = new HTableDescriptor(TableName.valueOf(TABLE)); htd.addFamily(hcd); HRegionInfo info = new HRegionInfo(htd.getTableName(), null, null, false); this.region = TEST_UTIL.createLocalHRegion(info, htd); byte[] row = Bytes.toBytes("row:" + 0); byte[] column = Bytes.toBytes("column:" + 0); Put put = new Put(row); put.setDurability(Durability.SKIP_WAL); for (long idx = 1; idx <= 4; idx++) { put.addColumn(FAMILY, column, idx, Bytes.toBytes("value-version-" + idx)); region.put(put); region.flush(true); Cell[] kvs = region.get(get).rawCells();
@Test public void testGetScanner_WithOkFamilies() throws IOException { byte[] fam1 = Bytes.toBytes("fam1"); byte[] fam2 = Bytes.toBytes("fam2"); byte[][] families = { fam1, fam2 }; // Setting up region this.region = initHRegion(tableName, method, CONF, families); Scan scan = new Scan(); scan.addFamily(fam1); scan.addFamily(fam2); try { region.getScanner(scan); } catch (Exception e) { assertTrue("Families could not be found in Region", false); } }
@Test public void testToShowNPEOnRegionScannerReseek() throws Exception { byte[] family = Bytes.toBytes("family"); this.region = initHRegion(tableName, method, CONF, family); Put put = new Put(Bytes.toBytes("r1")); put.addColumn(family, Bytes.toBytes("q1"), Bytes.toBytes("v1")); region.put(put); put = new Put(Bytes.toBytes("r2")); put.addColumn(family, Bytes.toBytes("q1"), Bytes.toBytes("v1")); region.put(put); region.flush(true); Scan scan = new Scan(); scan.setMaxVersions(3); // open the first scanner RegionScanner scanner1 = region.getScanner(scan); System.out.println("Smallest read point:" + region.getSmallestReadPoint()); region.compact(true); scanner1.reseek(Bytes.toBytes("r2")); List<Cell> results = new ArrayList<>(); scanner1.next(results); Cell keyValue = results.get(0); Assert.assertTrue(Bytes.compareTo(CellUtil.cloneRow(keyValue), Bytes.toBytes("r2")) == 0); scanner1.close(); }
/** * Setting up a Store * @throws IOException with error */ protected void initialize() throws IOException { Path basedir = new Path(DIR); String logName = "logs"; Path logdir = new Path(DIR, logName); HColumnDescriptor hcd = new HColumnDescriptor(Bytes.toBytes("family")); FileSystem fs = FileSystem.get(conf); fs.delete(logdir, true); HTableDescriptor htd = new HTableDescriptor(TableName.valueOf(Bytes.toBytes("table"))); htd.addFamily(hcd); HRegionInfo info = new HRegionInfo(htd.getTableName(), null, null, false); hlog = new FSHLog(fs, basedir, logName, conf); hlog.init(); ChunkCreator.initialize(MemStoreLABImpl.CHUNK_SIZE_DEFAULT, false, 0, 0, 0, null); region = HRegion.createHRegion(info, basedir, conf, htd, hlog); region.close(); Path tableDir = FSUtils.getTableDir(basedir, htd.getTableName()); region = new HRegion(tableDir, hlog, fs, conf, info, htd, null); store = new HStore(region, hcd, conf); TEST_FILE = region.getRegionFileSystem().createTempName(); fs.createNewFile(TEST_FILE); }
@Test public void testRemovePeer() throws Exception { TableName tableName = createTable(); try (Table table = UTIL.getConnection().getTable(tableName)) { for (int i = 0; i < 100; i++) { table.put(new Put(Bytes.toBytes(i)).addColumn(CF, CQ, Bytes.toBytes(i))); } } enablePeerAndWaitUntilReplicationDone(100); checkOrder(100); String encodedRegionName = UTIL.getMiniHBaseCluster().getRegions(tableName).get(0).getRegionInfo().getEncodedName(); ReplicationQueueStorage queueStorage = UTIL.getMiniHBaseCluster().getMaster().getReplicationPeerManager().getQueueStorage(); assertTrue(queueStorage.getLastSequenceId(encodedRegionName, PEER_ID) > 0); UTIL.getAdmin().removeReplicationPeer(PEER_ID); // confirm that we delete the last pushed sequence id assertEquals(HConstants.NO_SEQNUM, queueStorage.getLastSequenceId(encodedRegionName, PEER_ID)); }
final TableName tableName = TableName.valueOf(name.getMethodName()); Table table = TEST_UTIL.createTable(tableName, Bytes.toBytes("f")); for (int i = 0; i < 100; i++) { table.put(new Put(Bytes.toBytes(i)).addColumn(Bytes.toBytes("f"), Bytes.toBytes("q"), Bytes.toBytes(i))); HRegionServer rs = TEST_UTIL.getRSForFirstRegionInTable(table.getName()); List<HRegion> regions = rs.getRegions(tableName); Assert.assertEquals(1, regions.size()); byte[] regionName = region.getRegionInfo().getRegionName(); HStore store = region.getStore(Bytes.toBytes("f")); long expectedStoreFilesSize = store.getStorefilesSize(); Assert.assertNotNull(store); RegionInfo ri = ProtobufUtil.getRegionInfo(controller, conn.getAdmin(rs.getServerName()), regionName); Assert.assertEquals(region.getRegionInfo(), ri);
@Test public void testSplit() throws IOException, InterruptedException, ExecutionException, TimeoutException { try (Table table = createTable(false)) { table.put(new Put(Bytes.toBytes(0)).addColumn(CF, CQ, Bytes.toBytes(0))); table.put(new Put(Bytes.toBytes(1)).addColumn(CF, CQ, Bytes.toBytes(0))); } UTIL.flush(NAME); HRegionServer rs = UTIL.getRSForFirstRegionInTable(NAME); RegionInfo region = UTIL.getMiniHBaseCluster().getRegions(NAME).get(0).getRegionInfo(); UTIL.getAdmin().splitRegionAsync(region.getRegionName(), Bytes.toBytes(1)).get(1, TimeUnit.MINUTES); long maxSeqId = getMaxSeqId(rs, region); RegionLocator locator = UTIL.getConnection().getRegionLocator(NAME); HRegionLocation locA = locator.getRegionLocation(Bytes.toBytes(0), true); HRegionLocation locB = locator.getRegionLocation(Bytes.toBytes(1), true); assertEquals(maxSeqId + 1, locA.getSeqNum()); assertEquals(maxSeqId + 1, locB.getSeqNum()); }
protected void checkShouldFlush(Configuration conf, boolean expected) throws Exception { try { EnvironmentEdgeForMemstoreTest edge = new EnvironmentEdgeForMemstoreTest(); EnvironmentEdgeManager.injectEdge(edge); HBaseTestingUtility hbaseUtility = HBaseTestingUtility.createLocalHTU(conf); String cf = "foo"; HRegion region = hbaseUtility.createTestRegion("foobar", ColumnFamilyDescriptorBuilder.of(cf)); edge.setCurrentTimeMillis(1234); Put p = new Put(Bytes.toBytes("r")); p.add(KeyValueTestUtil.create("r", cf, "q", 100, "v")); region.put(p); edge.setCurrentTimeMillis(1234 + 100); StringBuilder sb = new StringBuilder(); assertTrue(!region.shouldFlush(sb)); edge.setCurrentTimeMillis(1234 + 10000); assertTrue(region.shouldFlush(sb) == expected); } finally { EnvironmentEdgeManager.reset(); } }
@Before public void setUp() throws Exception { TableDescriptor htd = TableDescriptorBuilder .newBuilder(TableName.valueOf("TestQualifierFilter")) .setColumnFamily(ColumnFamilyDescriptorBuilder.newBuilder(FAMILY).build()).build(); RegionInfo info = RegionInfoBuilder.newBuilder(htd.getTableName()).build(); this.region = HBaseTestingUtility .createRegionAndWAL(info, TEST_UTIL.getDataTestDir(), TEST_UTIL.getConfiguration(), htd); // Insert data for (byte[] ROW : ROWS) { Put p = new Put(ROW); p.setDurability(Durability.SKIP_WAL); for (byte[] QUALIFIER : QUALIFIERS) { p.addColumn(FAMILY, QUALIFIER, VALUE); } this.region.put(p); } // Flush this.region.flush(true); }
@Test public void testBulkLoadReplicationEnabled() throws IOException { TEST_UTIL.getConfiguration().setBoolean(HConstants.REPLICATION_BULKLOAD_ENABLE_KEY, true); final ServerName serverName = ServerName.valueOf(name.getMethodName(), 100, 42); final RegionServerServices rss = spy(TEST_UTIL.createMockRegionServerService(serverName)); HTableDescriptor htd = new HTableDescriptor(TableName.valueOf(name.getMethodName())); htd.addFamily(new HColumnDescriptor(fam1)); HRegionInfo hri = new HRegionInfo(htd.getTableName(), HConstants.EMPTY_BYTE_ARRAY, HConstants.EMPTY_BYTE_ARRAY); region = HRegion.openHRegion(hri, htd, rss.getWAL(hri), TEST_UTIL.getConfiguration(), rss, null); assertTrue(region.conf.getBoolean(HConstants.REPLICATION_BULKLOAD_ENABLE_KEY, false)); String plugins = region.conf.get(CoprocessorHost.REGION_COPROCESSOR_CONF_KEY, ""); String replicationCoprocessorClass = ReplicationObserver.class.getCanonicalName(); assertTrue(plugins.contains(replicationCoprocessorClass)); assertTrue(region.getCoprocessorHost(). getCoprocessors().contains(ReplicationObserver.class.getSimpleName())); }
@Test // HBASE-3516: Test CP Class loading from local file system public void testClassLoadingFromLocalFS() throws Exception { File jarFile = buildCoprocessorJar(cpName3); // create a table that references the jar HTableDescriptor htd = new HTableDescriptor(TableName.valueOf(cpName3)); htd.addFamily(new HColumnDescriptor("test")); htd.setValue("COPROCESSOR$1", getLocalPath(jarFile) + "|" + cpName3 + "|" + Coprocessor.PRIORITY_USER); Admin admin = TEST_UTIL.getAdmin(); admin.createTable(htd); waitForTable(htd.getTableName()); // verify that the coprocessor was loaded boolean found = false; MiniHBaseCluster hbase = TEST_UTIL.getHBaseCluster(); for (HRegion region: hbase.getRegionServer(0).getOnlineRegionsLocalContext()) { if (region.getRegionInfo().getRegionNameAsString().startsWith(cpName3)) { found = (region.getCoprocessorHost().findCoprocessor(cpName3) != null); } } assertTrue("Class " + cpName3 + " was missing on a region", found); }
HRegion initHRegion(byte[] tableName, String callingMethod, Configuration conf, byte[]... families) throws IOException { HTableDescriptor htd = new HTableDescriptor(TableName.valueOf(tableName)); for (byte[] family : families) { htd.addFamily(new HColumnDescriptor(family)); } ChunkCreator.initialize(MemStoreLABImpl.CHUNK_SIZE_DEFAULT, false, 0, 0, 0, null); HRegionInfo info = new HRegionInfo(htd.getTableName(), null, null, false); Path path = new Path(DIR + callingMethod); WAL wal = HBaseTestingUtility.createWal(conf, path, info); HRegion r = HRegion.createHRegion(info, path, conf, htd, wal); // this following piece is a hack. currently a coprocessorHost // is secretly loaded at OpenRegionHandler. we don't really // start a region server here, so just manually create cphost // and set it to region. RegionCoprocessorHost host = new RegionCoprocessorHost(r, null, conf); r.setCoprocessorHost(host); return r; }