@Before public void setUp() throws IOException { UTIL.getConfiguration().setLong(StoreScanner.STORESCANNER_PREAD_MAX_BYTES, 2048); StringBuilder sb = new StringBuilder(256); for (int i = 0; i < 255; i++) { sb.append((char) ThreadLocalRandom.current().nextInt('A', 'z' + 1)); } VALUE_PREFIX = sb.append("-").toString(); REGION = UTIL.createLocalHRegion( TableDescriptorBuilder.newBuilder(TABLE_NAME) .setColumnFamily( ColumnFamilyDescriptorBuilder.newBuilder(FAMILY).setBlocksize(1024).build()) .build(), null, null); for (int i = 0; i < 900; i++) { REGION .put(new Put(Bytes.toBytes(i)).addColumn(FAMILY, QUAL, Bytes.toBytes(VALUE_PREFIX + i))); } REGION.flush(true); for (int i = 900; i < 1000; i++) { REGION .put(new Put(Bytes.toBytes(i)).addColumn(FAMILY, QUAL, Bytes.toBytes(VALUE_PREFIX + i))); } }
HConstants.DEFAULT_HBASE_META_VERSIONS)) .setInMemory(true) .setBlocksize(conf.getInt(HConstants.HBASE_META_BLOCK_SIZE, HConstants.DEFAULT_HBASE_META_BLOCK_SIZE)) .setScope(HConstants.REPLICATION_SCOPE_LOCAL) HConstants.DEFAULT_HBASE_META_VERSIONS)) .setInMemory(true) .setBlocksize(8 * 1024) .setScope(HConstants.REPLICATION_SCOPE_LOCAL) HConstants.DEFAULT_HBASE_META_VERSIONS)) .setInMemory(true) .setBlocksize(conf.getInt(HConstants.HBASE_META_BLOCK_SIZE, HConstants.DEFAULT_HBASE_META_BLOCK_SIZE)) .setScope(HConstants.REPLICATION_SCOPE_LOCAL)
/** * Callers must afterward call {@link HBaseTestingUtility#closeRegionAndWAL(HRegion)} */ private HRegion initHRegion(byte[] tableName, String callingMethod, Configuration conf, String family, BlockCache blockCache) throws IOException { TableDescriptorBuilder builder = TableDescriptorBuilder.newBuilder(TableName.valueOf(tableName)); for (int i = 0; i < BLOOM_TYPE.length; i++) { BloomType bloomType = BLOOM_TYPE[i]; builder.setColumnFamily( ColumnFamilyDescriptorBuilder.newBuilder(Bytes.toBytes(family + "_" + bloomType)) .setBlocksize(1).setBloomFilterType(bloomType).build()); } RegionInfo info = RegionInfoBuilder.newBuilder(TableName.valueOf(tableName)).build(); Path path = new Path(DIR + callingMethod); if (blockCache != null) { return HBaseTestingUtility.createRegionAndWAL(info, path, conf, builder.build(), blockCache); } else { return HBaseTestingUtility.createRegionAndWAL(info, path, conf, builder.build()); } }
@Test(expected=IllegalArgumentException.class) public void testAddDuplicateFamilies() { byte[] familyName = Bytes.toBytes("cf"); ColumnFamilyDescriptor hcd = ColumnFamilyDescriptorBuilder.newBuilder(familyName) .setBlocksize(1000) .build(); TableDescriptor htd = TableDescriptorBuilder.newBuilder(TableName.valueOf(name.getMethodName())) .setColumnFamily(hcd) .build(); assertEquals(1000, htd.getColumnFamily(familyName).getBlocksize()); hcd = ColumnFamilyDescriptorBuilder.newBuilder(familyName) .setBlocksize(2000) .build(); // add duplicate column TableDescriptorBuilder.newBuilder(htd).setColumnFamily(hcd).build(); }
/** * Create the ACL table * @throws IOException */ private static void createACLTable(Admin admin) throws IOException { /** Table descriptor for ACL table */ ColumnFamilyDescriptor cfd = ColumnFamilyDescriptorBuilder.newBuilder(AccessControlLists.ACL_LIST_FAMILY). setMaxVersions(1). setInMemory(true). setBlockCacheEnabled(true). setBlocksize(8 * 1024). setBloomFilterType(BloomType.NONE). setScope(HConstants.REPLICATION_SCOPE_LOCAL).build(); TableDescriptor td = TableDescriptorBuilder.newBuilder(AccessControlLists.ACL_TABLE_NAME). setColumnFamily(cfd).build(); admin.createTable(td); }
ColumnFamilyDescriptorBuilder.newBuilder(CF_BYTES).setMaxVersions(MAX_VERSIONS). setDataBlockEncoding(encoding). setBlocksize(BLOCK_SIZE). setBloomFilterType(BloomType.NONE). setCompressTags(compressTags).build();
@Test public void testModifyColumnFamily() throws Exception { TableDescriptorBuilder tdBuilder = TableDescriptorBuilder.newBuilder(tableName); ColumnFamilyDescriptor cfd = ColumnFamilyDescriptorBuilder.of(FAMILY_0); int blockSize = cfd.getBlocksize(); admin.createTable(tdBuilder.setColumnFamily(cfd).build()).join(); admin.disableTable(tableName).join(); // Verify the table descriptor verifyTableDescriptor(tableName, FAMILY_0); int newBlockSize = 2 * blockSize; cfd = ColumnFamilyDescriptorBuilder.newBuilder(FAMILY_0).setBlocksize(newBlockSize).build(); // Modify colymn family admin.modifyColumnFamily(tableName, cfd).join(); TableDescriptor htd = admin.getDescriptor(tableName).get(); ColumnFamilyDescriptor hcfd = htd.getColumnFamily(FAMILY_0); assertTrue(hcfd.getBlocksize() == newBlockSize); }
@Test public void testModifyNonExistingColumnFamily() throws Exception { TableDescriptorBuilder tdBuilder = TableDescriptorBuilder.newBuilder(tableName); ColumnFamilyDescriptor cfd = ColumnFamilyDescriptorBuilder.of(FAMILY_0); int blockSize = cfd.getBlocksize(); admin.createTable(tdBuilder.setColumnFamily(cfd).build()).join(); admin.disableTable(tableName).join(); // Verify the table descriptor verifyTableDescriptor(tableName, FAMILY_0); int newBlockSize = 2 * blockSize; cfd = ColumnFamilyDescriptorBuilder.newBuilder(FAMILY_1).setBlocksize(newBlockSize).build(); // Modify a column family that is not in the table. try { admin.modifyColumnFamily(tableName, cfd).join(); Assert.fail("Modify a non-exist column family should fail"); } catch (Exception e) { // Expected. } }
@Test public void testCacheBlocks() throws IOException { // Set index block size to be the same as normal block size. TEST_UTIL.getConfiguration().setInt(HFileBlockIndex.MAX_CHUNK_SIZE_KEY, BLOCK_SIZE); BlockCache blockCache = BlockCacheFactory.createBlockCache(TEST_UTIL.getConfiguration()); ColumnFamilyDescriptor cfd = ColumnFamilyDescriptorBuilder.newBuilder(Bytes.toBytes(CF)).setMaxVersions(MAX_VERSIONS) .setCompressionType(COMPRESSION_ALGORITHM).setBloomFilterType(BLOOM_TYPE) .setBlocksize(BLOCK_SIZE).setBlockCacheEnabled(cfCacheEnabled).build(); HRegion region = TEST_UTIL.createTestRegion(TABLE, cfd, blockCache); CacheStats stats = blockCache.getStats(); writeTestData(region); assertEquals(0, stats.getHitCount()); assertEquals(0, HFile.DATABLOCK_READ_COUNT.sum()); // Do a single get, take count of caches. If we are NOT caching DATA blocks, the miss // count should go up. Otherwise, all should be cached and the miss count should not rise. region.get(new Get(Bytes.toBytes("row" + 0))); assertTrue(stats.getHitCount() > 0); assertTrue(HFile.DATABLOCK_READ_COUNT.sum() > 0); long missCount = stats.getMissCount(); region.get(new Get(Bytes.toBytes("row" + 0))); if (this.cfCacheEnabled) assertEquals(missCount, stats.getMissCount()); else assertTrue(stats.getMissCount() > missCount); }
.setColumnFamily(ColumnFamilyDescriptorBuilder .newBuilder(FAMILY_1) .setBlocksize(BLOCK_SIZE) .setBlockCacheEnabled(BLOCK_CACHE) .setTimeToLive(TTL)
@Test public void testModifyFamily() { byte[] familyName = Bytes.toBytes("cf"); ColumnFamilyDescriptor hcd = ColumnFamilyDescriptorBuilder.newBuilder(familyName) .setBlocksize(1000) .setDFSReplication((short) 3) .build(); TableDescriptor htd = TableDescriptorBuilder.newBuilder(TableName.valueOf(name.getMethodName())) .setColumnFamily(hcd) .build(); assertEquals(1000, htd.getColumnFamily(familyName).getBlocksize()); assertEquals(3, htd.getColumnFamily(familyName).getDFSReplication()); hcd = ColumnFamilyDescriptorBuilder.newBuilder(familyName) .setBlocksize(2000) .setDFSReplication((short) 1) .build(); htd = TableDescriptorBuilder.newBuilder(htd) .modifyColumnFamily(hcd) .build(); assertEquals(2000, htd.getColumnFamily(familyName).getBlocksize()); assertEquals(1, htd.getColumnFamily(familyName).getDFSReplication()); }
builder.setBlocksize(in.getBlockSize());
.setColumnFamily(ColumnFamilyDescriptorBuilder .newBuilder(FAMILY_1) .setBlocksize(BLOCK_SIZE) .setBlockCacheEnabled(BLOCK_CACHE) .setTimeToLive(TTL)
/** * Create a table. * @param htd table descriptor * @param families array of column families * @param splitKeys array of split keys * @param type Bloom type * @param blockSize block size * @param c Configuration to use * @return A Table instance for the created table. * @throws IOException if getAdmin or createTable fails */ public Table createTable(TableDescriptor htd, byte[][] families, byte[][] splitKeys, BloomType type, int blockSize, Configuration c) throws IOException { TableDescriptorBuilder builder = TableDescriptorBuilder.newBuilder(htd); for (byte[] family : families) { ColumnFamilyDescriptorBuilder cfdb = ColumnFamilyDescriptorBuilder.newBuilder(family) .setBloomFilterType(type) .setBlocksize(blockSize); if (isNewVersionBehaviorEnabled()) { cfdb.setNewVersionBehavior(true); } builder.setColumnFamily(cfdb.build()); } TableDescriptor td = builder.build(); getAdmin().createTable(td, splitKeys); // HBaseAdmin only waits for regions to appear in hbase:meta // we should wait until they are assigned waitUntilAllRegionsAssigned(td.getTableName()); return getConnection().getTable(td.getTableName()); }
.setBloomFilterType(BloomType.NONE); final int v = 123; builder.setBlocksize(v); builder.setTimeToLive(v); builder.setBlockCacheEnabled(!HColumnDescriptor.DEFAULT_BLOCKCACHE);
@Test(expected=IllegalArgumentException.class) public void testAddDuplicateFamilies() { byte[] familyName = Bytes.toBytes("cf"); ColumnFamilyDescriptor hcd = ColumnFamilyDescriptorBuilder.newBuilder(familyName) .setBlocksize(1000) .build(); TableDescriptor htd = TableDescriptorBuilder.newBuilder(TableName.valueOf(name.getMethodName())) .setColumnFamily(hcd) .build(); assertEquals(1000, htd.getColumnFamily(familyName).getBlocksize()); hcd = ColumnFamilyDescriptorBuilder.newBuilder(familyName) .setBlocksize(2000) .build(); // add duplicate column TableDescriptorBuilder.newBuilder(htd).setColumnFamily(hcd).build(); }
@Test public void testModifyFamily() { byte[] familyName = Bytes.toBytes("cf"); ColumnFamilyDescriptor hcd = ColumnFamilyDescriptorBuilder.newBuilder(familyName) .setBlocksize(1000) .setDFSReplication((short) 3) .build(); TableDescriptor htd = TableDescriptorBuilder.newBuilder(TableName.valueOf(name.getMethodName())) .setColumnFamily(hcd) .build(); assertEquals(1000, htd.getColumnFamily(familyName).getBlocksize()); assertEquals(3, htd.getColumnFamily(familyName).getDFSReplication()); hcd = ColumnFamilyDescriptorBuilder.newBuilder(familyName) .setBlocksize(2000) .setDFSReplication((short) 1) .build(); htd = TableDescriptorBuilder.newBuilder(htd) .modifyColumnFamily(hcd) .build(); assertEquals(2000, htd.getColumnFamily(familyName).getBlocksize()); assertEquals(1, htd.getColumnFamily(familyName).getDFSReplication()); }
.setBloomFilterType(BloomType.NONE); final int v = 123; builder.setBlocksize(v); builder.setTimeToLive(v); builder.setBlockCacheEnabled(!HColumnDescriptor.DEFAULT_BLOCKCACHE);
@Test public void testModifyColumnFamily() throws Exception { TableDescriptorBuilder tdBuilder = TableDescriptorBuilder.newBuilder(tableName); ColumnFamilyDescriptor cfd = ColumnFamilyDescriptorBuilder.of(FAMILY_0); int blockSize = cfd.getBlocksize(); admin.createTable(tdBuilder.setColumnFamily(cfd).build()).join(); admin.disableTable(tableName).join(); // Verify the table descriptor verifyTableDescriptor(tableName, FAMILY_0); int newBlockSize = 2 * blockSize; cfd = ColumnFamilyDescriptorBuilder.newBuilder(FAMILY_0).setBlocksize(newBlockSize).build(); // Modify colymn family admin.modifyColumnFamily(tableName, cfd).join(); TableDescriptor htd = admin.getDescriptor(tableName).get(); ColumnFamilyDescriptor hcfd = htd.getColumnFamily(FAMILY_0); assertTrue(hcfd.getBlocksize() == newBlockSize); }
@Test public void testModifyNonExistingColumnFamily() throws Exception { TableDescriptorBuilder tdBuilder = TableDescriptorBuilder.newBuilder(tableName); ColumnFamilyDescriptor cfd = ColumnFamilyDescriptorBuilder.of(FAMILY_0); int blockSize = cfd.getBlocksize(); admin.createTable(tdBuilder.setColumnFamily(cfd).build()).join(); admin.disableTable(tableName).join(); // Verify the table descriptor verifyTableDescriptor(tableName, FAMILY_0); int newBlockSize = 2 * blockSize; cfd = ColumnFamilyDescriptorBuilder.newBuilder(FAMILY_1).setBlocksize(newBlockSize).build(); // Modify a column family that is not in the table. try { admin.modifyColumnFamily(tableName, cfd).join(); Assert.fail("Modify a non-exist column family should fail"); } catch (Exception e) { // Expected. } }