@Override public void perform() throws Exception { final Random random = new Random(); final BloomType[] bloomArray = BloomType.values(); final int bloomArraySize = bloomArray.length; LOG.info("Performing action: Change bloom filter on all columns of table " + tableName); modifyAllTableColumns(tableName, (columnName, columnBuilder) -> { BloomType bloomType = bloomArray[random.nextInt(bloomArraySize)]; LOG.debug("Performing action: About to set bloom filter type to " + bloomType + " on column " + columnName + " of table " + tableName); columnBuilder.setBloomFilterType(bloomType); if (bloomType == BloomType.ROWPREFIX_FIXED_LENGTH) { columnBuilder.setConfiguration(BloomFilterUtil.PREFIX_LENGTH_KEY, "10"); } else if (bloomType == BloomType.ROWPREFIX_DELIMITED) { columnBuilder.setConfiguration(BloomFilterUtil.DELIMITER_KEY, "#"); } }); LOG.debug("Performing action: Just set bloom filter types on table " + tableName); } }
.setScope(HConstants.REPLICATION_SCOPE_LOCAL) .setBloomFilterType(BloomType.NONE) .build()) .setColumnFamily(ColumnFamilyDescriptorBuilder.newBuilder(HConstants.TABLE_FAMILY) .setScope(HConstants.REPLICATION_SCOPE_LOCAL) .setBloomFilterType(BloomType.NONE) .build()) .setColumnFamily(ColumnFamilyDescriptorBuilder .setScope(HConstants.REPLICATION_SCOPE_LOCAL) .setBloomFilterType(BloomType.NONE) .build()) .setColumnFamily(ColumnFamilyDescriptorBuilder .setScope(HConstants.REPLICATION_SCOPE_LOCAL) .setBloomFilterType(BloomType.NONE) .build()) .setCoprocessor(CoprocessorDescriptorBuilder.newBuilder(
private TableDescriptor buildHTD(TableName tableName, BloomType bloomType) { return TableDescriptorBuilder.newBuilder(tableName) .setColumnFamily( ColumnFamilyDescriptorBuilder.newBuilder(FAMILY).setBloomFilterType(bloomType).build()) .build(); }
/** * Callers must afterward call {@link HBaseTestingUtility#closeRegionAndWAL(HRegion)} */ private HRegion initHRegion(byte[] tableName, String callingMethod, Configuration conf, String family, BlockCache blockCache) throws IOException { TableDescriptorBuilder builder = TableDescriptorBuilder.newBuilder(TableName.valueOf(tableName)); for (int i = 0; i < BLOOM_TYPE.length; i++) { BloomType bloomType = BLOOM_TYPE[i]; builder.setColumnFamily( ColumnFamilyDescriptorBuilder.newBuilder(Bytes.toBytes(family + "_" + bloomType)) .setBlocksize(1).setBloomFilterType(bloomType).build()); } RegionInfo info = RegionInfoBuilder.newBuilder(TableName.valueOf(tableName)).build(); Path path = new Path(DIR + callingMethod); if (blockCache != null) { return HBaseTestingUtility.createRegionAndWAL(info, path, conf, builder.build(), blockCache); } else { return HBaseTestingUtility.createRegionAndWAL(info, path, conf, builder.build()); } }
/** * Create the ACL table * @throws IOException */ private static void createACLTable(Admin admin) throws IOException { /** Table descriptor for ACL table */ ColumnFamilyDescriptor cfd = ColumnFamilyDescriptorBuilder.newBuilder(AccessControlLists.ACL_LIST_FAMILY). setMaxVersions(1). setInMemory(true). setBlockCacheEnabled(true). setBlocksize(8 * 1024). setBloomFilterType(BloomType.NONE). setScope(HConstants.REPLICATION_SCOPE_LOCAL).build(); TableDescriptor td = TableDescriptorBuilder.newBuilder(AccessControlLists.ACL_TABLE_NAME). setColumnFamily(cfd).build(); admin.createTable(td); }
@Before public void setUp() throws IOException { // parameterized tests add [#] suffix get rid of [ and ]. table = Bytes.toBytes(name.getMethodName().replaceAll("[\\[\\]]", "_")); conf = TEST_UTIL.getConfiguration(); conf.setInt(HFile.FORMAT_VERSION_KEY, HFile.MAX_FORMAT_VERSION); conf.setBoolean(CacheConfig.CACHE_BLOCKS_ON_WRITE_KEY, false); conf.setBoolean(CacheConfig.CACHE_INDEX_BLOCKS_ON_WRITE_KEY, false); conf.setBoolean(CacheConfig.CACHE_BLOOM_BLOCKS_ON_WRITE_KEY, false); fs = HFileSystem.get(conf); // Create the schema ColumnFamilyDescriptor hcd = cowType .modifyFamilySchema( ColumnFamilyDescriptorBuilder.newBuilder(family).setBloomFilterType(BloomType.ROWCOL)) .build(); TableDescriptor htd = TableDescriptorBuilder.newBuilder(TableName.valueOf(table)).setColumnFamily(hcd).build(); // Create a store based on the schema String id = TestCacheOnWriteInSchema.class.getName(); Path logdir = new Path(FSUtils.getRootDir(conf), AbstractFSWALProvider.getWALDirectoryName(id)); fs.delete(logdir, true); RegionInfo info = RegionInfoBuilder.newBuilder(htd.getTableName()).build(); walFactory = new WALFactory(conf, id); region = TEST_UTIL.createLocalHRegion(info, htd, walFactory.getWAL(info)); region.setBlockCache(BlockCacheFactory.createBlockCache(conf)); store = new HStore(region, hcd, conf); }
setDataBlockEncoding(encoding). setBlocksize(BLOCK_SIZE). setBloomFilterType(BloomType.NONE). setCompressTags(compressTags).build(); HRegion region = testUtil.createTestRegion(TABLE_NAME, cfd, cache);
HRegion region = TEST_UTIL.createTestRegion(TABLE_NAME, ColumnFamilyDescriptorBuilder.newBuilder(FAMILY_BYTES).setCompressionType(comprAlgo) .setBloomFilterType(bloomType).setMaxVersions(MAX_VERSIONS) .setDataBlockEncoding(dataBlockEncoding).build(), BlockCacheFactory.createBlockCache(TEST_UTIL.getConfiguration()));
ColumnFamilyDescriptor cfd = ColumnFamilyDescriptorBuilder.newBuilder(cfBytes).setCompressionType(compress) .setBloomFilterType(BLOOM_TYPE).setMaxVersions(maxVersions) .setDataBlockEncoding(NoOpDataBlockEncoder.INSTANCE.getDataBlockEncoding()).build(); HRegion region = TEST_UTIL.createTestRegion(table, cfd, blockCache);
@Test public void testCacheBlocks() throws IOException { // Set index block size to be the same as normal block size. TEST_UTIL.getConfiguration().setInt(HFileBlockIndex.MAX_CHUNK_SIZE_KEY, BLOCK_SIZE); BlockCache blockCache = BlockCacheFactory.createBlockCache(TEST_UTIL.getConfiguration()); ColumnFamilyDescriptor cfd = ColumnFamilyDescriptorBuilder.newBuilder(Bytes.toBytes(CF)).setMaxVersions(MAX_VERSIONS) .setCompressionType(COMPRESSION_ALGORITHM).setBloomFilterType(BLOOM_TYPE) .setBlocksize(BLOCK_SIZE).setBlockCacheEnabled(cfCacheEnabled).build(); HRegion region = TEST_UTIL.createTestRegion(TABLE, cfd, blockCache); CacheStats stats = blockCache.getStats(); writeTestData(region); assertEquals(0, stats.getHitCount()); assertEquals(0, HFile.DATABLOCK_READ_COUNT.sum()); // Do a single get, take count of caches. If we are NOT caching DATA blocks, the miss // count should go up. Otherwise, all should be cached and the miss count should not rise. region.get(new Get(Bytes.toBytes("row" + 0))); assertTrue(stats.getHitCount() > 0); assertTrue(HFile.DATABLOCK_READ_COUNT.sum() > 0); long missCount = stats.getMissCount(); region.get(new Get(Bytes.toBytes("row" + 0))); if (this.cfCacheEnabled) assertEquals(missCount, stats.getMissCount()); else assertTrue(stats.getMissCount() > missCount); }
builder.setBloomFilterType(bloomFilterFromThrift(in.getBloomnFilterType()));
ColumnFamilyDescriptorBuilder.newBuilder(columnDesc); if (bloomType != null) { columnDescBuilder.setBloomFilterType(bloomType);
/** * Create a Mob table. * * @param util * @param tableName * @param families * @return An Table instance for the created table. * @throws IOException */ public static Table createMobTable(final HBaseTestingUtility util, final TableName tableName, final byte[]... families) throws IOException { TableDescriptorBuilder builder = TableDescriptorBuilder.newBuilder(tableName); for (byte[] family : families) { // Disable blooms (they are on by default as of 0.95) but we disable them // here because // tests have hard coded counts of what to expect in block cache, etc., // and blooms being // on is interfering. builder.setColumnFamily(ColumnFamilyDescriptorBuilder.newBuilder(family) .setBloomFilterType(BloomType.NONE) .setMobEnabled(true) .setMobThreshold(0L) .build()); } util.getAdmin().createTable(builder.build()); // HBaseAdmin only waits for regions to appear in hbase:meta we should wait // until they are assigned util.waitUntilAllRegionsAssigned(tableName); return ConnectionFactory.createConnection(util.getConfiguration()).getTable(tableName); }
/** * Create a table. * @param htd table descriptor * @param families array of column families * @param splitKeys array of split keys * @param type Bloom type * @param blockSize block size * @param c Configuration to use * @return A Table instance for the created table. * @throws IOException if getAdmin or createTable fails */ public Table createTable(TableDescriptor htd, byte[][] families, byte[][] splitKeys, BloomType type, int blockSize, Configuration c) throws IOException { TableDescriptorBuilder builder = TableDescriptorBuilder.newBuilder(htd); for (byte[] family : families) { ColumnFamilyDescriptorBuilder cfdb = ColumnFamilyDescriptorBuilder.newBuilder(family) .setBloomFilterType(type) .setBlocksize(blockSize); if (isNewVersionBehaviorEnabled()) { cfdb.setNewVersionBehavior(true); } builder.setColumnFamily(cfdb.build()); } TableDescriptor td = builder.build(); getAdmin().createTable(td, splitKeys); // HBaseAdmin only waits for regions to appear in hbase:meta // we should wait until they are assigned waitUntilAllRegionsAssigned(td.getTableName()); return getConnection().getTable(td.getTableName()); }
private ColumnFamilyDescriptor generateColumnFamilyDescriptor(Pair<byte[],Map<String,Object>> family, PTableType tableType) throws SQLException { ColumnFamilyDescriptorBuilder columnDescBuilder = ColumnFamilyDescriptorBuilder.newBuilder(family.getFirst()); if (tableType != PTableType.VIEW) { columnDescBuilder.setDataBlockEncoding(SchemaUtil.DEFAULT_DATA_BLOCK_ENCODING); columnDescBuilder.setBloomFilterType(BloomType.NONE); for (Entry<String,Object> entry : family.getSecond().entrySet()) { String key = entry.getKey(); Object value = entry.getValue(); setHColumnDescriptorValue(columnDescBuilder, key, value); } } return columnDescBuilder.build(); }
.setInMemory(true) .setScope(HConstants.REPLICATION_SCOPE_LOCAL) .setBloomFilterType(BloomType.NONE); final int v = 123; builder.setBlocksize(v); builder.setScope(v); builder.setDataBlockEncoding(DataBlockEncoding.FAST_DIFF); builder.setBloomFilterType(BloomType.ROW); builder.setCompressionType(Algorithm.SNAPPY); builder.setMobEnabled(true);
private ColumnFamilyDescriptor generateColumnFamilyDescriptor(Pair<byte[],Map<String,Object>> family, PTableType tableType) throws SQLException { ColumnFamilyDescriptorBuilder columnDescBuilder = ColumnFamilyDescriptorBuilder.newBuilder(family.getFirst()); if (tableType != PTableType.VIEW) { columnDescBuilder.setDataBlockEncoding(SchemaUtil.DEFAULT_DATA_BLOCK_ENCODING); columnDescBuilder.setBloomFilterType(BloomType.NONE); for (Entry<String,Object> entry : family.getSecond().entrySet()) { String key = entry.getKey(); Object value = entry.getValue(); setHColumnDescriptorValue(columnDescBuilder, key, value); } } return columnDescBuilder.build(); }
private ColumnFamilyDescriptor generateColumnFamilyDescriptor(Pair<byte[],Map<String,Object>> family, PTableType tableType) throws SQLException { ColumnFamilyDescriptorBuilder columnDescBuilder = ColumnFamilyDescriptorBuilder.newBuilder(family.getFirst()); if (tableType != PTableType.VIEW) { columnDescBuilder.setDataBlockEncoding(SchemaUtil.DEFAULT_DATA_BLOCK_ENCODING); columnDescBuilder.setBloomFilterType(BloomType.NONE); for (Entry<String,Object> entry : family.getSecond().entrySet()) { String key = entry.getKey(); Object value = entry.getValue(); setHColumnDescriptorValue(columnDescBuilder, key, value); } } return columnDescBuilder.build(); }
.setInMemory(true) .setScope(HConstants.REPLICATION_SCOPE_LOCAL) .setBloomFilterType(BloomType.NONE); final int v = 123; builder.setBlocksize(v); builder.setScope(v); builder.setDataBlockEncoding(DataBlockEncoding.FAST_DIFF); builder.setBloomFilterType(BloomType.ROW); builder.setCompressionType(Algorithm.SNAPPY); builder.setMobEnabled(true);
private TableDescriptor buildHTD(TableName tableName, BloomType bloomType) { return TableDescriptorBuilder.newBuilder(tableName) .setColumnFamily( ColumnFamilyDescriptorBuilder.newBuilder(FAMILY).setBloomFilterType(bloomType).build()) .build(); }