/** * Create a cache configuration using the specified configuration object and * defaults for family level settings. * @param conf hbase configuration */ public CacheConfig(Configuration conf) { this(CacheConfig.instantiateBlockCache(conf), DEFAULT_CACHE_DATA_ON_READ, DEFAULT_IN_MEMORY, // This is a family-level setting so can't be set // strictly from conf conf.getBoolean(CACHE_BLOCKS_ON_WRITE_KEY, DEFAULT_CACHE_DATA_ON_WRITE), conf.getBoolean(CACHE_INDEX_BLOCKS_ON_WRITE_KEY, DEFAULT_CACHE_INDEXES_ON_WRITE), conf.getBoolean(CACHE_BLOOM_BLOCKS_ON_WRITE_KEY, DEFAULT_CACHE_BLOOMS_ON_WRITE), conf.getBoolean(EVICT_BLOCKS_ON_CLOSE_KEY, DEFAULT_EVICT_ON_CLOSE), conf.getBoolean(CACHE_DATA_BLOCKS_COMPRESSED_KEY, DEFAULT_COMPRESSED_CACHE) ); }
public static HeapMemoryManager create(Configuration conf, FlushRequester memStoreFlusher, Server server, RegionServerAccounting regionServerAccounting) { BlockCache blockCache = CacheConfig.instantiateBlockCache(conf); if (blockCache instanceof ResizableBlockCache) { return new HeapMemoryManager((ResizableBlockCache) blockCache, memStoreFlusher, server, regionServerAccounting); } return null; }
/** * Create a cache configuration using the specified configuration object and * defaults for family level settings. * @param conf hbase configuration */ public CacheConfig(Configuration conf) { this(CacheConfig.instantiateBlockCache(conf), DEFAULT_CACHE_DATA_ON_READ, DEFAULT_IN_MEMORY, // This is a family-level setting so can't be set // strictly from conf conf.getBoolean(CACHE_BLOCKS_ON_WRITE_KEY, DEFAULT_CACHE_DATA_ON_WRITE), conf.getBoolean(CACHE_INDEX_BLOCKS_ON_WRITE_KEY, DEFAULT_CACHE_INDEXES_ON_WRITE), conf.getBoolean(CACHE_BLOOM_BLOCKS_ON_WRITE_KEY, DEFAULT_CACHE_BLOOMS_ON_WRITE), conf.getBoolean(EVICT_BLOCKS_ON_CLOSE_KEY, DEFAULT_EVICT_ON_CLOSE), conf.getBoolean(CACHE_DATA_BLOCKS_COMPRESSED_KEY, DEFAULT_CACHE_DATA_COMPRESSED), conf.getBoolean(PREFETCH_BLOCKS_ON_OPEN_KEY, DEFAULT_PREFETCH_ON_OPEN), conf.getBoolean(HColumnDescriptor.CACHE_DATA_IN_L1, HColumnDescriptor.DEFAULT_CACHE_DATA_IN_L1), conf.getBoolean(DROP_BEHIND_CACHE_COMPACTION_KEY,DROP_BEHIND_CACHE_COMPACTION_DEFAULT) ); }
/** * Create a cache configuration using the specified configuration object and * family descriptor. * @param conf hbase configuration * @param family column family configuration */ public CacheConfig(Configuration conf, HColumnDescriptor family) { this(CacheConfig.instantiateBlockCache(conf), family.isBlockCacheEnabled(), family.isInMemory(), // For the following flags we enable them regardless of per-schema settings // if they are enabled in the global configuration. conf.getBoolean(CACHE_BLOCKS_ON_WRITE_KEY, DEFAULT_CACHE_DATA_ON_WRITE) || family.shouldCacheDataOnWrite(), conf.getBoolean(CACHE_INDEX_BLOCKS_ON_WRITE_KEY, DEFAULT_CACHE_INDEXES_ON_WRITE) || family.shouldCacheIndexesOnWrite(), conf.getBoolean(CACHE_BLOOM_BLOCKS_ON_WRITE_KEY, DEFAULT_CACHE_BLOOMS_ON_WRITE) || family.shouldCacheBloomsOnWrite(), conf.getBoolean(EVICT_BLOCKS_ON_CLOSE_KEY, DEFAULT_EVICT_ON_CLOSE) || family.shouldEvictBlocksOnClose(), conf.getBoolean(CACHE_DATA_BLOCKS_COMPRESSED_KEY, DEFAULT_COMPRESSED_CACHE) ); }
@BeforeClass public static void setUp() throws Exception { // disable compactions in this test. TEST_UTIL.getConfiguration().setInt("hbase.hstore.compactionThreshold", 10000); CacheConfig.instantiateBlockCache(TEST_UTIL.getConfiguration()); }
private void setUp(boolean useBucketCache) throws IOException { test_util = HBaseTestingUtility.createLocalHTU(); conf = test_util.getConfiguration(); if (useBucketCache) { conf.setInt("hbase.bucketcache.size", 400); conf.setStrings(HConstants.BUCKET_CACHE_IOENGINE_KEY, "offheap"); conf.setInt("hbase.bucketcache.writer.threads", 10); conf.setFloat("hfile.block.cache.size", 0.2f); conf.setFloat("hbase.regionserver.global.memstore.size", 0.1f); } tableName = TableName.valueOf(name.getMethodName()); CacheConfig.instantiateBlockCache(conf); }
@Before public void setUp() throws IOException { conf = TEST_UTIL.getConfiguration(); // This test requires the most recent HFile format (i.e. v2). conf.setInt(HFile.FORMAT_VERSION_KEY, HFile.MAX_FORMAT_VERSION); fs = FileSystem.get(conf); CacheConfig.instantiateBlockCache(conf); cacheConf = new CacheConfig(conf); blockCache = cacheConf.getBlockCache(); assertNotNull(blockCache); }
@Before public void setUp() throws IOException { conf = TEST_UTIL.getConfiguration(); conf.setBoolean(CacheConfig.PREFETCH_BLOCKS_ON_OPEN_KEY, true); fs = HFileSystem.get(conf); CacheConfig.blockCacheDisabled = false; CacheConfig.instantiateBlockCache(conf); cacheConf = new CacheConfig(conf); }
/** * Create a cache configuration using the specified configuration object and * family descriptor. * @param conf hbase configuration * @param family column family configuration */ public CacheConfig(Configuration conf, HColumnDescriptor family) { this(CacheConfig.instantiateBlockCache(conf), family.isBlockCacheEnabled(), family.isInMemory(), // For the following flags we enable them regardless of per-schema settings // if they are enabled in the global configuration. conf.getBoolean(CACHE_BLOCKS_ON_WRITE_KEY, DEFAULT_CACHE_DATA_ON_WRITE) || family.isCacheDataOnWrite(), conf.getBoolean(CACHE_INDEX_BLOCKS_ON_WRITE_KEY, DEFAULT_CACHE_INDEXES_ON_WRITE) || family.isCacheIndexesOnWrite(), conf.getBoolean(CACHE_BLOOM_BLOCKS_ON_WRITE_KEY, DEFAULT_CACHE_BLOOMS_ON_WRITE) || family.isCacheBloomsOnWrite(), conf.getBoolean(EVICT_BLOCKS_ON_CLOSE_KEY, DEFAULT_EVICT_ON_CLOSE) || family.isEvictBlocksOnClose(), conf.getBoolean(CACHE_DATA_BLOCKS_COMPRESSED_KEY, DEFAULT_CACHE_DATA_COMPRESSED), conf.getBoolean(PREFETCH_BLOCKS_ON_OPEN_KEY, DEFAULT_PREFETCH_ON_OPEN) || family.isPrefetchBlocksOnOpen(), conf.getBoolean(HColumnDescriptor.CACHE_DATA_IN_L1, HColumnDescriptor.DEFAULT_CACHE_DATA_IN_L1) || family.isCacheDataInL1(), conf.getBoolean(DROP_BEHIND_CACHE_COMPACTION_KEY,DROP_BEHIND_CACHE_COMPACTION_DEFAULT) ); }
@Before public void setup() { // Make sure we make a new one each time. CacheConfig.clearGlobalInstances(); HFile.DATABLOCK_READ_COUNT.reset(); CacheConfig.instantiateBlockCache(TEST_UTIL.getConfiguration()); }
private static List<BlockCache> getBlockCaches() throws IOException { Configuration conf = TEST_UTIL.getConfiguration(); List<BlockCache> blockcaches = new ArrayList<>(); // default CacheConfig.instantiateBlockCache(conf); blockcaches.add(new CacheConfig(conf).getBlockCache()); //set LruBlockCache.LRU_HARD_CAPACITY_LIMIT_FACTOR_CONFIG_NAME to 2.0f due to HBASE-16287 TEST_UTIL.getConfiguration().setFloat(LruBlockCache.LRU_HARD_CAPACITY_LIMIT_FACTOR_CONFIG_NAME, 2.0f); // memory BlockCache lru = new LruBlockCache(128 * 1024 * 1024, 64 * 1024, TEST_UTIL.getConfiguration()); blockcaches.add(lru); // bucket cache FileSystem.get(conf).mkdirs(TEST_UTIL.getDataTestDir()); int[] bucketSizes = { INDEX_BLOCK_SIZE, DATA_BLOCK_SIZE, BLOOM_BLOCK_SIZE, 64 * 1024, 128 * 1024 }; BlockCache bucketcache = new BucketCache("offheap", 128 * 1024 * 1024, 64 * 1024, bucketSizes, 5, 64 * 100, null); blockcaches.add(bucketcache); return blockcaches; }
@Override @Before public void setUp() throws Exception { super.setUp(); TEST_UTIL = new HBaseTestingUtility(); CacheConfig.instantiateBlockCache(TEST_UTIL.getConfiguration()); }
/** * HBASE-12782 ITBLL fails for me if generator does anything but 5M per maptask. * Create a region. Close it. Then copy into place a file to replay, one that is bigger than * configured flush size so we bring on lots of flushes. Then reopen and confirm all edits * made it in. * @throws IOException */ @Test public void testReplayWorksThoughLotsOfFlushing() throws IOException { CacheConfig.instantiateBlockCache(TEST_UTIL.getConfiguration()); for(MemoryCompactionPolicy policy : MemoryCompactionPolicy.values()) { testReplayWorksWithMemoryCompactionPolicy(policy); } }
@Test public void testBucketCache() throws IOException { this.conf.set(HConstants.BUCKET_CACHE_IOENGINE_KEY, "offheap"); this.conf.setInt(HConstants.BUCKET_CACHE_SIZE_KEY, 100); CacheConfig.instantiateBlockCache(this.conf); CacheConfig cc = new CacheConfig(this.conf); assertTrue(cc.getBlockCache() instanceof CombinedBlockCache); logPerBlock(cc.getBlockCache()); final int count = 3; addDataAndHits(cc.getBlockCache(), count); // The below has no asserts. It is just exercising toString and toJSON code. LOG.info(Objects.toString(cc.getBlockCache().getStats())); BlockCacheUtil.CachedBlocksByFile cbsbf = logPerBlock(cc.getBlockCache()); LOG.info(Objects.toString(cbsbf)); logPerFile(cbsbf); bucketCacheReport(cc.getBlockCache()); LOG.info(BlockCacheUtil.toJSON(cbsbf)); }
CacheConfig.instantiateBlockCache(conf); CacheConfig cacheConf = new CacheConfig(conf); LruBlockCache cache = (LruBlockCache) cacheConf.getBlockCache();
@Before public void setUp() throws IOException { // parameterized tests add [#] suffix get rid of [ and ]. table = Bytes.toBytes(name.getMethodName().replaceAll("[\\[\\]]", "_")); conf = TEST_UTIL.getConfiguration(); conf.setInt(HFile.FORMAT_VERSION_KEY, HFile.MAX_FORMAT_VERSION); conf.setBoolean(CacheConfig.CACHE_BLOCKS_ON_WRITE_KEY, false); conf.setBoolean(CacheConfig.CACHE_INDEX_BLOCKS_ON_WRITE_KEY, false); conf.setBoolean(CacheConfig.CACHE_BLOOM_BLOCKS_ON_WRITE_KEY, false); CacheConfig.instantiateBlockCache(conf); fs = HFileSystem.get(conf); // Create the schema ColumnFamilyDescriptor hcd = cowType .modifyFamilySchema( ColumnFamilyDescriptorBuilder.newBuilder(family).setBloomFilterType(BloomType.ROWCOL)) .build(); TableDescriptor htd = TableDescriptorBuilder.newBuilder(TableName.valueOf(table)).setColumnFamily(hcd).build(); // Create a store based on the schema String id = TestCacheOnWriteInSchema.class.getName(); Path logdir = new Path(FSUtils.getRootDir(conf), AbstractFSWALProvider.getWALDirectoryName(id)); fs.delete(logdir, true); RegionInfo info = RegionInfoBuilder.newBuilder(htd.getTableName()).build(); walFactory = new WALFactory(conf, id); region = TEST_UTIL.createLocalHRegion(info, htd, walFactory.getWAL(info)); store = new HStore(region, hcd, conf); }
testUtil.getConfiguration().setInt(HFile.FORMAT_VERSION_KEY, 3); CacheConfig.instantiateBlockCache(testUtil.getConfiguration()); LruBlockCache cache = (LruBlockCache)new CacheConfig(testUtil.getConfiguration()).getBlockCache();
@Test public void testLruBlockCache() throws IOException { CacheConfig.instantiateBlockCache(this.conf); CacheConfig cc = new CacheConfig(this.conf); assertTrue(cc.isBlockCacheEnabled()); assertTrue(CacheConfig.DEFAULT_IN_MEMORY == cc.isInMemory()); assertTrue(cc.getBlockCache() instanceof LruBlockCache); logPerBlock(cc.getBlockCache()); addDataAndHits(cc.getBlockCache(), 3); // The below has no asserts. It is just exercising toString and toJSON code. BlockCache bc = cc.getBlockCache(); LOG.info("count=" + bc.getBlockCount() + ", currentSize=" + bc.getCurrentSize() + ", freeSize=" + bc.getFreeSize() ); LOG.info(Objects.toString(cc.getBlockCache().getStats())); BlockCacheUtil.CachedBlocksByFile cbsbf = logPerBlock(cc.getBlockCache()); LOG.info(Objects.toString(cbsbf)); logPerFile(cbsbf); bucketCacheReport(cc.getBlockCache()); LOG.info(BlockCacheUtil.toJSON(cbsbf)); }
@Test public void testCacheConfigDefaultLRUBlockCache() { CacheConfig.instantiateBlockCache(this.conf); CacheConfig cc = new CacheConfig(this.conf); assertTrue(cc.isBlockCacheEnabled()); assertTrue(CacheConfig.DEFAULT_IN_MEMORY == cc.isInMemory()); basicBlockCacheOps(cc, false, true); assertTrue(cc.getBlockCache() instanceof LruBlockCache); }
private void doBucketCacheConfigTest() { final int bcSize = 100; this.conf.setInt(HConstants.BUCKET_CACHE_SIZE_KEY, bcSize); CacheConfig.instantiateBlockCache(this.conf); CacheConfig cc = new CacheConfig(this.conf); basicBlockCacheOps(cc, false, false); assertTrue(cc.getBlockCache() instanceof CombinedBlockCache); // TODO: Assert sizes allocated are right and proportions. CombinedBlockCache cbc = (CombinedBlockCache)cc.getBlockCache(); BlockCache [] bcs = cbc.getBlockCaches(); assertTrue(bcs[0] instanceof LruBlockCache); LruBlockCache lbc = (LruBlockCache)bcs[0]; assertEquals(MemorySizeUtil.getOnHeapCacheSize(this.conf), lbc.getMaxSize()); assertTrue(bcs[1] instanceof BucketCache); BucketCache bc = (BucketCache)bcs[1]; // getMaxSize comes back in bytes but we specified size in MB assertEquals(bcSize, bc.getMaxSize() / (1024 * 1024)); }