public CombinedBlockCache(LruBlockCache onHeapCache, BlockCache l2Cache) { this.onHeapCache = onHeapCache; this.l2Cache = l2Cache; this.combinedCacheStats = new CombinedCacheStats(onHeapCache.getStats(), l2Cache.getStats()); }
private void __jamon_innerUnit__evictions_tmpl(final java.io.Writer jamonWriter, final BlockCache bc) throws java.io.IOException { // 171, 1 AgeSnapshot ageAtEvictionSnapshot = bc.getStats().getAgeAtEvictionSnapshot(); // Only show if non-zero mean and stddev as is the case in combinedblockcache // 175, 5 jamonWriter.write("<tr>\n <td>Evicted</td>\n <td>"); // 177, 13 org.jamon.escaping.Escaping.HTML.write(org.jamon.emit.StandardEmitter.valueOf(String.format("%,d", bc.getStats().getEvictedCount())), jamonWriter); // 177, 72 jamonWriter.write("</td>\n <td>The total number of blocks evicted</td>\n </tr>\n <tr>\n <td>Evictions</td>\n <td>"); // 182, 13 org.jamon.escaping.Escaping.HTML.write(org.jamon.emit.StandardEmitter.valueOf(String.format("%,d", bc.getStats().getEvictionCount())), jamonWriter); // 182, 73 jamonWriter.write("</td>\n <td>The total number of times an eviction has occurred</td>\n </tr>\n <tr>\n <td>Mean</td>\n <td>"); // 187, 13 org.jamon.escaping.Escaping.HTML.write(org.jamon.emit.StandardEmitter.valueOf(String.format("%,d", (long)ageAtEvictionSnapshot.getMean())), jamonWriter); // 187, 78 jamonWriter.write("</td>\n <td>Mean age of Blocks at eviction time (seconds)</td>\n </tr>\n"); }
private void __jamon_innerUnit__hits_tmpl(final java.io.Writer jamonWriter, final BlockCache bc) throws java.io.IOException { // 196, 5 jamonWriter.write("<tr>\n <td>Hits</td>\n <td>"); // 198, 13 org.jamon.escaping.Escaping.HTML.write(org.jamon.emit.StandardEmitter.valueOf(String.format("%,d", bc.getStats().getHitCount())), jamonWriter); // 198, 68 jamonWriter.write("</td>\n <td>Number requests that were cache hits</td>\n </tr>\n <tr>\n <td>Hits Caching</td>\n <td>"); // 203, 13 org.jamon.escaping.Escaping.HTML.write(org.jamon.emit.StandardEmitter.valueOf(String.format("%,d", bc.getStats().getHitCachingCount())), jamonWriter); // 203, 75 jamonWriter.write("</td>\n <td>Cache hit block requests but only requests set to cache block if a miss</td>\n </tr>\n <tr>\n <td>Misses</td>\n <td>"); // 208, 13 org.jamon.escaping.Escaping.HTML.write(org.jamon.emit.StandardEmitter.valueOf(String.format("%,d", bc.getStats().getMissCount())), jamonWriter); // 208, 69 jamonWriter.write("</td>\n <td>Block requests that were cache misses but set to cache missed blocks</td>\n </tr>\n <tr>\n <td>Misses Caching</td>\n <td>"); // 213, 13 org.jamon.escaping.Escaping.HTML.write(org.jamon.emit.StandardEmitter.valueOf(String.format("%,d", bc.getStats().getMissCount())), jamonWriter); // 213, 69 jamonWriter.write("</td>\n <td>Block requests that were cache misses but only requests set to use block cache</td>\n </tr>\n <tr>\n <td>Hit Ratio</td>\n <td>"); // 218, 13 org.jamon.escaping.Escaping.HTML.write(org.jamon.emit.StandardEmitter.valueOf(String.format("%,.2f", bc.getStats().getHitRatio() * 100)), jamonWriter); // 218, 76 org.jamon.escaping.Escaping.HTML.write(org.jamon.emit.StandardEmitter.valueOf("%"), jamonWriter); // 218, 85 jamonWriter.write("</td>\n <td>Hit Count divided by total requests count</td>\n </tr>\n\n"); }
bucketCacheStats = (BucketCacheStats)bc.getStats(); bucketAllocator = ((BucketCache)bc).getAllocator();
@Test public void testBucketCache() throws IOException { this.conf.set(HConstants.BUCKET_CACHE_IOENGINE_KEY, "offheap"); this.conf.setInt(HConstants.BUCKET_CACHE_SIZE_KEY, 100); BlockCache blockCache = BlockCacheFactory.createBlockCache(this.conf); assertTrue(blockCache instanceof CombinedBlockCache); logPerBlock(blockCache); final int count = 3; addDataAndHits(blockCache, count); // The below has no asserts. It is just exercising toString and toJSON code. LOG.info(Objects.toString(blockCache.getStats())); BlockCacheUtil.CachedBlocksByFile cbsbf = logPerBlock(blockCache); LOG.info(Objects.toString(cbsbf)); logPerFile(cbsbf); bucketCacheReport(blockCache); LOG.info(BlockCacheUtil.toJSON(cbsbf)); }
@Test public void testLruBlockCache() throws IOException { CacheConfig cc = new CacheConfig(this.conf); assertTrue(CacheConfig.DEFAULT_IN_MEMORY == cc.isInMemory()); BlockCache blockCache = BlockCacheFactory.createBlockCache(this.conf); assertTrue(blockCache instanceof LruBlockCache); logPerBlock(blockCache); addDataAndHits(blockCache, 3); // The below has no asserts. It is just exercising toString and toJSON code. LOG.info("count=" + blockCache.getBlockCount() + ", currentSize=" + blockCache.getCurrentSize() + ", freeSize=" + blockCache.getFreeSize()); LOG.info(Objects.toString(blockCache.getStats())); BlockCacheUtil.CachedBlocksByFile cbsbf = logPerBlock(blockCache); LOG.info(Objects.toString(cbsbf)); logPerFile(cbsbf); bucketCacheReport(blockCache); LOG.info(BlockCacheUtil.toJSON(cbsbf)); }
assertTrue(toBeTested.getStats().getEvictedCount() > 0);
long startBlockHits = cache.getStats().getHitCount(); long startBlockMiss = cache.getStats().getMissCount(); Thread.sleep(100); if (startBlockCount != cache.getBlockCount() || startBlockHits != cache.getStats().getHitCount() || startBlockMiss != cache.getStats().getMissCount()) { startBlockCount = cache.getBlockCount(); startBlockHits = cache.getStats().getHitCount(); startBlockMiss = cache.getStats().getMissCount(); i = -1; assertEquals(startBlockHits, cache.getStats().getHitCount()); assertEquals(startBlockMiss, cache.getStats().getMissCount()); long expectedBlockMiss = startBlockMiss; assertEquals(expectedBlockCount, cache.getBlockCount()); assertEquals(expectedBlockHits, cache.getStats().getHitCount()); assertEquals(expectedBlockMiss, cache.getStats().getMissCount()); assertEquals(++expectedBlockHits, cache.getStats().getHitCount()); assertEquals(expectedBlockMiss, cache.getStats().getMissCount()); assertTrue(Bytes.equals(r.getValue(FAMILY, QUALIFIER2), data2)); assertEquals(expectedBlockCount, cache.getBlockCount()); assertEquals(++expectedBlockHits, cache.getStats().getHitCount()); assertEquals(expectedBlockMiss, cache.getStats().getMissCount()); assertEquals(expectedBlockHits, cache.getStats().getHitCount());
r.flush(true); CacheStats stats = blockCache.getStats(); long before = stats.getHitCount() + stats.getMissCount();
private void addDataAndHits(final BlockCache bc, final int count) { Cacheable dce = new DataCacheEntry(); Cacheable ice = new IndexCacheEntry(); for (int i = 0; i < count; i++) { BlockCacheKey bckd = new BlockCacheKey("f", i); BlockCacheKey bcki = new BlockCacheKey("f", i + count); bc.getBlock(bckd, true, false, true); bc.cacheBlock(bckd, dce); bc.cacheBlock(bcki, ice); bc.getBlock(bckd, true, false, true); bc.getBlock(bcki, true, false, true); } assertEquals(2 * count /*Data and Index blocks*/, bc.getStats().getHitCount()); BlockCacheKey bckd = new BlockCacheKey("f", 0); BlockCacheKey bcki = new BlockCacheKey("f", 0 + count); bc.evictBlock(bckd); bc.evictBlock(bcki); bc.getStats().getEvictedCount(); }
@Test public void testCacheBlocks() throws IOException { // Set index block size to be the same as normal block size. TEST_UTIL.getConfiguration().setInt(HFileBlockIndex.MAX_CHUNK_SIZE_KEY, BLOCK_SIZE); BlockCache blockCache = BlockCacheFactory.createBlockCache(TEST_UTIL.getConfiguration()); ColumnFamilyDescriptor cfd = ColumnFamilyDescriptorBuilder.newBuilder(Bytes.toBytes(CF)).setMaxVersions(MAX_VERSIONS) .setCompressionType(COMPRESSION_ALGORITHM).setBloomFilterType(BLOOM_TYPE) .setBlocksize(BLOCK_SIZE).setBlockCacheEnabled(cfCacheEnabled).build(); HRegion region = TEST_UTIL.createTestRegion(TABLE, cfd, blockCache); CacheStats stats = blockCache.getStats(); writeTestData(region); assertEquals(0, stats.getHitCount()); assertEquals(0, HFile.DATABLOCK_READ_COUNT.sum()); // Do a single get, take count of caches. If we are NOT caching DATA blocks, the miss // count should go up. Otherwise, all should be cached and the miss count should not rise. region.get(new Get(Bytes.toBytes("row" + 0))); assertTrue(stats.getHitCount() > 0); assertTrue(HFile.DATABLOCK_READ_COUNT.sum() > 0); long missCount = stats.getMissCount(); region.get(new Get(Bytes.toBytes("row" + 0))); if (this.cfCacheEnabled) assertEquals(missCount, stats.getMissCount()); else assertTrue(stats.getMissCount() > missCount); }
CacheStats cs = bc.getStats(); long startHit = cs.getHitCount(); long startMiss = cs.getMissCount();
/** * It's possible that due to threading the block cache could not be initialized * yet (testing multiple region servers in one jvm). So we need to try and initialize * the blockCache and cacheStats reference multiple times until we succeed. */ private synchronized void initBlockCache() { CacheConfig cacheConfig = this.regionServer.cacheConfig; if (cacheConfig != null && this.blockCache == null) { this.blockCache = cacheConfig.getBlockCache(); } if (this.blockCache != null && this.cacheStats == null) { this.cacheStats = blockCache.getStats(); } }
public CombinedBlockCache(LruBlockCache lruCache, BlockCache l2Cache) { this.lruCache = lruCache; this.l2Cache = l2Cache; this.combinedCacheStats = new CombinedCacheStats(lruCache.getStats(), l2Cache.getStats()); }
@Test public void testBucketCache() throws IOException { this.conf.set(HConstants.BUCKET_CACHE_IOENGINE_KEY, "offheap"); this.conf.setInt(HConstants.BUCKET_CACHE_SIZE_KEY, 100); CacheConfig.instantiateBlockCache(this.conf); CacheConfig cc = new CacheConfig(this.conf); assertTrue(cc.getBlockCache() instanceof CombinedBlockCache); logPerBlock(cc.getBlockCache()); final int count = 3; addDataAndHits(cc.getBlockCache(), count); // The below has no asserts. It is just exercising toString and toJSON code. LOG.info(Objects.toString(cc.getBlockCache().getStats())); BlockCacheUtil.CachedBlocksByFile cbsbf = logPerBlock(cc.getBlockCache()); LOG.info(Objects.toString(cbsbf)); logPerFile(cbsbf); bucketCacheReport(cc.getBlockCache()); LOG.info(BlockCacheUtil.toJSON(cbsbf)); }
r.flush(true); CacheStats stats = new CacheConfig(TEST_UTIL.getConfiguration()).getBlockCache().getStats(); long before = stats.getHitCount() + stats.getMissCount();
@Test public void testLruBlockCache() throws IOException { CacheConfig.instantiateBlockCache(this.conf); CacheConfig cc = new CacheConfig(this.conf); assertTrue(cc.isBlockCacheEnabled()); assertTrue(CacheConfig.DEFAULT_IN_MEMORY == cc.isInMemory()); assertTrue(cc.getBlockCache() instanceof LruBlockCache); logPerBlock(cc.getBlockCache()); addDataAndHits(cc.getBlockCache(), 3); // The below has no asserts. It is just exercising toString and toJSON code. BlockCache bc = cc.getBlockCache(); LOG.info("count=" + bc.getBlockCount() + ", currentSize=" + bc.getCurrentSize() + ", freeSize=" + bc.getFreeSize() ); LOG.info(Objects.toString(cc.getBlockCache().getStats())); BlockCacheUtil.CachedBlocksByFile cbsbf = logPerBlock(cc.getBlockCache()); LOG.info(Objects.toString(cbsbf)); logPerFile(cbsbf); bucketCacheReport(cc.getBlockCache()); LOG.info(BlockCacheUtil.toJSON(cbsbf)); }
private void addDataAndHits(final BlockCache bc, final int count) { Cacheable dce = new DataCacheEntry(); Cacheable ice = new IndexCacheEntry(); for (int i = 0; i < count; i++) { BlockCacheKey bckd = new BlockCacheKey("f", i); BlockCacheKey bcki = new BlockCacheKey("f", i + count); bc.getBlock(bckd, true, false, true); bc.cacheBlock(bckd, dce); bc.cacheBlock(bcki, ice); bc.getBlock(bckd, true, false, true); bc.getBlock(bcki, true, false, true); } assertEquals(2 * count /*Data and Index blocks*/, bc.getStats().getHitCount()); BlockCacheKey bckd = new BlockCacheKey("f", 0); BlockCacheKey bcki = new BlockCacheKey("f", 0 + count); bc.evictBlock(bckd); bc.evictBlock(bcki); bc.getStats().getEvictedCount(); }