@Override public void shutdown() { onHeapCache.shutdown(); l2Cache.shutdown(); }
@Test public void testLruBlockCache() throws IOException { CacheConfig cc = new CacheConfig(this.conf); assertTrue(CacheConfig.DEFAULT_IN_MEMORY == cc.isInMemory()); BlockCache blockCache = BlockCacheFactory.createBlockCache(this.conf); assertTrue(blockCache instanceof LruBlockCache); logPerBlock(blockCache); addDataAndHits(blockCache, 3); // The below has no asserts. It is just exercising toString and toJSON code. LOG.info("count=" + blockCache.getBlockCount() + ", currentSize=" + blockCache.getCurrentSize() + ", freeSize=" + blockCache.getFreeSize()); LOG.info(Objects.toString(blockCache.getStats())); BlockCacheUtil.CachedBlocksByFile cbsbf = logPerBlock(blockCache); LOG.info(Objects.toString(cbsbf)); logPerFile(cbsbf); bucketCacheReport(blockCache); LOG.info(BlockCacheUtil.toJSON(cbsbf)); }
private void addDataAndHits(final BlockCache bc, final int count) { Cacheable dce = new DataCacheEntry(); Cacheable ice = new IndexCacheEntry(); for (int i = 0; i < count; i++) { BlockCacheKey bckd = new BlockCacheKey("f", i); BlockCacheKey bcki = new BlockCacheKey("f", i + count); bc.getBlock(bckd, true, false, true); bc.cacheBlock(bckd, dce); bc.cacheBlock(bcki, ice); bc.getBlock(bckd, true, false, true); bc.getBlock(bcki, true, false, true); } assertEquals(2 * count /*Data and Index blocks*/, bc.getStats().getHitCount()); BlockCacheKey bckd = new BlockCacheKey("f", 0); BlockCacheKey bcki = new BlockCacheKey("f", 0 + count); bc.evictBlock(bckd); bc.evictBlock(bcki); bc.getStats().getEvictedCount(); }
@Override public void doAnAction() throws Exception { toBeTested.evictBlock(key); toBeTested.cacheBlock(key, bac); Thread.sleep(blockEvictPeriod); } };
Cacheable existingBlock = blockCache.getBlock(cacheKey, false, false, false); try { int comparison = BlockCacheUtil.validateBlockAddition(existingBlock, newBlock, cacheKey); blockCache.returnBlock(cacheKey, existingBlock);
private void bucketCacheReport(final BlockCache bc) { LOG.info(bc.getClass().getSimpleName() + ": " + bc.getStats()); BlockCache [] bcs = bc.getBlockCaches(); if (bcs != null) { for (BlockCache sbc: bc.getBlockCaches()) { LOG.info(bc.getClass().getSimpleName() + ": " + sbc.getStats()); } } }
/** * @param bc The block cache instance. * @param cc Cache config. * @param doubling If true, addition of element ups counter by 2, not 1, because element added * to onheap and offheap caches. * @param sizing True if we should run sizing test (doesn't always apply). */ void basicBlockCacheOps(final BlockCache bc, final CacheConfig cc, final boolean doubling, final boolean sizing) { assertTrue(CacheConfig.DEFAULT_IN_MEMORY == cc.isInMemory()); BlockCacheKey bck = new BlockCacheKey("f", 0); Cacheable c = new DataCacheEntry(); // Do asserts on block counting. long initialBlockCount = bc.getBlockCount(); bc.cacheBlock(bck, c, cc.isInMemory()); assertEquals(doubling ? 2 : 1, bc.getBlockCount() - initialBlockCount); bc.evictBlock(bck); assertEquals(initialBlockCount, bc.getBlockCount()); // Do size accounting. Do it after the above 'warm-up' because it looks like some // buffers do lazy allocation so sizes are off on first go around. if (sizing) { long originalSize = bc.getCurrentSize(); bc.cacheBlock(bck, c, cc.isInMemory()); assertTrue(bc.getCurrentSize() > originalSize); bc.evictBlock(bck); long size = bc.getCurrentSize(); assertEquals(originalSize, size); } }
bucketCacheStats = (BucketCacheStats)bc.getStats(); bucketAllocator = ((BucketCache)bc).getAllocator(); org.jamon.escaping.Escaping.HTML.write(org.jamon.emit.StandardEmitter.valueOf(TraditionalBinaryPrefix.long2String(bc.getMaxSize(), "B", 1)), jamonWriter); org.jamon.escaping.Escaping.HTML.write(org.jamon.emit.StandardEmitter.valueOf(String.format("%,d", bc.getBlockCount())), jamonWriter); org.jamon.escaping.Escaping.HTML.write(org.jamon.emit.StandardEmitter.valueOf(String.format("%,d", bc.getDataBlockCount())), jamonWriter); org.jamon.escaping.Escaping.HTML.write(org.jamon.emit.StandardEmitter.valueOf(TraditionalBinaryPrefix.long2String(bc.getCurrentSize(), "B", 1)), jamonWriter); org.jamon.escaping.Escaping.HTML.write(org.jamon.emit.StandardEmitter.valueOf(TraditionalBinaryPrefix.long2String(bc.getCurrentDataSize(), "B", 1)), jamonWriter);
org.jamon.escaping.Escaping.HTML.write(org.jamon.emit.StandardEmitter.valueOf(TraditionalBinaryPrefix.long2String(bc.getCurrentSize(), "B", 1)), jamonWriter); org.jamon.escaping.Escaping.HTML.write(org.jamon.emit.StandardEmitter.valueOf(TraditionalBinaryPrefix.long2String(bc.getFreeSize(), "B", 1)), jamonWriter); org.jamon.escaping.Escaping.HTML.write(org.jamon.emit.StandardEmitter.valueOf(String.format("%,d", bc.getBlockCount())), jamonWriter);
this.metrics.blockCacheCount.set(blockCache.size()); this.metrics.blockCacheFree.set(blockCache.getFreeSize()); this.metrics.blockCacheSize.set(blockCache.getCurrentSize()); CacheStats cacheStats = blockCache.getStats(); this.metrics.blockCacheHitCount.set(cacheStats.getHitCount()); this.metrics.blockCacheMissCount.set(cacheStats.getMissCount()); this.metrics.blockCacheEvictedCount.set(blockCache.getEvictedCount()); double ratio = blockCache.getStats().getHitRatio(); int percent = (int) (ratio * 100); this.metrics.blockCacheHitRatio.set(percent); ratio = blockCache.getStats().getHitCachingRatio(); percent = (int) (ratio * 100); this.metrics.blockCacheHitCachingRatio.set(percent);
@Override public void doAnAction() throws Exception { if (!blocksToTest.isEmpty()) { HFileBlockPair ourBlock = blocksToTest.poll(); // if we run out of blocks to test, then we should stop the tests. if (ourBlock == null) { ctx.setStopFlag(true); return; } toBeTested.cacheBlock(ourBlock.blockName, ourBlock.block); Cacheable retrievedBlock = toBeTested.getBlock(ourBlock.blockName, false, false, true); if (retrievedBlock != null) { assertEquals(ourBlock.block, retrievedBlock); toBeTested.evictBlock(ourBlock.blockName); hits.incrementAndGet(); assertNull(toBeTested.getBlock(ourBlock.blockName, false, false, true)); } else { miss.incrementAndGet(); } totalQueries.incrementAndGet(); } } };
public static void getBlockAndAssertEquals(BlockCache cache, BlockCacheKey key, Cacheable blockToCache, ByteBuffer destBuffer, ByteBuffer expectedBuffer) { destBuffer.clear(); cache.cacheBlock(key, blockToCache); Cacheable actualBlock = cache.getBlock(key, false, false, false); actualBlock.serialize(destBuffer, true); assertEquals(expectedBuffer, destBuffer); cache.returnBlock(key, actualBlock); } }
@Override public int evictBlocksByHfileName(String hfileName) { return onHeapCache.evictBlocksByHfileName(hfileName) + l2Cache.evictBlocksByHfileName(hfileName); }
if (cache != null) { HFileBlock cachedBlock = (HFileBlock) cache.getBlock(cacheKey, cacheBlock, useLock, updateCacheMetrics); if (cachedBlock != null) { if (cacheConf.shouldCacheCompressed(cachedBlock.getBlockType().getCategory())) { cache.returnBlock(cacheKey, compressedBlock); cache.returnBlock(cacheKey, cachedBlock); cache.evictBlock(cacheKey);
public CombinedBlockCache(LruBlockCache onHeapCache, BlockCache l2Cache) { this.onHeapCache = onHeapCache; this.l2Cache = l2Cache; this.combinedCacheStats = new CombinedCacheStats(onHeapCache.getStats(), l2Cache.getStats()); }
@Override public void doAnAction() throws Exception { for (int j = 0; j < 100; j++) { BlockCacheKey key = new BlockCacheKey("key_" + finalI + "_" + j, 0); Arrays.fill(buf, (byte) (finalI * j)); final ByteArrayCacheable bac = new ByteArrayCacheable(buf); ByteArrayCacheable gotBack = (ByteArrayCacheable) toBeTested .getBlock(key, true, false, true); if (gotBack != null) { assertArrayEquals(gotBack.buf, bac.buf); } else { toBeTested.cacheBlock(key, bac); } } totalQueries.incrementAndGet(); } };
/** * * @param cacheKey The block's cache key. * @param buf The block contents wrapped in a ByteBuffer. * @param inMemory Whether block should be treated as in-memory. This parameter is only useful for * the L1 lru cache. */ @Override public void cacheBlock(BlockCacheKey cacheKey, Cacheable buf, boolean inMemory) { // This is the inclusive part of the combined block cache. // Every block is placed into both block caches. onHeapCache.cacheBlock(cacheKey, buf, inMemory); // This assumes that insertion into the L2 block cache is either async or very fast. l2Cache.cacheBlock(cacheKey, buf, inMemory); }
@Override public long getCurrentSize() { return onHeapCache.getCurrentSize() + l2Cache.getCurrentSize(); }
@Override public void run() { Scan s = new Scan().withStartRow(ROW4).withStopRow(ROW5).setCaching(1); try { while(!doScan.get()) { try { // Sleep till you start scan Thread.sleep(1); } catch (InterruptedException e) { } } List<BlockCacheKey> cacheList = new ArrayList<>(); Iterator<CachedBlock> iterator = cache.iterator(); // evict all the blocks while (iterator.hasNext()) { CachedBlock next = iterator.next(); BlockCacheKey cacheKey = new BlockCacheKey(next.getFilename(), next.getOffset()); cacheList.add(cacheKey); // evict what ever is available cache.evictBlock(cacheKey); } try (ResultScanner scanner = table.getScanner(s)) { while (scanner.next() != null) { } } compactReadLatch.countDown(); } catch (IOException e) { } } }
@Override public Cacheable getBlock(BlockCacheKey cacheKey, boolean caching, boolean repeat, boolean updateCacheMetrics) { // TODO: is there a hole here, or just awkwardness since in the lruCache getBlock // we end up calling l2Cache.getBlock. // We are not in a position to exactly look at LRU cache or BC as BlockType may not be getting // passed always. return onHeapCache.containsBlock(cacheKey)? onHeapCache.getBlock(cacheKey, caching, repeat, updateCacheMetrics): l2Cache.getBlock(cacheKey, caching, repeat, updateCacheMetrics); }