@Override public Cacheable getBlock(BlockCacheKey cacheKey, boolean caching, boolean repeat, boolean updateCacheMetrics) { // On all external cache set ups the lru should have the l2 cache set as the victimHandler // Because of that all requests that miss inside of the lru block cache will be // tried in the l2 block cache. return onHeapCache.getBlock(cacheKey, caching, repeat, updateCacheMetrics); }
cache.getBlock(block.cacheKey, true, false, true); assertEquals(null, cache.getBlock(singleBlocks[0].cacheKey, true, false, true)); assertEquals(null, cache.getBlock(singleBlocks[1].cacheKey, true, false, true)); assertEquals(null, cache.getBlock(multiBlocks[0].cacheKey, true, false, true)); assertEquals(null, cache.getBlock(multiBlocks[1].cacheKey, true, false, true));
@Override public Cacheable getBlock(BlockCacheKey cacheKey, boolean caching, boolean repeat, boolean updateCacheMetrics) { // TODO: is there a hole here, or just awkwardness since in the lruCache getBlock // we end up calling l2Cache.getBlock. // We are not in a position to exactly look at LRU cache or BC as BlockType may not be getting // passed always. return onHeapCache.containsBlock(cacheKey)? onHeapCache.getBlock(cacheKey, caching, repeat, updateCacheMetrics): l2Cache.getBlock(cacheKey, caching, repeat, updateCacheMetrics); }
assertTrue(cache.getBlock(block.cacheKey, true, false, true) == null); HeapSize buf = cache.getBlock(block.cacheKey, true, false, true); assertTrue(buf != null); assertEquals(buf.heapSize(), block.heapSize()); HeapSize buf = cache.getBlock(block.cacheKey, true, false, true); assertTrue(buf != null); assertEquals(buf.heapSize(), block.heapSize());
cache.getBlock(multiBlocks[i].cacheKey, true, false, true); assertEquals(null, cache.getBlock(singleBlocks[i].cacheKey, true, false, true)); assertEquals(null, cache.getBlock(multiBlocks[i].cacheKey, true, false, true)); assertEquals(null, cache.getBlock(memoryBlocks[i].cacheKey, true, false, true)); assertEquals(singleBlocks[i], cache.getBlock(singleBlocks[i].cacheKey, true, false, true)); assertEquals(multiBlocks[i], cache.getBlock(multiBlocks[i].cacheKey, true, false, true)); assertEquals(memoryBlocks[i], cache.getBlock(memoryBlocks[i].cacheKey, true, false, true));
cache.cacheBlock(block.cacheKey, block); expectedCacheSize += block.cacheBlockHeapSize(); assertEquals(cache.getBlock(block.cacheKey, true, false, true), block); assertTrue(cache.getBlock(singleBlocks[0].cacheKey, true, false, true) == null); assertTrue(cache.getBlock(multiBlocks[0].cacheKey, true, false, true) == null); assertEquals(cache.getBlock(singleBlocks[i].cacheKey, true, false, true), singleBlocks[i]); assertEquals(cache.getBlock(multiBlocks[i].cacheKey, true, false, true), multiBlocks[i]);
assertTrue(cache.getBlock(block.cacheKey, true, false, true) == null); HeapSize buf = cache.getBlock(block.cacheKey, true, false, true); assertTrue(buf != null); assertEquals(buf.heapSize(), block.heapSize()); HeapSize buf = cache.getBlock(block.cacheKey, true, false, true); assertTrue(buf != null); assertEquals(buf.heapSize(), block.heapSize());
assertNull(cache.getBlock(key, false, false, false)); CacheTestUtils.getBlockAndAssertEquals(cache, key, blockWithoutNextBlockMetadata, actualBuffer, block2Buffer);
cache.getBlock(multiBlocks[i].cacheKey, true, false, true); assertEquals(null, cache.getBlock(singleBlocks[0].cacheKey, true, false, true)); cache.getBlock(singleBlocks[1].cacheKey, true, false, true); assertEquals(null, cache.getBlock(multiBlocks[0].cacheKey, true, false, true)); assertEquals(null, cache.getBlock(memoryBlocks[0].cacheKey, true, false, true)); assertEquals(null, cache.getBlock(singleBlocks[2].cacheKey, true, false, true)); assertEquals(null, cache.getBlock(singleBlocks[3].cacheKey, true, false, true)); assertEquals(null, cache.getBlock(singleBlocks[4].cacheKey, true, false, true)); cache.getBlock(bigBlocks[0].cacheKey, true, false, true); assertEquals(null, cache.getBlock(singleBlocks[1].cacheKey, true, false, true)); assertEquals(null, cache.getBlock(multiBlocks[1].cacheKey, true, false, true)); assertEquals(null, cache.getBlock(multiBlocks[2].cacheKey, true, false, true)); assertEquals(null, cache.getBlock(memoryBlocks[1].cacheKey, true, false, true)); assertEquals(null, cache.getBlock(memoryBlocks[2].cacheKey, true, false, true)); assertEquals(null, cache.getBlock(memoryBlocks[3].cacheKey, true, false, true));
@Test public void testCacheEvictionSimple() throws Exception { long maxSize = 100000; long blockSize = calculateBlockSizeDefault(maxSize, 10); LruBlockCache cache = new LruBlockCache(maxSize,blockSize,false); CachedItem [] blocks = generateFixedBlocks(10, blockSize, "block"); long expectedCacheSize = cache.heapSize(); // Add all the blocks for (CachedItem block : blocks) { cache.cacheBlock(block.cacheKey, block); expectedCacheSize += block.cacheBlockHeapSize(); } // A single eviction run should have occurred assertEquals(1, cache.getStats().getEvictionCount()); // Our expected size overruns acceptable limit assertTrue(expectedCacheSize > (maxSize * LruBlockCache.DEFAULT_ACCEPTABLE_FACTOR)); // But the cache did not grow beyond max assertTrue(cache.heapSize() < maxSize); // And is still below the acceptable limit assertTrue(cache.heapSize() < (maxSize * LruBlockCache.DEFAULT_ACCEPTABLE_FACTOR)); // All blocks except block 0 should be in the cache assertTrue(cache.getBlock(blocks[0].cacheKey, true, false, true) == null); for(int i=1;i<blocks.length;i++) { assertEquals(cache.getBlock(blocks[i].cacheKey, true, false, true), blocks[i]); } }
private void testEncodingWithCacheInternals(boolean useTag) throws IOException { List<KeyValue> kvs = generator.generateTestKeyValues(60, useTag); HFileBlock block = getSampleHFileBlock(kvs, useTag); HFileBlock cacheBlock = createBlockOnDisk(kvs, block, useTag); LruBlockCache blockCache = new LruBlockCache(8 * 1024 * 1024, 32 * 1024); BlockCacheKey cacheKey = new BlockCacheKey("test", 0); blockCache.cacheBlock(cacheKey, cacheBlock); HeapSize heapSize = blockCache.getBlock(cacheKey, false, false, true); assertTrue(heapSize instanceof HFileBlock); HFileBlock returnedBlock = (HFileBlock) heapSize; if (blockEncoder.getDataBlockEncoding() == DataBlockEncoding.NONE) { assertEquals(block.getBufferReadOnly(), returnedBlock.getBufferReadOnly()); } else { if (BlockType.ENCODED_DATA != returnedBlock.getBlockType()) { System.out.println(blockEncoder); } assertEquals(BlockType.ENCODED_DATA, returnedBlock.getBlockType()); } }
cache.getBlock(multiBlocks[i].cacheKey, true, false, true); assertEquals(1, cache.getStats().getEvictedCount()); assertEquals(null, cache.getBlock(singleBlocks[0].cacheKey, true, false, true)); assertEquals(2, cache.getStats().getEvictedCount()); assertEquals(null, cache.getBlock(singleBlocks[1].cacheKey, true, false, true)); assertEquals(6, cache.getStats().getEvictedCount()); assertEquals(null, cache.getBlock(singleBlocks[2].cacheKey, true, false, true)); assertEquals(null, cache.getBlock(singleBlocks[3].cacheKey, true, false, true)); assertEquals(null, cache.getBlock(multiBlocks[0].cacheKey, true, false, true)); assertEquals(null, cache.getBlock(multiBlocks[1].cacheKey, true, false, true)); assertEquals(9, cache.getStats().getEvictedCount()); assertEquals(null, cache.getBlock(singleBlocks[4].cacheKey, true, false, true)); assertEquals(null, cache.getBlock(multiBlocks[2].cacheKey, true, false, true)); assertEquals(null, cache.getBlock(multiBlocks[3].cacheKey, true, false, true)); assertEquals(10, cache.getStats().getEvictedCount()); assertEquals(null, cache.getBlock(memoryBlocks[0].cacheKey, true, false, true)); assertEquals(null, cache.getBlock(singleBlocks[9].cacheKey, true, false, true));
@Override public Cacheable getBlock(BlockCacheKey cacheKey, boolean caching, boolean repeat, boolean updateCacheMetrics) { // On all external cache set ups the lru should have the l2 cache set as the victimHandler // Because of that all requests that miss inside of the lru block cache will be // tried in the l2 block cache. return lruCache.getBlock(cacheKey, caching, repeat, updateCacheMetrics); }
@Override public Cacheable getBlock(BlockCacheKey cacheKey, boolean caching, boolean repeat, boolean updateCacheMetrics) { // TODO: is there a hole here, or just awkwardness since in the lruCache getBlock // we end up calling l2Cache.getBlock. if (lruCache.containsBlock(cacheKey)) { return lruCache.getBlock(cacheKey, caching, repeat, updateCacheMetrics); } Cacheable result = l2Cache.getBlock(cacheKey, caching, repeat, updateCacheMetrics); return result; }
@Override public Cacheable getBlock(BlockCacheKey cacheKey, boolean caching, boolean repeat) { Cacheable cachedBlock; if ((cachedBlock = onHeapCache.getBlock(cacheKey, caching, repeat)) != null) { stats.hit(caching); return cachedBlock; } else if ((cachedBlock = offHeapCache.getBlock(cacheKey, caching, repeat)) != null) { if (caching) { onHeapCache.cacheBlock(cacheKey, cachedBlock); } stats.hit(caching); return cachedBlock; } if (!repeat) stats.miss(caching); return null; }
assertNull(cache.getBlock(key, false, false, false)); CacheTestUtils.getBlockAndAssertEquals(cache, key, blockWithoutNextBlockMetadata, actualBuffer, block2Buffer);
private void testEncodingWithCacheInternals(boolean useTag) throws IOException { List<KeyValue> kvs = generator.generateTestKeyValues(60, useTag); HFileBlock block = getSampleHFileBlock(kvs, useTag); HFileBlock cacheBlock = createBlockOnDisk(kvs, block, useTag); LruBlockCache blockCache = new LruBlockCache(8 * 1024 * 1024, 32 * 1024); BlockCacheKey cacheKey = new BlockCacheKey("test", 0); blockCache.cacheBlock(cacheKey, cacheBlock); HeapSize heapSize = blockCache.getBlock(cacheKey, false, false, true); assertTrue(heapSize instanceof HFileBlock); HFileBlock returnedBlock = (HFileBlock) heapSize;; if (blockEncoder.getDataBlockEncoding() == DataBlockEncoding.NONE) { assertEquals(block.getBufferReadOnly(), returnedBlock.getBufferReadOnly()); } else { if (BlockType.ENCODED_DATA != returnedBlock.getBlockType()) { System.out.println(blockEncoder); } assertEquals(BlockType.ENCODED_DATA, returnedBlock.getBlockType()); } }
@Test public void testCacheEvictionSimple() throws Exception { long maxSize = 100000; long blockSize = calculateBlockSizeDefault(maxSize, 10); LruBlockCache cache = new LruBlockCache(maxSize,blockSize,false); CachedItem [] blocks = generateFixedBlocks(10, blockSize, "block"); long expectedCacheSize = cache.heapSize(); // Add all the blocks for (CachedItem block : blocks) { cache.cacheBlock(block.cacheKey, block); expectedCacheSize += block.cacheBlockHeapSize(); } // A single eviction run should have occurred assertEquals(1, cache.getStats().getEvictionCount()); // Our expected size overruns acceptable limit assertTrue(expectedCacheSize > (maxSize * LruBlockCache.DEFAULT_ACCEPTABLE_FACTOR)); // But the cache did not grow beyond max assertTrue(cache.heapSize() < maxSize); // And is still below the acceptable limit assertTrue(cache.heapSize() < (maxSize * LruBlockCache.DEFAULT_ACCEPTABLE_FACTOR)); // All blocks except block 0 should be in the cache assertTrue(cache.getBlock(blocks[0].cacheKey, true, false, true) == null); for(int i=1;i<blocks.length;i++) { assertEquals(cache.getBlock(blocks[i].cacheKey, true, false, true), blocks[i]); } }