BucketCache.DEFAULT_ERROR_TOLERATION_DURATION); bucketCache = new BucketCache(bucketCacheIOEngineName, bucketCacheSize, blockSize, bucketSizes, writerThreads, writerQueueLen, persistentPath, ioErrorsTolerationDuration, c);
private void checkConfigValues(Configuration conf, Map<String, float[]> configMap, boolean[] expectSuccess) throws IOException { Set<String> configNames = configMap.keySet(); for (int i = 0; i < expectSuccess.length; i++) { try { for (String configName : configNames) { conf.setFloat(configName, configMap.get(configName)[i]); } BucketCache cache = new BucketCache(ioEngineName, capacitySize, constructedBlockSize, constructedBlockSizes, writeThreads, writerQLen, persistencePath, 100, conf); assertTrue("Created BucketCache and expected it to succeed: " + expectSuccess[i] + ", but it actually was: " + !expectSuccess[i], expectSuccess[i]); } catch (IllegalArgumentException e) { assertFalse("Created BucketCache and expected it to succeed: " + expectSuccess[i] + ", but it actually was: " + !expectSuccess[i], expectSuccess[i]); } } }
private static List<BlockCache> getBlockCaches() throws IOException { Configuration conf = TEST_UTIL.getConfiguration(); List<BlockCache> blockcaches = new ArrayList<>(); // default blockcaches.add(BlockCacheFactory.createBlockCache(conf)); //set LruBlockCache.LRU_HARD_CAPACITY_LIMIT_FACTOR_CONFIG_NAME to 2.0f due to HBASE-16287 TEST_UTIL.getConfiguration().setFloat(LruBlockCache.LRU_HARD_CAPACITY_LIMIT_FACTOR_CONFIG_NAME, 2.0f); // memory BlockCache lru = new LruBlockCache(128 * 1024 * 1024, 64 * 1024, TEST_UTIL.getConfiguration()); blockcaches.add(lru); // bucket cache FileSystem.get(conf).mkdirs(TEST_UTIL.getDataTestDir()); int[] bucketSizes = { INDEX_BLOCK_SIZE, DATA_BLOCK_SIZE, BLOOM_BLOCK_SIZE, 64 * 1024, 128 * 1024 }; BlockCache bucketcache = new BucketCache("offheap", 128 * 1024 * 1024, 64 * 1024, bucketSizes, 5, 64 * 100, null); blockcaches.add(bucketcache); return blockcaches; }
@Test public void testGetPartitionSize() throws IOException { //Test default values validateGetPartitionSize(cache, BucketCache.DEFAULT_SINGLE_FACTOR, BucketCache.DEFAULT_MIN_FACTOR); Configuration conf = HBaseConfiguration.create(); conf.setFloat(BucketCache.MIN_FACTOR_CONFIG_NAME, 0.5f); conf.setFloat(BucketCache.SINGLE_FACTOR_CONFIG_NAME, 0.1f); conf.setFloat(BucketCache.MULTI_FACTOR_CONFIG_NAME, 0.7f); conf.setFloat(BucketCache.MEMORY_FACTOR_CONFIG_NAME, 0.2f); BucketCache cache = new BucketCache(ioEngineName, capacitySize, constructedBlockSize, constructedBlockSizes, writeThreads, writerQLen, persistencePath, 100, conf); validateGetPartitionSize(cache, 0.1f, 0.5f); validateGetPartitionSize(cache, 0.7f, 0.5f); validateGetPartitionSize(cache, 0.2f, 0.5f); }
String persistencePath = testDir + "/bucket.persistence"; BucketCache bucketCache = new BucketCache(ioEngineName, capacitySize, constructedBlockSize, constructedBlockSizes, writeThreads, writerQLen, persistencePath); long usedSize = bucketCache.getAllocator().getUsedSize(); bucketCache = new BucketCache(ioEngineName, capacitySize, constructedBlockSize, constructedBlockSizes, writeThreads, writerQLen, persistencePath); assertFalse(new File(persistencePath).exists()); bucketCache = new BucketCache(ioEngineName, capacitySize, constructedBlockSize, smallBucketSizes, writeThreads, writerQLen, persistencePath); assertFalse(new File(persistencePath).exists());
@Test public void testValidBucketCacheConfigs() throws IOException { Configuration conf = HBaseConfiguration.create(); conf.setFloat(BucketCache.ACCEPT_FACTOR_CONFIG_NAME, 0.9f); conf.setFloat(BucketCache.MIN_FACTOR_CONFIG_NAME, 0.5f); conf.setFloat(BucketCache.EXTRA_FREE_FACTOR_CONFIG_NAME, 0.5f); conf.setFloat(BucketCache.SINGLE_FACTOR_CONFIG_NAME, 0.1f); conf.setFloat(BucketCache.MULTI_FACTOR_CONFIG_NAME, 0.7f); conf.setFloat(BucketCache.MEMORY_FACTOR_CONFIG_NAME, 0.2f); BucketCache cache = new BucketCache(ioEngineName, capacitySize, constructedBlockSize, constructedBlockSizes, writeThreads, writerQLen, persistencePath, 100, conf); assertEquals(BucketCache.ACCEPT_FACTOR_CONFIG_NAME + " failed to propagate.", 0.9f, cache.getAcceptableFactor(), 0); assertEquals(BucketCache.MIN_FACTOR_CONFIG_NAME + " failed to propagate.", 0.5f, cache.getMinFactor(), 0); assertEquals(BucketCache.EXTRA_FREE_FACTOR_CONFIG_NAME + " failed to propagate.", 0.5f, cache.getExtraFreeFactor(), 0); assertEquals(BucketCache.SINGLE_FACTOR_CONFIG_NAME + " failed to propagate.", 0.1f, cache.getSingleFactor(), 0); assertEquals(BucketCache.MULTI_FACTOR_CONFIG_NAME + " failed to propagate.", 0.7f, cache.getMultiFactor(), 0); assertEquals(BucketCache.MEMORY_FACTOR_CONFIG_NAME + " failed to propagate.", 0.2f, cache.getMemoryFactor(), 0); }
BucketCache.DEFAULT_ERROR_TOLERATION_DURATION); bucketCache = new BucketCache(bucketCacheIOEngineName, bucketCacheSize, blockSize, bucketSizes, writerThreads, writerQueueLen, persistentPath, ioErrorsTolerationDuration);
private void checkConfigValues(Configuration conf, Map<String, float[]> configMap, boolean[] expectSuccess) throws IOException { Set<String> configNames = configMap.keySet(); for (int i = 0; i < expectSuccess.length; i++) { try { for (String configName : configNames) { conf.setFloat(configName, configMap.get(configName)[i]); } BucketCache cache = new BucketCache(ioEngineName, capacitySize, constructedBlockSize, constructedBlockSizes, writeThreads, writerQLen, persistencePath, 100, conf); assertTrue("Created BucketCache and expected it to succeed: " + expectSuccess[i] + ", but it actually was: " + !expectSuccess[i], expectSuccess[i]); } catch (IllegalArgumentException e) { assertFalse("Created BucketCache and expected it to succeed: " + expectSuccess[i] + ", but it actually was: " + !expectSuccess[i], expectSuccess[i]); } } }
private static List<BlockCache> getBlockCaches() throws IOException { Configuration conf = TEST_UTIL.getConfiguration(); List<BlockCache> blockcaches = new ArrayList<>(); // default CacheConfig.instantiateBlockCache(conf); blockcaches.add(new CacheConfig(conf).getBlockCache()); //set LruBlockCache.LRU_HARD_CAPACITY_LIMIT_FACTOR_CONFIG_NAME to 2.0f due to HBASE-16287 TEST_UTIL.getConfiguration().setFloat(LruBlockCache.LRU_HARD_CAPACITY_LIMIT_FACTOR_CONFIG_NAME, 2.0f); // memory BlockCache lru = new LruBlockCache(128 * 1024 * 1024, 64 * 1024, TEST_UTIL.getConfiguration()); blockcaches.add(lru); // bucket cache FileSystem.get(conf).mkdirs(TEST_UTIL.getDataTestDir()); int[] bucketSizes = { INDEX_BLOCK_SIZE, DATA_BLOCK_SIZE, BLOOM_BLOCK_SIZE, 64 * 1024, 128 * 1024 }; BlockCache bucketcache = new BucketCache("offheap", 128 * 1024 * 1024, 64 * 1024, bucketSizes, 5, 64 * 100, null); blockcaches.add(bucketcache); return blockcaches; }
@Test public void testGetPartitionSize() throws IOException { //Test default values validateGetPartitionSize(cache, BucketCache.DEFAULT_SINGLE_FACTOR, BucketCache.DEFAULT_MIN_FACTOR); Configuration conf = HBaseConfiguration.create(); conf.setFloat(BucketCache.MIN_FACTOR_CONFIG_NAME, 0.5f); conf.setFloat(BucketCache.SINGLE_FACTOR_CONFIG_NAME, 0.1f); conf.setFloat(BucketCache.MULTI_FACTOR_CONFIG_NAME, 0.7f); conf.setFloat(BucketCache.MEMORY_FACTOR_CONFIG_NAME, 0.2f); BucketCache cache = new BucketCache(ioEngineName, capacitySize, constructedBlockSize, constructedBlockSizes, writeThreads, writerQLen, persistencePath, 100, conf); validateGetPartitionSize(cache, 0.1f, 0.5f); validateGetPartitionSize(cache, 0.7f, 0.5f); validateGetPartitionSize(cache, 0.2f, 0.5f); }
TEST_UTIL.getTestFileSystem().mkdirs(testDir); BucketCache bucketCache = new BucketCache("file:" + testDir + "/bucket.cache", capacitySize, constructedBlockSize, constructedBlockSizes, writeThreads, writerQLen, testDir + "/bucket.persistence"); bucketCache = new BucketCache("file:" + testDir + "/bucket.cache", capacitySize, constructedBlockSize, constructedBlockSizes, writeThreads, writerQLen, testDir + "/bucket.persistence"); bucketCache = new BucketCache("file:" + testDir + "/bucket.cache", capacitySize, constructedBlockSize, smallBucketSizes, writeThreads, writerQLen, testDir + "/bucket.persistence");
@Test public void testValidBucketCacheConfigs() throws IOException { Configuration conf = HBaseConfiguration.create(); conf.setFloat(BucketCache.ACCEPT_FACTOR_CONFIG_NAME, 0.9f); conf.setFloat(BucketCache.MIN_FACTOR_CONFIG_NAME, 0.5f); conf.setFloat(BucketCache.EXTRA_FREE_FACTOR_CONFIG_NAME, 0.5f); conf.setFloat(BucketCache.SINGLE_FACTOR_CONFIG_NAME, 0.1f); conf.setFloat(BucketCache.MULTI_FACTOR_CONFIG_NAME, 0.7f); conf.setFloat(BucketCache.MEMORY_FACTOR_CONFIG_NAME, 0.2f); BucketCache cache = new BucketCache(ioEngineName, capacitySize, constructedBlockSize, constructedBlockSizes, writeThreads, writerQLen, persistencePath, 100, conf); assertEquals(BucketCache.ACCEPT_FACTOR_CONFIG_NAME + " failed to propagate.", 0.9f, cache.getAcceptableFactor(), 0); assertEquals(BucketCache.MIN_FACTOR_CONFIG_NAME + " failed to propagate.", 0.5f, cache.getMinFactor(), 0); assertEquals(BucketCache.EXTRA_FREE_FACTOR_CONFIG_NAME + " failed to propagate.", 0.5f, cache.getExtraFreeFactor(), 0); assertEquals(BucketCache.SINGLE_FACTOR_CONFIG_NAME + " failed to propagate.", 0.1f, cache.getSingleFactor(), 0); assertEquals(BucketCache.MULTI_FACTOR_CONFIG_NAME + " failed to propagate.", 0.7f, cache.getMultiFactor(), 0); assertEquals(BucketCache.MEMORY_FACTOR_CONFIG_NAME + " failed to propagate.", 0.2f, cache.getMemoryFactor(), 0); }