CompositeDataSupport toCompositeData() { Object[] values = new Object[]{ stats.getName(), stats.getRequestCount(), stats.getHitCount(), new BigDecimal(stats.getHitRate(),new MathContext(2)), stats.getMissCount(), new BigDecimal(stats.getMissRate(), new MathContext(2)), stats.getLoadCount(), stats.getLoadSuccessCount(), stats.getLoadExceptionCount(), timeInWords(stats.getTotalLoadTime()), TimeUnit.NANOSECONDS.toMillis((long) stats.getAverageLoadPenalty()) + "ms", stats.getEvictionCount(), stats.getElementCount(), humanReadableByteCount(stats.estimateCurrentWeight()), humanReadableByteCount(stats.estimateCurrentMemoryWeight()), humanReadableByteCount(stats.getMaxTotalWeight()), }; try { return new CompositeDataSupport(TYPE, FIELD_NAMES, values); } catch (OpenDataException e) { throw new IllegalStateException(e); } } }
private static void assertCacheStats(DataStoreCacheStatsMBean cache, long elems, long weight, long hits, long count) { assertEquals("elements don't match", elems, cache.getElementCount()); assertEquals("weight doesn't match", weight, cache.estimateCurrentWeight()); assertEquals("hits count don't match", hits, cache.getHitCount()); assertEquals("requests count don't match", count, cache.getRequestCount()); }
private static void assertCacheStats(FileCache cache, long elems, long weight, long loads, long loadSuccesses) { assertEquals(elems, cache.getStats().getElementCount()); assertEquals(weight, cache.getStats().estimateCurrentWeight()); assertEquals(loads, cache.getStats().getLoadCount()); assertEquals(loadSuccesses, cache.getStats().getLoadSuccessCount()); } }
assertEquals(2, cache.getStagingCacheStats().getLoadCount()); assertEquals(0, cache.getStagingCacheStats().getLoadSuccessCount()); assertEquals(2, cache.getCacheStats().getLoadCount()); assertEquals(2, cache.getCacheStats().getLoadSuccessCount());
/** * Invalidate cache entry. * @throws Exception */ @Test public void invalidate() throws Exception { LOG.info("Starting invalidate"); File f = copyToFile(randomStream(0, 4 * 1024), folder.newFile()); loader.write(ID_PREFIX + 0, f); // present after loading File cached = cache.get(ID_PREFIX + 0); assertNotNull(cached); assertTrue(Files.equal(f, cached)); cache.invalidate(ID_PREFIX + 0); // Not present now cached = cache.getIfPresent(ID_PREFIX + 0); assertNull(cached); assertCacheStats(cache.getStagingCacheStats(), 0, 0, 0, 0); assertEquals(2, cache.getStagingCacheStats().getLoadCount()); assertEquals(0, cache.getStagingCacheStats().getLoadSuccessCount()); assertCacheStats(cache.getCacheStats(), 0, 0, 0, 3); assertEquals(1, cache.getCacheStats().getLoadCount()); assertEquals(1, cache.getCacheStats().getLoadSuccessCount()); /** Check eviction count */ assertEquals(0, cache.getCacheStats().getEvictionCount()); LOG.info("Finished invalidate"); }
@Override public void close() throws IOException { try { while (store.getStats().get(1).getElementCount() > 0) { Thread.sleep(100); } } catch (InterruptedException e) { throw new IOException(e); } finally { try { store.close(); } catch (DataStoreException e) { throw new IOException(e); } } } };
@Test public void zeroCache() throws IOException { LOG.info("Starting zeroCache"); cache = new CompositeDataStoreCache(root.getAbsolutePath(), null, 0 /* bytes */, 10, 1/*threads*/, loader, uploader, statsProvider, executor, scheduledExecutor, fileCacheExecutor, 3000, 6000); closer.register(cache); File f = copyToFile(randomStream(0, 4 * 1024), folder.newFile()); boolean accepted = cache.stage(ID_PREFIX + 0, f); assertFalse(accepted); assertNull(cache.getIfPresent(ID_PREFIX + 0)); assertNull(cache.get(ID_PREFIX + 0)); assertEquals(0, cache.getStagingCache().getStats().getMaxTotalWeight()); assertEquals(0, cache.getStagingCacheStats().getMaxTotalWeight()); assertEquals(0,cache.getDownloadCache().getStats().getMaxTotalWeight()); assertEquals(0,cache.getCacheStats().getMaxTotalWeight()); cache.invalidate(ID_PREFIX + 0); cache.close(); LOG.info("Finished zeroCache"); }
/** * Error in putting file to stage. * @throws Exception */ @Test public void testPutMoveFileError() throws Exception { File empty = new File(folder.getRoot(), String.valueOf(System.currentTimeMillis())); assertFalse(empty.exists()); Optional<SettableFuture<Integer>> future = stagingCache.put(ID_PREFIX + 0, empty); // assert no file assertFalse(future.isPresent()); assertEquals(1, stagingCache.getStats().getMissCount()); assertCacheStats(stagingCache, 0, 0, 0, 1); }
/** * Put and retrieve different files concurrently. * @throws Exception */ @Test public void testGetAddDifferent() throws Exception { //add load List<ListenableFuture<Integer>> futures = put(folder); // Create an async retrieve task final SettableFuture<File> retFuture = SettableFuture.create(); Thread t = new Thread(new Runnable() { @Override public void run() { retFuture.set(stagingCache.getIfPresent(ID_PREFIX + 1)); } }); //start taskLatch.countDown(); callbackLatch.countDown(); t.start(); //assert no file retrieve assertNull(retFuture.get()); assertEquals(1, stagingCache.getStats().getLoadCount()); assertFuture(futures, 0); assertCacheStats(stagingCache, 0, 0, 1, 1); }
/** * Load and get from cache. * @throws Exception */ @Test public void add() throws Exception { LOG.info("Started add"); File f = createFile(0, loader, cache, folder); assertCache(0, cache, f); assertCacheStats(cache, 1, 4 * 1024, 1, 1); assertEquals("Memory weight different", getWeight(ID_PREFIX + 0, cache.getIfPresent(ID_PREFIX + 0)), cache.getStats().estimateCurrentMemoryWeight()); LOG.info("Finished add"); }
private static void assertCacheStats(UploadStagingCache cache, long elems, long weight, long hits, long count) { assertEquals(elems, cache.getStats().getElementCount()); assertEquals(weight, cache.getStats().estimateCurrentWeight()); assertEquals(hits, cache.getStats().getHitCount()); assertEquals(count, cache.getStats().getRequestCount()); } }
/** * Load and get from the download cache. * @throws Exception */ @Test public void getLoad() throws Exception { LOG.info("Starting getLoad"); File f = copyToFile(randomStream(0, 4 * 1024), folder.newFile()); loader.write(ID_PREFIX + 0, f); // Not present yet File cached = cache.getIfPresent(ID_PREFIX + 0); assertNull(cached); // present after loading cached = cache.get(ID_PREFIX + 0); assertNotNull(cached); assertTrue(Files.equal(f, cached)); assertCacheStats(cache.getStagingCacheStats(), 0, 0, 0, 0); assertEquals(2, cache.getStagingCacheStats().getLoadCount()); assertEquals(0, cache.getStagingCacheStats().getLoadSuccessCount()); assertCacheStats(cache.getCacheStats(), 1, 4 * 1024, 0, 3); assertEquals(1, cache.getCacheStats().getLoadCount()); assertEquals(1, cache.getCacheStats().getLoadSuccessCount()); LOG.info("Finished getLoad"); }
@Override public void close() throws IOException { try { while (store.getStats().get(1).getElementCount() > 0) { Thread.sleep(100); } } catch (InterruptedException e) { throw new IOException(e); } finally { try { store.close(); } catch (DataStoreException e) { throw new IOException(e); } } } };
@Test public void zeroCache() throws Exception { LOG.info("Started zeroCache"); cache = FileCache.build(0/* KB */, root, loader, null); closer.register(cache); File f = createFile(0, loader, cache, folder); cache.put(ID_PREFIX + 0, f); assertNull(cache.getIfPresent(ID_PREFIX + 0)); assertNull(cache.get(ID_PREFIX + 0)); assertEquals(0, cache.getStats().getMaxTotalWeight()); cache.invalidate(ID_PREFIX + 0); assertFalse(cache.containsKey(ID_PREFIX + 0)); cache.close(); LOG.info("Finished zeroCache"); }
/** * Tests {@link FileCache#getIfPresent(Object)} when no cache. */ @Test public void getIfPresentObjectNoCache() { LOG.info("Started getIfPresentObjectNoCache"); File file = cache.getIfPresent((Object) (ID_PREFIX + 0)); assertNull(file); assertCacheStats(cache, 0, 0, 0, 0); assertEquals(1, cache.getStats().getMissCount()); LOG.info("Finished getIfPresentObjectNoCache"); }
/** * Retrieve without adding. * @throws Exception */ @Test public void testGetNoAdd() throws Exception { File ret = stagingCache.getIfPresent(ID_PREFIX + 0); // assert no file assertNull(ret); assertEquals(1, stagingCache.getStats().getLoadCount()); assertCacheStats(stagingCache, 0, 0, 0, 0); }
CompositeDataSupport toCompositeData() { Object[] values = new Object[]{ stats.getName(), stats.getRequestCount(), stats.getHitCount(), new BigDecimal(stats.getHitRate(),new MathContext(2)), stats.getMissCount(), new BigDecimal(stats.getMissRate(), new MathContext(2)), stats.getLoadCount(), stats.getLoadSuccessCount(), stats.getLoadExceptionCount(), timeInWords(stats.getTotalLoadTime()), TimeUnit.NANOSECONDS.toMillis((long) stats.getAverageLoadPenalty()) + "ms", stats.getEvictionCount(), stats.getElementCount(), humanReadableByteCount(stats.estimateCurrentWeight()), humanReadableByteCount(stats.estimateCurrentMemoryWeight()), humanReadableByteCount(stats.getMaxTotalWeight()), }; try { return new CompositeDataSupport(TYPE, FIELD_NAMES, values); } catch (OpenDataException e) { throw new IllegalStateException(e); } } }
assertEquals(2, cache.getStagingCacheStats().getLoadCount()); assertEquals(1, cache.getStagingCacheStats().getLoadSuccessCount()); assertEquals(1, cache.getCacheStats().getLoadCount()); assertEquals(1, cache.getCacheStats().getLoadSuccessCount());
/** * evict implicitly. * @throws Exception */ @Test public void evictImplicit() throws Exception { LOG.info("Started evictImplicit"); for (int i = 0; i < 15; i++) { File f = createFile(i, loader, cache, folder); assertCache(i, cache, f); } File f = createFile(30, loader, cache, folder); assertCache(30, cache, f); // One of the entries should have been evicted assertTrue(cache.getStats().getElementCount() == 15); assertCacheStats(cache, 15, 60 * 1024, 16, 16); LOG.info("Finished evictImplicit"); }
@Test public void testZeroCache() throws IOException { stagingCache = UploadStagingCache.build(root, null, 1/*threads*/, 0 /* bytes */, uploader, null/*cache*/, statsProvider, executor, null, 3000, 6000); closer.register(stagingCache); File f = copyToFile(randomStream(0, 4 * 1024), folder.newFile()); Optional<SettableFuture<Integer>> future = stagingCache.put(ID_PREFIX + 0, f); assertFalse(future.isPresent()); assertNull(stagingCache.getIfPresent(ID_PREFIX + 0)); assertEquals(0, Iterators.size(stagingCache.getAllIdentifiers())); assertEquals(0, stagingCache.getStats().getMaxTotalWeight()); }