public CompositeDataStoreCache(String path, File home, long size, int uploadSplitPercentage, int uploadThreads, CacheLoader<String, InputStream> loader, final StagingUploader uploader, StatisticsProvider statsProvider, ListeningExecutorService listeningExecutor, ScheduledExecutorService scheduledExecutor /* purge scheduled executor */, ExecutorService executor /* File cache executor */, int purgeInterval /* async purge interval secs */, int stagingRetryInterval /* async retry interval secs */) { checkArgument(uploadSplitPercentage >= 0 && uploadSplitPercentage < 100, "Upload percentage should be between 0 and 100"); this.directory = new File(path); long uploadSize = (size * uploadSplitPercentage) / 100; this.stagingCache = UploadStagingCache .build(directory, home, uploadThreads, uploadSize, uploader, null, statsProvider, listeningExecutor, scheduledExecutor, purgeInterval, stagingRetryInterval); this.downloadCache = FileCache.build((size - uploadSize), directory, loader, executor); stagingCache.setDownloadCache(downloadCache); }
public CompositeDataStoreCache(String path, File home, long size, int uploadSplitPercentage, int uploadThreads, CacheLoader<String, InputStream> loader, final StagingUploader uploader, StatisticsProvider statsProvider, ListeningExecutorService listeningExecutor, ScheduledExecutorService scheduledExecutor /* purge scheduled executor */, ExecutorService executor /* File cache executor */, int purgeInterval /* async purge interval secs */, int stagingRetryInterval /* async retry interval secs */) { checkArgument(uploadSplitPercentage >= 0 && uploadSplitPercentage < 100, "Upload percentage should be between 0 and 100"); this.directory = new File(path); long uploadSize = (size * uploadSplitPercentage) / 100; this.stagingCache = UploadStagingCache .build(directory, home, uploadThreads, uploadSize, uploader, null, statsProvider, listeningExecutor, scheduledExecutor, purgeInterval, stagingRetryInterval); this.downloadCache = FileCache.build((size - uploadSize), directory, loader, executor); stagingCache.setDownloadCache(downloadCache); }
public CompositeDataStoreCache(String path, File home, long size, int uploadSplitPercentage, int uploadThreads, CacheLoader<String, InputStream> loader, final StagingUploader uploader, StatisticsProvider statsProvider, ListeningExecutorService listeningExecutor, ScheduledExecutorService scheduledExecutor /* purge scheduled executor */, ExecutorService executor /* File cache executor */, int purgeInterval /* async purge interval secs */, int stagingRetryInterval /* async retry interval secs */) { checkArgument(uploadSplitPercentage >= 0 && uploadSplitPercentage < 100, "Upload percentage should be between 0 and 100"); this.directory = new File(path); long uploadSize = (size * uploadSplitPercentage) / 100; this.stagingCache = UploadStagingCache .build(directory, home, uploadThreads, uploadSize, uploader, null, statsProvider, listeningExecutor, scheduledExecutor, purgeInterval, stagingRetryInterval); this.downloadCache = FileCache.build((size - uploadSize), directory, loader, executor); stagingCache.setDownloadCache(downloadCache); }
@Before public void setup() throws Exception { LOG.info("Started setup"); root = folder.newFolder(); closer = Closer.create(); loader = new TestCacheLoader<String, InputStream>(folder.newFolder()); CountDownLatch beforeLatch = new CountDownLatch(1); CountDownLatch afterLatch = new CountDownLatch(1); afterExecuteLatch = new CountDownLatch(1); TestExecutor executor = new TestExecutor(1, beforeLatch, afterLatch, afterExecuteLatch); beforeLatch.countDown(); afterLatch.countDown(); cache = FileCache.build(4 * 1024/* KB */, root, loader, executor); Futures.successfulAsList((Iterable<? extends ListenableFuture<?>>) executor.futures).get(); closer.register(cache); LOG.info("Finished setup"); }
/** * Trigger build cache on start. * @throws Exception */ @Test public void rebuild() throws Exception { LOG.info("Started rebuild"); afterExecuteLatch.await(); LOG.info("Cache built"); File f = createFile(0, loader, cache, folder); assertCache(0, cache, f); cache.close(); CountDownLatch beforeLatch = new CountDownLatch(1); CountDownLatch afterLatch = new CountDownLatch(1); afterExecuteLatch = new CountDownLatch(1); TestExecutor executor = new TestExecutor(1, beforeLatch, afterLatch, afterExecuteLatch); beforeLatch.countDown(); afterLatch.countDown(); cache = FileCache.build(4 * 1024/* bytes */, root, loader, executor); closer.register(cache); afterExecuteLatch.await(); Futures.successfulAsList((Iterable<? extends ListenableFuture<?>>) executor.futures).get(); LOG.info("Cache rebuilt"); assertCacheIfPresent(0, cache, f); assertCacheStats(cache, 1, 4 * 1024, 0, 0); LOG.info("Finished rebuild"); }
@Test public void loadError() throws Exception { LOG.info("Started loadError"); loader = new TestErrorCacheLoader<String, InputStream>(folder.newFolder(), 8192); cache = FileCache.build(12 * 1024/* KB */, root, loader, null); closer.register(cache); createFile(0, loader, cache, folder, 12 * 1024); try { cache.get(ID_PREFIX + 0); } catch (IOException e) { } expectedEx.expect(IOException.class); cache.get(ID_PREFIX + 0); LOG.info("Finished loadError"); }
/** * Trigger upgrade cache on start. * @throws Exception */ @Test public void upgrade() throws Exception { LOG.info("Started upgrade"); afterExecuteLatch.await(); File f = createFile(0, loader, cache, folder); assertCache(0, cache, f); cache.close(); copyToFile(randomStream(1, 4 * 1024), getFile(ID_PREFIX + 1, root)); CountDownLatch beforeLatch = new CountDownLatch(1); CountDownLatch afterLatch = new CountDownLatch(1); afterExecuteLatch = new CountDownLatch(1); TestExecutor executor = new TestExecutor(1, beforeLatch, afterLatch, afterExecuteLatch); beforeLatch.countDown(); afterLatch.countDown(); cache = FileCache.build(4 * 1024/* bytes */, root, loader, executor); closer.register(cache); afterExecuteLatch.await(); Futures.successfulAsList((Iterable<? extends ListenableFuture<?>>) executor.futures).get(); LOG.info("Cache rebuilt"); assertCacheIfPresent(0, cache, f); assertCacheIfPresent(1, cache, copyToFile(randomStream(1, 4 * 1024), folder.newFile())); assertFalse(getFile(ID_PREFIX + 1, root).exists()); assertCacheStats(cache, 2, 8 * 1024, 0, 0); LOG.info("Finished upgrade"); }
@Test public void zeroCache() throws Exception { LOG.info("Started zeroCache"); cache = FileCache.build(0/* KB */, root, loader, null); closer.register(cache); File f = createFile(0, loader, cache, folder); cache.put(ID_PREFIX + 0, f); assertNull(cache.getIfPresent(ID_PREFIX + 0)); assertNull(cache.get(ID_PREFIX + 0)); assertEquals(0, cache.getStats().getMaxTotalWeight()); cache.invalidate(ID_PREFIX + 0); assertFalse(cache.containsKey(ID_PREFIX + 0)); cache.close(); LOG.info("Finished zeroCache"); }