@Nullable public File getIfPresent(String key) { // Check if the file scheduled for async upload File staged = stagingCache.getIfPresent(key); if (staged != null && staged.exists()) { return staged; } return downloadCache.getIfPresent(key); }
@Nullable public File getIfPresent(String key) { // Check if the file scheduled for async upload File staged = stagingCache.getIfPresent(key); if (staged != null && staged.exists()) { return staged; } return downloadCache.getIfPresent(key); }
@Nullable public File getIfPresent(String key) { // Check if the file scheduled for async upload File staged = stagingCache.getIfPresent(key); if (staged != null && staged.exists()) { return staged; } return downloadCache.getIfPresent(key); }
public File get(String key) throws IOException { try { // Check if the file scheduled for async upload File staged = stagingCache.getIfPresent(key); if (staged != null && staged.exists()) { return staged; } // get from cache and download if not available return downloadCache.get(key); } catch (IOException e) { LOG.error("Error loading [{}] from cache", key); throw e; } }
public File get(String key) throws IOException { try { // Check if the file scheduled for async upload File staged = stagingCache.getIfPresent(key); if (staged != null && staged.exists()) { return staged; } // get from cache and download if not available return downloadCache.get(key); } catch (IOException e) { LOG.error("Error loading [{}] from cache", key); throw e; } }
public File get(String key) throws IOException { try { // Check if the file scheduled for async upload File staged = stagingCache.getIfPresent(key); if (staged != null && staged.exists()) { return staged; } // get from cache and download if not available return downloadCache.get(key); } catch (IOException e) { LOG.error("Error loading [{}] from cache", key); throw e; } }
@Override public void run() { retFuture.set(stagingCache.getIfPresent(ID_PREFIX + 1)); } });
@Override public Boolean call() { try { LOG.info("Starting put"); futures.addAll(put(folder)); LOG.info("Finished put"); File f = stagingCache.getIfPresent(ID_PREFIX + 0); LOG.info("Retrieved file {}, {}", f, f.exists()); return f != null && f.exists(); } catch (Exception e) { LOG.info("Exception in get", e); } return false; } });
/** * Invalidate after staging before upload. * @throws Exception */ @Test public void testInvalidate() throws Exception { // add load List<ListenableFuture<Integer>> futures = put(folder); // Check invalidate stagingCache.invalidate(ID_PREFIX + 0); File file = stagingCache.getIfPresent(ID_PREFIX + 0); assertNull(file); //start taskLatch.countDown(); callbackLatch.countDown(); waitFinish(futures); assertCacheStats(stagingCache, 0, 0, 1, 1); // Should not return anything file = stagingCache.getIfPresent(ID_PREFIX + 0); assertNull(file); }
/** * Concurrently retrieve after stage but before upload. * @throws Exception */ @Test public void testConcurrentGetDelete() throws Exception { ListeningExecutorService executorService = MoreExecutors.listeningDecorator(Executors.newFixedThreadPool(2)); closer.register(new ExecutorCloser(executorService)); // Add load List<ListenableFuture<Integer>> futures = put(folder); // Get a handle to the file and open stream File file = stagingCache.getIfPresent(ID_PREFIX + 0); final InputStream fStream = Files.asByteSource(file).openStream(); // task to copy the steam to a file simulating read from the stream File temp = folder.newFile(); CountDownLatch copyThreadLatch = new CountDownLatch(1); SettableFuture<File> future1 = copyStreamThread(executorService, fStream, temp, copyThreadLatch); //start taskLatch.countDown(); callbackLatch.countDown(); waitFinish(futures); // trying copying now copyThreadLatch.countDown(); future1.get(); assertTrue(Files.equal(temp, uploader.read(ID_PREFIX + 0))); }
private void assertUpgrade(File pendingUploadFile) throws IOException { assertNull(stagingCache.getIfPresent(ID_PREFIX + 1)); assertTrue(Files.equal(copyToFile(randomStream(1, 4 * 1024), folder.newFile()), uploader.read(ID_PREFIX + 1))); assertFalse(pendingUploadFile.exists()); }
/** * Retrieve without adding. * @throws Exception */ @Test public void testGetNoAdd() throws Exception { File ret = stagingCache.getIfPresent(ID_PREFIX + 0); // assert no file assertNull(ret); assertEquals(1, stagingCache.getStats().getLoadCount()); assertCacheStats(stagingCache, 0, 0, 0, 0); }
/** * Test build on start. * @throws Exception */ @Test public void testBuild() throws Exception { // Add load List<ListenableFuture<Integer>> futures = put(folder); // Close before uploading finished closer.close(); // Start again init(1); taskLatch.countDown(); callbackLatch.countDown(); afterExecuteLatch.await(); waitFinish(futures); assertNull(stagingCache.getIfPresent(ID_PREFIX + 0)); assertTrue(Files.equal(copyToFile(randomStream(0, 4 * 1024), folder.newFile()), uploader.read(ID_PREFIX + 0))); assertCacheStats(stagingCache, 0, 0, 1, 1); }
@Test public void testDefaultStatsProvider() throws Exception { stagingCache = UploadStagingCache.build(root, null, 1/*threads*/, 8 * 1024 /* bytes */, uploader, null/*cache*/, null, executor, null, 3000, 6000); // add load File f = copyToFile(randomStream(0, 4 * 1024), folder.newFile()); Optional<SettableFuture<Integer>> future = stagingCache.put(ID_PREFIX + 0, f); assertTrue(future.isPresent()); assertNotNull(stagingCache.getIfPresent(ID_PREFIX + 0)); assertCacheStats(stagingCache, 1, 4 * 1024, 1, 1); }
@Test public void testZeroCache() throws IOException { stagingCache = UploadStagingCache.build(root, null, 1/*threads*/, 0 /* bytes */, uploader, null/*cache*/, statsProvider, executor, null, 3000, 6000); closer.register(stagingCache); File f = copyToFile(randomStream(0, 4 * 1024), folder.newFile()); Optional<SettableFuture<Integer>> future = stagingCache.put(ID_PREFIX + 0, f); assertFalse(future.isPresent()); assertNull(stagingCache.getIfPresent(ID_PREFIX + 0)); assertEquals(0, Iterators.size(stagingCache.getAllIdentifiers())); assertEquals(0, stagingCache.getStats().getMaxTotalWeight()); }
/** * Test upgrade with build on start. * @throws Exception */ @Test public void testUpgrade() throws Exception { // Add load List<ListenableFuture<Integer>> futures = put(folder); // Close before uploading finished closer.close(); // Create pre-upgrade load File home = folder.newFolder(); File pendingUploadsFile = new File(home, DataStoreCacheUpgradeUtils.UPLOAD_MAP); createUpgradeLoad(home, pendingUploadsFile); // Start again init(2, new TestStagingUploader(folder.newFolder()), home); taskLatch.countDown(); callbackLatch.countDown(); afterExecuteLatch.await(); waitFinish(futures); assertNull(stagingCache.getIfPresent(ID_PREFIX + 0)); assertTrue(Files.equal(copyToFile(randomStream(0, 4 * 1024), folder.newFile()), uploader.read(ID_PREFIX + 0))); assertUpgrade(pendingUploadsFile); assertCacheStats(stagingCache, 0, 0, 2, 2); }
assertNull(stagingCache.getIfPresent(ID_PREFIX + 1)); assertNull(stagingCache.getIfPresent(ID_PREFIX + 2)); assertNull(stagingCache.getIfPresent(ID_PREFIX + 3));
File ret = stagingCache.getIfPresent(ID_PREFIX + 0); assertTrue(Files.equal(copyToFile(randomStream(0, 4 * 1024), folder.newFile()), ret)); ret = stagingCache.getIfPresent(ID_PREFIX + 0); assertNull(ret); assertTrue(Files.equal(copyToFile(randomStream(0, 4 * 1024), folder.newFile()),