BinaryStore diskstore = new BerkeleyDBBinaryStore("mydb", ...); SimpleSerializationMap mapDisk = SimpleSerializationMap(diskstore); LocalCache mapRAM = new LocalCache(100 * 1024 * 1024); // 100MB in RAM OverflowMap cache = new OverflowMap(mapRAM, mapDisk);
@Override public void close() throws DataStoreException { cache.close(); backend.close(); downloadExecService.shutdown(); }
@Override public Long get(Object key) { if( isInPurgeMode()) { LOG.debug("cache is purge mode: get is no-op"); return null; } synchronized (this) { return super.get(key); } }
/** * This method tries purging of local cache. It checks if local cache * has exceeded the defined limit then it triggers purge cache job in a * seperate thread. */ synchronized void tryPurge() { if (!isInPurgeMode() && cache.currentSizeInBytes > cache.cachePurgeTrigSize) { setPurgeMode(true); LOG.info( "cache.entries = [{}], currentSizeInBytes=[{}] exceeds cachePurgeTrigSize=[{}]", new Object[] { cache.size(), cache.currentSizeInBytes, cache.cachePurgeTrigSize }); new Thread(new PurgeJob()).start(); } else { LOG.debug( "currentSizeInBytes=[{}],cachePurgeTrigSize=[{}], isInPurgeMode =[{}]", new Object[] { cache.currentSizeInBytes, cache.cachePurgeTrigSize, isInPurgeMode() }); } }
public File getFileIfStored(String fileName) throws IOException { fileName = fileName.replace("\\", "/"); File f = getFile(fileName); long diff = (System.currentTimeMillis() - cacheMissDuration.get()) / 1000; // logged at 5 minute interval minimum if (diff > 5 * 60) { LOG.info("local cache misses [{}] in [{}] sec", new Object[] { cacheMissCounter.getAndSet(0), diff }); cacheMissDuration.set(System.currentTimeMillis()); } // return file in purge mode = true and file present in asyncUploadCache // as asyncUploadCache's files will be not be deleted in cache purge. if (!f.exists() || (isInPurgeMode() && !asyncUploadCache.hasEntry(fileName, false))) { LOG.debug( "getFileIfStored returned: purgeMode=[{}], file=[{}] exists=[{}]", new Object[] { isInPurgeMode(), f.getAbsolutePath(), f.exists() }); cacheMissCounter.incrementAndGet(); return null; } else { // touch entry in LRU caches f.setLastModified(System.currentTimeMillis()); cache.get(fileName); return f; } }
pendingFiles.init(tempDirPath, cacheDirPath, 100); pendingFiles.reset(); LocalCache cache = new LocalCache(cacheDirPath, tempDirPath, 400, 0.95, 0.70, pendingFiles); Random random = new Random(12345); cache.store("a1", new ByteArrayInputStream(byteMap.get("a1"))); cache.store("a2", new ByteArrayInputStream(byteMap.get("a2"))); cache.store("a3", new ByteArrayInputStream(byteMap.get("a3"))); InputStream result = cache.getIfStored("a1"); assertEquals(new ByteArrayInputStream(byteMap.get("a1")), result); IOUtils.closeQuietly(result); result = cache.getIfStored("a2"); assertEquals(new ByteArrayInputStream(byteMap.get("a2")), result); IOUtils.closeQuietly(result); result = cache.getIfStored("a3"); assertEquals(new ByteArrayInputStream(byteMap.get("a3")), result); IOUtils.closeQuietly(result); cache.store("a4", new ByteArrayInputStream(byteMap.get("a4"))); do { Thread.sleep(1000); } while (cache.isInPurgeMode()); result = cache.getIfStored("a1"); assertNull("a1 should be null", result); IOUtils.closeQuietly(result);
throws IOException { fileName = fileName.replace("\\", "/"); File f = getFile(fileName); long length = 0; if (!f.exists() || isInPurgeMode()) { OutputStream out = null; File transFile = null; if (canAdmitFile(length) && (f.getParentFile().exists() || f.getParentFile().mkdirs()) && transFile.renameTo(f) && f.exists()) { cache.put(fileName, f.length()); tryPurge(); return new LazyFileInputStream(f);
pendingFiles.init(tempDirPath, cacheDirPath, 100); pendingFiles.reset(); LocalCache cache = new LocalCache(cacheDirPath, tempDirPath, 400, 0.95, 0.70, pendingFiles); Random random = new Random(12345); byteMap.put("a3", data); cache.store("a1", new ByteArrayInputStream(byteMap.get("a1"))); cache.store("a2", new ByteArrayInputStream(byteMap.get("a2"))); cache.store("a3", new ByteArrayInputStream(byteMap.get("a3"))); InputStream result = cache.getIfStored("a1"); assertEquals(new ByteArrayInputStream(byteMap.get("a1")), result); IOUtils.closeQuietly(result); result = cache.getIfStored("a2"); assertEquals(new ByteArrayInputStream(byteMap.get("a2")), result); IOUtils.closeQuietly(result); result = cache.getIfStored("a3"); assertEquals(new ByteArrayInputStream(byteMap.get("a3")), result); IOUtils.closeQuietly(result);
pendingFiles.init(tempDirPath, cacheDirPath, 100); pendingFiles.reset(); LocalCache cache = new LocalCache(cacheDirPath, tempDirPath, 10000000, 0.95, 0.70, pendingFiles); Random random = new Random(12345); String key = "a" + i; byteMap.put(key, data); cache.store(key, new ByteArrayInputStream(byteMap.get(key))); cache.close(); cache = new LocalCache(cacheDirPath, tempDirPath, 10000000, 0.95, 0.70, pendingFiles); executor.execute(new StoreWorker(cache, byteMap));
/** * Get stream of record from {@link LocalCache}. If record is not available * in {@link LocalCache}, this method fetches record from {@link Backend} * and stores it to {@link LocalCache}. Stream is then returned from cached * record. */ InputStream getStream(DataIdentifier identifier) throws DataStoreException { InputStream in = null; try { String fileName = getFileName(identifier); InputStream cached = cache.getIfStored(fileName); if (cached != null) { return cached; } in = backend.read(identifier); return cache.store(fileName, in); } catch (IOException e) { throw new DataStoreException("IO Exception: " + identifier, e); } finally { IOUtils.closeQuietly(in); } }
boolean tryForAsyncUpload) throws IOException { fileName = fileName.replace("\\", "/"); File dest = getFile(fileName); File parent = dest.getParentFile(); AsyncUploadCacheResult result = new AsyncUploadCacheResult(); if ((destExists = dest.exists()) || (src.exists() && !dest.exists() && !src.equals(dest) && canAdmitFile(src.length()) && (parent.exists() || parent.mkdirs()) && (src.renameTo(dest)))) { if (destExists) { fileName, destExists); tryPurge(); return result;
/** * Store an item along with file in cache. Cache size is increased by * {@link File#length()} If file already exists in cache, * {@link File#setLastModified(long)} is updated with current time. * * @param fileName the key of cache. * @param src file to be added to cache. */ public File store(String fileName, final File src) { try { return store(fileName, src, false).getFile(); } catch (IOException ioe) { LOG.warn("Exception in addding file [" + fileName + "] to local cache.", ioe); } return null; }
public void deleteFromCache(DataIdentifier identifier) throws DataStoreException { try { // order is important here recLenCache.remove(identifier); String fileName = getFileName(identifier); asyncWriteCache.delete(fileName); cache.delete(fileName); } catch (IOException ioe) { throw new DataStoreException(ioe); } }
/** * Return the inputstream from from cache, or null if not in the cache. * * @param fileName name of file. * @return stream or null. */ public InputStream getIfStored(String fileName) throws IOException { File file = getFileIfStored(fileName); return file == null ? null : new LazyFileInputStream(file); }
/** * Close the cache. Cache maintain set of files which it was not able to * delete successfully. This method will an attempt to delete all * unsuccessful delete files. */ public void close() { LOG.debug("close"); deleteOldFiles(); }
/** * This method tries to delete a file. If it is not able to delete file due * to any reason, it add it toBeDeleted list. * * @param fileName name of the file which will be deleted. * @return true if this method deletes file successfuly else return false. */ boolean tryDelete(final String fileName) { LOG.debug("try deleting file [{}]", fileName); File f = getFile(fileName); if (f.exists() && f.delete()) { LOG.info("File [{}] deleted successfully", f.getAbsolutePath()); toBeDeleted.remove(fileName); while (true) { f = f.getParentFile(); if (f.equals(directory) || f.list().length > 0) { break; } // delete empty parent folders (except the main directory) f.delete(); } return true; } else if (f.exists()) { LOG.info("not able to delete file [{}]", f.getAbsolutePath()); toBeDeleted.add(fileName); return false; } return true; }
throws IOException { fileName = fileName.replace("\\", "/"); File f = getFile(fileName); long length = 0; if (!f.exists() || isInPurgeMode()) { OutputStream out = null; File transFile = null; if (canAdmitFile(length) && (f.getParentFile().exists() || f.getParentFile().mkdirs()) && transFile.renameTo(f) && f.exists()) { cache.put(fileName, f.length()); tryPurge(); return new LazyFileInputStream(f);
pendingFiles.init(tempDirPath, cacheDirPath, 100); pendingFiles.reset(); LocalCache cache = new LocalCache(cacheDirPath, tempDirPath, 400, 0.95, 0.70, pendingFiles); Random random = new Random(12345); fos.write(byteMap.get("a1")); fos.close(); AsyncUploadCacheResult result = cache.store("a1", f, true); assertTrue("should be able to add to pending upload", result.canAsyncUpload()); fos.write(byteMap.get("a2")); fos.close(); result = cache.store("a2", f, true); assertTrue("should be able to add to pending upload", result.canAsyncUpload()); fos.write(byteMap.get("a3")); fos.close(); result = cache.store("a3", f, true); assertTrue("should be able to add to pending upload", result.canAsyncUpload()); InputStream inp = cache.getIfStored("a1"); assertEquals(new ByteArrayInputStream(byteMap.get("a1")), inp); IOUtils.closeQuietly(inp); inp = cache.getIfStored("a2"); assertEquals(new ByteArrayInputStream(byteMap.get("a2")), inp);
/** * Get stream of record from {@link LocalCache}. If record is not available * in {@link LocalCache}, this method fetches record from {@link Backend} * and stores it to {@link LocalCache}. Stream is then returned from cached * record. */ InputStream getStream(DataIdentifier identifier) throws DataStoreException { InputStream in = null; try { String fileName = getFileName(identifier); InputStream cached = cache.getIfStored(fileName); if (cached != null) { return cached; } in = backend.read(identifier); return cache.store(fileName, in); } catch (IOException e) { throw new DataStoreException("IO Exception: " + identifier, e); } finally { IOUtils.closeQuietly(in); } }