@Override protected void readInternal(byte[] b, int off, int len) throws IOException { long position = getFilePointer(); while (len > 0) { int length = fetchBlock(position, b, off, len); position += length; len -= length; off += length; } }
@Override public long size() { return _blockCache.getSize(); } }
public boolean fetch(BlockCacheKey blockCacheKey, byte[] buffer) { checkLength(buffer); return fetch(blockCacheKey, buffer, 0, 0, _blockSize); }
public boolean store(BlockCacheKey blockCacheKey, byte[] data) { checkLength(data); BlockCacheLocation location = _cache.get(blockCacheKey); boolean newLocation = false; if (location == null) { newLocation = true; location = new BlockCacheLocation(); if (!findEmptyLocation(location)) { return false; } } if (location.isRemoved()) { return false; } int slabId = location.getSlabId(); int offset = location.getBlock() * _blockSize; ByteBuffer slab = getSlab(slabId); slab.position(offset); slab.put(data, 0, _blockSize); if (newLocation) { releaseLocation(_cache.put(blockCacheKey.clone(), location)); _metrics.blockCacheSize.incrementAndGet(); } return true; }
@Override public void update(String name, long blockId, byte[] buffer) { Integer file = _names.get(name); if (file == null) { file = _counter.incrementAndGet(); _names.put(name, file); } BlockCacheKey blockCacheKey = new BlockCacheKey(); blockCacheKey.setBlock(blockId); blockCacheKey.setFile(file); _blockCache.store(blockCacheKey, buffer); }
@Override public boolean fetch(String name, long blockId, int blockOffset, byte[] b, int off, int lengthToReadInBlock) { Integer file = _names.get(name); if (file == null) { return false; } BlockCacheKey blockCacheKey = new BlockCacheKey(); blockCacheKey.setBlock(blockId); blockCacheKey.setFile(file); boolean fetch = _blockCache.fetch(blockCacheKey, b, blockOffset, off, lengthToReadInBlock); if (fetch) { _blurMetrics.blockCacheHit.incrementAndGet(); } else { _blurMetrics.blockCacheMiss.incrementAndGet(); } return fetch; }
public boolean fetch(BlockCacheKey blockCacheKey, byte[] buffer, int blockOffset, int off, int length) { BlockCacheLocation location = _cache.get(blockCacheKey); if (location == null) { return false; } if (location.isRemoved()) { return false; } int slabId = location.getSlabId(); int offset = location.getBlock() * _blockSize; location.touch(); ByteBuffer slab = getSlab(slabId); slab.position(offset + blockOffset); slab.get(buffer, off, length); return true; }
private int fetchBlock(long position, byte[] b, int off, int len) throws IOException { // read whole block into cache and then provide needed data long blockId = getBlock(position); int blockOffset = (int) getPosition(position); int lengthToReadInBlock = Math.min(len, _blockSize - blockOffset); if (checkCache(blockId, blockOffset, b, off, lengthToReadInBlock)) { return lengthToReadInBlock; } else { readIntoCacheAndResult(blockId, blockOffset, b, off, lengthToReadInBlock); } return lengthToReadInBlock; }
private void releaseLocation(BlockCacheLocation location) { if (location == null) { return; } int slabId = location.getSlabId(); int block = location.getBlock(); location.setRemoved(true); _locks[slabId].clear(block); _lockCounters[slabId].decrementAndGet(); _metrics.blockCacheEviction.incrementAndGet(); _metrics.blockCacheSize.decrementAndGet(); }
@Override public void close() throws IOException { String[] files = listAll(); for (String file : files) { _cache.delete(getFileCacheName(file)); } _directory.close(); }
public IndexInput openInput(String name, int bufferSize) throws IOException { final IndexInput source = _directory.openInput(name, _blockSize); if (_blockCacheFileTypes == null || isCachableFile(name)) { return new CachedIndexInput(source, _blockSize, name, getFileCacheName(name), _cache, bufferSize); } return source; }
@Before public void setUp() throws IOException { file = new File("./tmp"); rm(file); file.mkdirs(); FSDirectory dir = FSDirectory.open(new File(file, "base")); directory = new BlockDirectory("test", dir, getBasicCache()); seed = new Random().nextLong(); random = new Random(seed); }
public void deleteFile(String name) throws IOException { _cache.delete(getFileCacheName(name)); _directory.deleteFile(name); }
private void readIntoCacheAndResult(long blockId, int blockOffset, byte[] b, int off, int lengthToReadInBlock) throws IOException { long position = getRealPosition(blockId, 0); int length = (int) Math.min(_blockSize, _fileLength - position); _source.seek(position); byte[] buf = BufferStore.takeBuffer(_blockSize); _source.readBytes(buf, 0, length); System.arraycopy(buf, blockOffset, b, off, lengthToReadInBlock); _cache.update(_cacheName, blockId, buf); BufferStore.putBuffer(buf); }
private boolean checkCache(long blockId, int blockOffset, byte[] b, int off, int lengthToReadInBlock) { return _cache.fetch(_cacheName, blockId, blockOffset, b, off, lengthToReadInBlock); }
@Override public IndexInput openInput(final String name) throws IOException { return openInput(name, _blockSize); }
private String getFileCacheName(String name) throws IOException { return _dirName + "/" + name + ":" + fileModified(name); }
public BlockDirectory(String dirName, Directory directory, Cache cache, Set<String> blockCacheFileTypes) throws IOException { _dirName = dirName; _directory = directory; _blockSize = BLOCK_SIZE; _cache = cache; if (blockCacheFileTypes == null || blockCacheFileTypes.isEmpty()) { _blockCacheFileTypes = null; } else { _blockCacheFileTypes = blockCacheFileTypes; } setLockFactory(directory.getLockFactory()); }
@Override public void update(String name, long blockId, byte[] buffer) { map.put(name + blockId, copy(buffer)); }
public static void rm(File file) { if (!file.exists()) { return; } if (file.isDirectory()) { for (File f : file.listFiles()) { rm(f); } } file.delete(); }