/** * Create a {@link ShortCircuitCache} object from a {@link Configuration} */ public static ShortCircuitCache fromConf(Configuration conf) { return new ShortCircuitCache( conf.getInt(DFSConfigKeys.DFS_CLIENT_READ_SHORTCIRCUIT_STREAMS_CACHE_SIZE_KEY, DFSConfigKeys.DFS_CLIENT_READ_SHORTCIRCUIT_STREAMS_CACHE_SIZE_DEFAULT), conf.getLong(DFSConfigKeys.DFS_CLIENT_READ_SHORTCIRCUIT_STREAMS_CACHE_EXPIRY_MS_KEY, DFSConfigKeys.DFS_CLIENT_READ_SHORTCIRCUIT_STREAMS_CACHE_EXPIRY_MS_DEFAULT), conf.getInt(DFSConfigKeys.DFS_CLIENT_MMAP_CACHE_SIZE, DFSConfigKeys.DFS_CLIENT_MMAP_CACHE_SIZE_DEFAULT), conf.getLong(DFSConfigKeys.DFS_CLIENT_MMAP_CACHE_TIMEOUT_MS, DFSConfigKeys.DFS_CLIENT_MMAP_CACHE_TIMEOUT_MS_DEFAULT), conf.getLong(DFSConfigKeys.DFS_CLIENT_MMAP_RETRY_TIMEOUT_MS, DFSConfigKeys.DFS_CLIENT_MMAP_RETRY_TIMEOUT_MS_DEFAULT), conf.getLong(DFSConfigKeys.DFS_CLIENT_SHORT_CIRCUIT_REPLICA_STALE_THRESHOLD_MS, DFSConfigKeys.DFS_CLIENT_SHORT_CIRCUIT_REPLICA_STALE_THRESHOLD_MS_DEFAULT), conf.getInt(DFSConfigKeys.DFS_SHORT_CIRCUIT_SHARED_MEMORY_WATCHER_INTERRUPT_CHECK_MS, DFSConfigKeys.DFS_SHORT_CIRCUIT_SHARED_MEMORY_WATCHER_INTERRUPT_CHECK_MS_DEFAULT)); }
cache.scheduleSlotReleaser(slot); if (LOG.isTraceEnabled()) { suffix += " scheduling " + slot + " for later release.";
/** * Trim the eviction lists. */ private void trimEvictionMaps() { long now = Time.monotonicNow(); demoteOldEvictableMmaped(now); while (true) { long evictableSize = evictable.size(); long evictableMmappedSize = evictableMmapped.size(); if (evictableSize + evictableMmappedSize <= maxTotalSize) { return; } ShortCircuitReplica replica; if (evictableSize == 0) { replica = evictableMmapped.firstEntry().getValue(); } else { replica = evictable.firstEntry().getValue(); } if (LOG.isTraceEnabled()) { LOG.trace(this + ": trimEvictionMaps is purging " + replica + StringUtils.getStackTrace(Thread.currentThread())); } purge(replica); } }
@Test(timeout=60000) public void testCreateAndDestroy() throws Exception { ShortCircuitCache cache = new ShortCircuitCache(10, 1, 10, 1, 1, 10000, 0); cache.close(); }
@Test(timeout=60000) public void testExpiry() throws Exception { final ShortCircuitCache cache = new ShortCircuitCache(2, 1, 1, 10000000, 1, 10000000, 0); final TestFileDescriptorPair pair = new TestFileDescriptorPair(); ShortCircuitReplicaInfo replicaInfo1 = cache.fetchOrCreate( new ExtendedBlockId(123, "test_bp1"), new SimpleReplicaCreator(123, cache, pair)); Thread.sleep(10); ShortCircuitReplicaInfo replicaInfo2 = cache.fetchOrCreate( new ExtendedBlockId(123, "test_bp1"), new ShortCircuitReplicaCreator() { @Override cache.close();
try { MutableBoolean usedPeer = new MutableBoolean(false); slot = cache.allocShmSlot(datanode, peer, usedPeer, new ExtendedBlockId(block.getBlockId(), block.getBlockPoolId()), clientName); } catch (IOException e) { if (slot != null) { cache.freeSlot(slot);
final ShortCircuitCache cache = fs.getClient().getClientContext().getShortCircuitCache(); cache.getDfsClientShmManager().visit(new Visitor() { @Override public void visit(HashMap<DatanodeInfo, PerDatanodeVisitorInfo> info) new DatanodeInfo(cluster.getDataNodes().get(0).getDatanodeId()); Slot slot = cache.allocShmSlot(datanode, peer, usedPeer, blockId, "testAllocShm_client"); Assert.assertNotNull(slot); Assert.assertTrue(usedPeer.booleanValue()); cache.getDfsClientShmManager().visit(new Visitor() { @Override public void visit(HashMap<DatanodeInfo, PerDatanodeVisitorInfo> info) cache.scheduleSlotReleaser(slot);
ShortCircuitReplicaInfo info = cache.fetchOrCreate(key, this); InvalidToken exc = info.getInvalidTokenException(); if (exc != null) {
public ClientMmap getOrCreateClientMmap(boolean anchor) { return cache.getOrCreateClientMmap(this, anchor); }
long staleThresholdMs = cache.getStaleThresholdMs(); if (deltaMs > staleThresholdMs) { if (LOG.isTraceEnabled()) {
@Test(timeout=60000) public void testAddAndRetrieve() throws Exception { final ShortCircuitCache cache = new ShortCircuitCache(10, 10000000, 10, 10000000, 1, 10000, 0); final TestFileDescriptorPair pair = new TestFileDescriptorPair(); ShortCircuitReplicaInfo replicaInfo1 = cache.fetchOrCreate(new ExtendedBlockId(123, "test_bp1"), new SimpleReplicaCreator(123, cache, pair)); Preconditions.checkNotNull(replicaInfo1.getReplica()); replicaInfo1.getReplica().getMetaStream()); ShortCircuitReplicaInfo replicaInfo2 = cache.fetchOrCreate(new ExtendedBlockId(123, "test_bp1"), new ShortCircuitReplicaCreator() { @Override cache.fetchOrCreate( new ExtendedBlockId(123, "test_bp1"), new ShortCircuitReplicaCreator() { @Override cache.close();
try { MutableBoolean usedPeer = new MutableBoolean(false); slot = cache.allocShmSlot(datanode, peer, usedPeer, new ExtendedBlockId(block.getBlockId(), block.getBlockPoolId()), clientName); } catch (IOException e) { if (slot != null) { cache.freeSlot(slot);
/** * Trim the eviction lists. */ private void trimEvictionMaps() { long now = Time.monotonicNow(); demoteOldEvictableMmaped(now); while (true) { long evictableSize = evictable.size(); long evictableMmappedSize = evictableMmapped.size(); if (evictableSize + evictableMmappedSize <= maxTotalSize) { return; } ShortCircuitReplica replica; if (evictableSize == 0) { replica = evictableMmapped.firstEntry().getValue(); } else { replica = evictable.firstEntry().getValue(); } if (LOG.isTraceEnabled()) { LOG.trace(this + ": trimEvictionMaps is purging " + replica + StringUtils.getStackTrace(Thread.currentThread())); } purge(replica); } }
@Override public Boolean get() { ShortCircuitReplicaInfo info = cache.fetchOrCreate( new ExtendedBlockId(0, "test_bp1"), new ShortCircuitReplicaCreator() { @Override public ShortCircuitReplicaInfo createShortCircuitReplicaInfo() { return null; } }); if (info.getReplica() != null) { info.getReplica().unref(); return false; } return true; } }, 500, 60000);
public ClientMmap getOrCreateClientMmap(boolean anchor) { return cache.getOrCreateClientMmap(this, anchor); }
/** * Check if the replica is stale. * * Must be called with the cache lock held. */ boolean isStale() { if (slot != null) { // Check staleness by looking at the shared memory area we use to // communicate with the DataNode. boolean stale = !slot.isValid(); LOG.trace("{}: checked shared memory segment. isStale={}", this, stale); return stale; } else { // Fall back to old, time-based staleness method. long deltaMs = Time.monotonicNow() - creationTimeMs; long staleThresholdMs = cache.getStaleThresholdMs(); if (deltaMs > staleThresholdMs) { LOG.trace("{} is stale because it's {} ms old and staleThreadholdMS={}", this, deltaMs, staleThresholdMs); return true; } else { LOG.trace("{} is not stale because it's only {} ms old " + "and staleThresholdMs={}", this, deltaMs, staleThresholdMs); return false; } } }
new ShortCircuitCache(2, 10000000, 1, 10000000, 1, 10, 0); final TestFileDescriptorPair pairs[] = new TestFileDescriptorPair[] { new TestFileDescriptorPair(), final Integer iVal = new Integer(i); final ExtendedBlockId key = new ExtendedBlockId(i, "test_bp1"); replicaInfos[i] = cache.fetchOrCreate(key, new ShortCircuitReplicaCreator() { @Override ShortCircuitReplicaInfo info = cache.fetchOrCreate( new ExtendedBlockId(1, "test_bp1"), new ShortCircuitReplicaCreator() { @Override replicaInfos[i].getReplica().unref(); cache.close();
try { MutableBoolean usedPeer = new MutableBoolean(false); slot = cache.allocShmSlot(datanode, peer, usedPeer, new ExtendedBlockId(block.getBlockId(), block.getBlockPoolId()), clientName); } catch (IOException e) { if (slot != null) { cache.freeSlot(slot);
/** * Trim the eviction lists. */ private void trimEvictionMaps() { long now = Time.monotonicNow(); demoteOldEvictableMmaped(now); while (true) { long evictableSize = evictable.size(); long evictableMmappedSize = evictableMmapped.size(); if (evictableSize + evictableMmappedSize <= maxTotalSize) { return; } ShortCircuitReplica replica; try { if (evictableSize == 0) { replica = (ShortCircuitReplica)evictableMmapped.get(evictableMmapped .firstKey()); } else { replica = (ShortCircuitReplica)evictable.get(evictable.firstKey()); } } catch (NoSuchElementException e) { break; } if (LOG.isTraceEnabled()) { LOG.trace(this + ": trimEvictionMaps is purging " + replica + StringUtils.getStackTrace(Thread.currentThread())); } purge(replica); } }