NameNode.LOG.info("Caching file names occurring more than " + threshold + " times"); nameCache = new NameCache<ByteArray>(threshold); namesystem = ns; this.editLog = ns.getEditLog();
void markNameCacheInitialized() { writeLock(); try { nameCache.initialized(); } finally { writeUnlock(); } }
/** * Caches frequently used file names to reuse file name objects and * reduce heap size. */ void cacheName(INode inode) { // Name is cached only for files if (!inode.isFile()) { return; } ByteArray name = new ByteArray(inode.getLocalNameBytes()); name = nameCache.put(name); if (name != null) { inode.setLocalName(name.getBytes()); } }
@Test public void testDictionary() throws Exception { // Create dictionary with useThreshold 2 NameCache<String> cache = new NameCache<String>(2); String[] matching = {"part1", "part10000000", "fileabc", "abc", "filepart"}; String[] notMatching = {"spart1", "apart", "abcd", "def"}; for (String s : matching) { // Add useThreshold times so the names are promoted to dictionary cache.put(s); assertTrue(s == cache.put(s)); } for (String s : notMatching) { // Add < useThreshold times so the names are not promoted to dictionary cache.put(s); } // Mark dictionary as initialized cache.initialized(); for (String s : matching) { verifyNameReuse(cache, s, true); } // Check dictionary size assertEquals(matching.length, cache.size()); for (String s : notMatching) { verifyNameReuse(cache, s, false); } }
/** * Mark the name cache as initialized. The use count is no longer tracked * and the transient map used for initializing the cache is discarded to * save heap space. */ void initialized() { LOG.info("initialized with " + size() + " entries " + lookups + " lookups"); this.initialized = true; transientMap.clear(); transientMap = null; }
private void verifyNameReuse(NameCache<String> cache, String s, boolean reused) { cache.put(s); int lookupCount = cache.getLookupCount(); if (reused) { // Dictionary returns non null internal value assertNotNull(cache.put(s)); // Successful lookup increments lookup count assertEquals(lookupCount + 1, cache.getLookupCount()); } else { // Dictionary returns null - since name is not in the dictionary assertNull(cache.put(s)); // Lookup count remains the same assertEquals(lookupCount, cache.getLookupCount()); } } }
void shutdown() { nameCache.reset(); inodeMap.clear(); }
/** * Add a given name to the cache or track use count. * exist. If the name already exists, then the internal value is returned. * * @param name name to be looked up * @return internal value for the name if found; otherwise null */ K put(final K name) { K internal = cache.get(name); if (internal != null) { lookups++; return internal; } // Track the usage count only during initialization if (!initialized) { UseCount useCount = transientMap.get(name); if (useCount != null) { useCount.increment(); if (useCount.get() >= useThreshold) { promote(name); } return useCount.value; } useCount = new UseCount(name); transientMap.put(name, useCount); } return null; }
new NameCache<String>(2); String[] matching = {"part1", "part10000000", "fileabc", "abc", "filepart"}; String[] notMatching = {"spart1", "apart", "abcd", "def"}; cache.put(s); assertTrue(s == cache.put(s)); cache.put(s); cache.initialized(); assertEquals(matching.length, cache.size()); cache.reset(); cache.initialized();
/** * Mark the name cache as initialized. The use count is no longer tracked * and the transient map used for initializing the cache is discarded to * save heap space. */ void initialized() { LOG.info("initialized with " + size() + " entries " + lookups + " lookups"); this.initialized = true; transientMap.clear(); transientMap = null; }
private void verifyNameReuse(NameCache<String> cache, String s, boolean reused) { cache.put(s); int lookupCount = cache.getLookupCount(); if (reused) { // Dictionary returns non null internal value assertNotNull(cache.put(s)); // Successful lookup increments lookup count assertEquals(lookupCount + 1, cache.getLookupCount()); } else { // Dictionary returns null - since name is not in the dictionary assertNull(cache.put(s)); // Lookup count remains the same assertEquals(lookupCount, cache.getLookupCount()); } } }
/** * Reset the entire namespace tree. */ void reset() { writeLock(); try { rootDir = createRoot(getFSNamesystem()); inodeMap.clear(); addToInodeMap(rootDir); nameCache.reset(); inodeId.setCurrentValue(INodeId.LAST_RESERVED_ID); } finally { writeUnlock(); } }
/** * Add a given name to the cache or track use count. * exist. If the name already exists, then the internal value is returned. * * @param name name to be looked up * @return internal value for the name if found; otherwise null */ K put(final K name) { K internal = cache.get(name); if (internal != null) { lookups++; return internal; } // Track the usage count only during initialization if (!initialized) { UseCount useCount = transientMap.get(name); if (useCount != null) { useCount.increment(); if (useCount.get() >= useThreshold) { promote(name); } return useCount.value; } useCount = new UseCount(name); transientMap.put(name, useCount); } return null; }
void markNameCacheInitialized() { writeLock(); try { nameCache.initialized(); } finally { writeUnlock(); } }
/** * Mark the name cache as initialized. The use count is no longer tracked * and the transient map used for initializing the cache is discarded to * save heap space. */ void initialized() { LOG.info("initialized with " + size() + " entries " + lookups + " lookups"); this.initialized = true; transientMap.clear(); transientMap = null; }
/** * Caches frequently used file names to reuse file name objects and * reduce heap size. */ void cacheName(INode inode) { // Name is cached only for files if (!inode.isFile()) { return; } ByteArray name = new ByteArray(inode.getLocalNameBytes()); name = nameCache.put(name); if (name != null) { inode.setLocalName(name.getBytes()); } }
FSDirectory(FSImage fsImage, FSNamesystem ns, Configuration conf) { rootDir = new INodeDirectoryWithQuota(INodeDirectory.ROOT_NAME, ns.createFsOwnerPermissions(new FsPermission((short)0755)), Integer.MAX_VALUE, Long.MAX_VALUE); this.fsImage = fsImage; this.fsImage.setFSNamesystem(ns); int configuredLimit = conf.getInt( "dfs.ls.limit", 1000); this.lsLimit = configuredLimit>0 ? configuredLimit : 1000; int threshold = conf.getInt( "dfs.namenode.name.cache.threshold", 10); NameNode.LOG.info("Caching file names occuring more than " + threshold + " times "); nameCache = new NameCache<ByteArray>(threshold); initialize(conf); }
void shutdown() { nameCache.reset(); inodeMap.clear(); }
/** * Add a given name to the cache or track use count. * exist. If the name already exists, then the internal value is returned. * * @param name name to be looked up * @return internal value for the name if found; otherwise null */ K put(final K name) { K internal = cache.get(name); if (internal != null) { lookups++; return internal; } // Track the usage count only during initialization if (!initialized) { UseCount useCount = transientMap.get(name); if (useCount != null) { useCount.increment(); if (useCount.get() >= useThreshold) { promote(name); } return useCount.value; } useCount = new UseCount(name); transientMap.put(name, useCount); } return null; }
void markNameCacheInitialized() { writeLock(); try { nameCache.initialized(); } finally { writeUnlock(); } }