private File getBlockFile(String bpid, ExtendedBlock blk) throws IOException { return new File(DatanodeUtil.idToBlockDir(getFinalizedDir(bpid), blk.getBlockId()).toString() + "/" + blk.getBlockName()); }
ris.dropCacheBehindReads(block.getBlockName(), 0, 0, POSIX_FADV_SEQUENTIAL);
/** * close opened files. */ @Override public void close() throws IOException { if (ris.getDataInFd() != null && ((dropCacheBehindAllReads) || (dropCacheBehindLargeReads && isLongRead()))) { try { ris.dropCacheBehindReads(block.getBlockName(), lastCacheDropOffset, offset - lastCacheDropOffset, POSIX_FADV_DONTNEED); } catch (Exception e) { LOG.warn("Unable to drop cache on file close", e); } } if (curReadahead != null) { curReadahead.cancel(); } try { ris.closeStreams(); } finally { IOUtils.closeStream(ris); ris = null; } }
streams.dropCacheBehindWrites(block.getBlockName(), 0, dropPos, POSIX_FADV_DONTNEED);
/** * Manage the OS buffer cache by performing read-ahead * and drop-behind. */ private void manageOsCache() throws IOException { // We can't manage the cache for this block if we don't have a file // descriptor to work with. if (ris.getDataInFd() == null) { return; } // Perform readahead if necessary if ((readaheadLength > 0) && (datanode.readaheadPool != null) && (alwaysReadahead || isLongRead())) { curReadahead = datanode.readaheadPool.readaheadStream( clientTraceFmt, ris.getDataInFd(), offset, readaheadLength, Long.MAX_VALUE, curReadahead); } // Drop what we've just read from cache, since we aren't // likely to need it again if (dropCacheBehindAllReads || (dropCacheBehindLargeReads && isLongRead())) { long nextCacheDropOffset = lastCacheDropOffset + CACHE_DROP_INTERVAL_BYTES; if (offset >= nextCacheDropOffset) { long dropLength = offset - lastCacheDropOffset; ris.dropCacheBehindReads(block.getBlockName(), lastCacheDropOffset, dropLength, POSIX_FADV_DONTNEED); lastCacheDropOffset = offset; } } }
corruptSize += block.getNumBytes(); out.print("\n" + path + ": CORRUPT blockpool " + block.getBlockPoolId() + " block " + block.getBlockName() + "\n");
private void setCachedLocations(LocatedBlock block) { CachedBlock cachedBlock = new CachedBlock(block.getBlock().getBlockId(), (short)0, false); cachedBlock = cachedBlocks.get(cachedBlock); if (cachedBlock == null) { return; } List<DatanodeDescriptor> cachedDNs = cachedBlock.getDatanodes(Type.CACHED); for (DatanodeDescriptor datanode : cachedDNs) { // Filter out cached blocks that do not have a backing replica. // // This should not happen since it means the CacheManager thinks // something is cached that does not exist, but it's a safety // measure. boolean found = false; for (DatanodeInfo loc : block.getLocations()) { if (loc.equals(datanode)) { block.addCachedLoc(loc); found = true; break; } } if (!found) { LOG.warn("Datanode {} is not a valid cache location for block {} " + "because that node does not have a backing replica!", datanode, block.getBlock().getBlockName()); } } }
try { NativeIO.POSIX.getCacheManipulator().posixFadviseIfPossible( block.getBlockName(), blockInFd, lastCacheDropOffset, offset - lastCacheDropOffset, NativeIO.POSIX.POSIX_FADV_DONTNEED);
try { NativeIO.POSIX.getCacheManipulator().posixFadviseIfPossible( block.getBlockName(), blockInFd, lastCacheDropOffset, offset - lastCacheDropOffset, NativeIO.POSIX.POSIX_FADV_DONTNEED);
/** * Get file correpsonding to a block * @param storageDir storage directory * @param blk the block * @return data file corresponding to the block */ public static File getBlockFile(File storageDir, ExtendedBlock blk) { return new File(DatanodeUtil.idToBlockDir(getFinalizedDir(storageDir, blk.getBlockPoolId()), blk.getBlockId()), blk.getBlockName()); }
/** * Manage the OS buffer cache by performing read-ahead * and drop-behind. */ private void manageOsCache() throws IOException { // We can't manage the cache for this block if we don't have a file // descriptor to work with. if (blockInFd == null) return; // Perform readahead if necessary if ((readaheadLength > 0) && (datanode.readaheadPool != null) && (alwaysReadahead || isLongRead())) { curReadahead = datanode.readaheadPool.readaheadStream( clientTraceFmt, blockInFd, offset, readaheadLength, Long.MAX_VALUE, curReadahead); } // Drop what we've just read from cache, since we aren't // likely to need it again if (dropCacheBehindAllReads || (dropCacheBehindLargeReads && isLongRead())) { long nextCacheDropOffset = lastCacheDropOffset + CACHE_DROP_INTERVAL_BYTES; if (offset >= nextCacheDropOffset) { long dropLength = offset - lastCacheDropOffset; NativeIO.POSIX.getCacheManipulator().posixFadviseIfPossible( block.getBlockName(), blockInFd, lastCacheDropOffset, dropLength, NativeIO.POSIX.POSIX_FADV_DONTNEED); lastCacheDropOffset = offset; } } }
protected final boolean verifyBlockDeletedFromDir(File dir, LocatedBlocks locatedBlocks) { for (LocatedBlock lb : locatedBlocks.getLocatedBlocks()) { File targetDir = DatanodeUtil.idToBlockDir(dir, lb.getBlock().getBlockId()); File blockFile = new File(targetDir, lb.getBlock().getBlockName()); if (blockFile.exists()) { LOG.warn("blockFile: " + blockFile.getAbsolutePath() + " exists after deletion."); return false; } File metaFile = new File(targetDir, DatanodeUtil.getMetaName(lb.getBlock().getBlockName(), lb.getBlock().getGenerationStamp())); if (metaFile.exists()) { LOG.warn("metaFile: " + metaFile.getAbsolutePath() + " exists after deletion."); return false; } } return true; }
if (dropPos > 0 && dropCacheBehindWrites) { NativeIO.POSIX.getCacheManipulator().posixFadviseIfPossible( block.getBlockName(), outFd, 0, dropPos, NativeIO.POSIX.POSIX_FADV_DONTNEED);
/** * Manage the OS buffer cache by performing read-ahead * and drop-behind. */ private void manageOsCache() throws IOException { // We can't manage the cache for this block if we don't have a file // descriptor to work with. if (blockInFd == null) return; // Perform readahead if necessary if ((readaheadLength > 0) && (datanode.readaheadPool != null) && (alwaysReadahead || isLongRead())) { curReadahead = datanode.readaheadPool.readaheadStream( clientTraceFmt, blockInFd, offset, readaheadLength, Long.MAX_VALUE, curReadahead); } // Drop what we've just read from cache, since we aren't // likely to need it again if (dropCacheBehindAllReads || (dropCacheBehindLargeReads && isLongRead())) { long nextCacheDropOffset = lastCacheDropOffset + CACHE_DROP_INTERVAL_BYTES; if (offset >= nextCacheDropOffset) { long dropLength = offset - lastCacheDropOffset; NativeIO.POSIX.getCacheManipulator().posixFadviseIfPossible( block.getBlockName(), blockInFd, lastCacheDropOffset, dropLength, NativeIO.POSIX.POSIX_FADV_DONTNEED); lastCacheDropOffset = offset; } } }
/** * Get the latest metadata file correpsonding to a block * @param storageDir storage directory * @param blk the block * @return metadata file corresponding to the block */ public static File getBlockMetadataFile(File storageDir, ExtendedBlock blk) { return new File(DatanodeUtil.idToBlockDir(getFinalizedDir(storageDir, blk.getBlockPoolId()), blk.getBlockId()), blk.getBlockName() + "_" + blk.getGenerationStamp() + Block.METADATA_EXTENSION); }
File targetDir = DatanodeUtil.idToBlockDir(lazyPersistDir, blockId); File blockFile = new File(targetDir, lb.getBlock().getBlockName()); if (blockFile.exists()) {
for (ExtendedBlock b : blocks2Remove) { if(LOG.isDebugEnabled()) { LOG.debug("Removing the block " + b.getBlockName()); new MyFileFilter(b.getBlockName(), true))) { DataNodeTestUtils.getFSDataset(dn0).unfinalizeBlock(b); if (!f.delete()) { LOG.warn("Couldn't delete " + b.getBlockName()); } else { LOG.debug("Deleted file " + f.toString());
if (dropPos > 0 && dropCacheBehindWrites) { NativeIO.POSIX.getCacheManipulator().posixFadviseIfPossible( block.getBlockName(), outFd, 0, dropPos, NativeIO.POSIX.POSIX_FADV_DONTNEED);
/** * Create a file with one block and corrupt some/all of the block replicas. */ private void createAFileWithCorruptedBlockReplicas(Path filePath, short repl, int corruptBlockCount) throws IOException, AccessControlException, FileNotFoundException, UnresolvedLinkException, InterruptedException, TimeoutException { DFSTestUtil.createFile(dfs, filePath, BLOCK_SIZE, repl, 0); DFSTestUtil.waitReplication(dfs, filePath, repl); // Locate the file blocks by asking name node final LocatedBlocks locatedblocks = dfs.dfs.getNamenode() .getBlockLocations(filePath.toString(), 0L, BLOCK_SIZE); Assert.assertEquals(repl, locatedblocks.get(0).getLocations().length); // The file only has one block LocatedBlock lblock = locatedblocks.get(0); DatanodeInfo[] datanodeinfos = lblock.getLocations(); ExtendedBlock block = lblock.getBlock(); // corrupt some /all of the block replicas for (int i = 0; i < corruptBlockCount; i++) { DatanodeInfo dninfo = datanodeinfos[i]; final DataNode dn = cluster.getDataNode(dninfo.getIpcPort()); corruptBlock(block, dn); LOG.debug("Corrupted block " + block.getBlockName() + " on data node " + dninfo); } }
} catch (Exception e) { Assert.assertTrue( e.getMessage().contains(blocks[NON_EXISTENT].getBlockName())); Assert.assertTrue(e instanceof ReplicaAlreadyExistsException);