/** * Copy the block and meta files for the given block to the given destination. * @return the new meta and block files. * @throws IOException */ static File[] copyBlockFiles(long blockId, long genStamp, ReplicaInfo srcReplica, File destRoot, boolean calculateChecksum, int smallBufferSize, final Configuration conf) throws IOException { final File destDir = DatanodeUtil.idToBlockDir(destRoot, blockId); // blockName is same as the filename for the block final File dstFile = new File(destDir, srcReplica.getBlockName()); final File dstMeta = FsDatasetUtil.getMetaFile(dstFile, genStamp); return copyBlockFiles(srcReplica, dstMeta, dstFile, calculateChecksum, smallBufferSize, conf); }
static File moveBlockFiles(Block b, ReplicaInfo replicaInfo, File destdir) throws IOException { final File dstfile = new File(destdir, b.getBlockName()); final File dstmeta = FsDatasetUtil.getMetaFile(dstfile, b.getGenerationStamp()); try { replicaInfo.renameMeta(dstmeta.toURI()); } catch (IOException e) { throw new IOException("Failed to move meta file for " + b + " from " + replicaInfo.getMetadataURI() + " to " + dstmeta, e); } try { replicaInfo.renameData(dstfile.toURI()); } catch (IOException e) { throw new IOException("Failed to move block file for " + b + " from " + replicaInfo.getBlockURI() + " to " + dstfile.getAbsolutePath(), e); } if (LOG.isDebugEnabled()) { LOG.debug("addFinalizedBlock: Moved " + replicaInfo.getMetadataURI() + " to " + dstmeta + " and " + replicaInfo.getBlockURI() + " to " + dstfile); } return dstfile; }
private File[] copyReplicaWithNewBlockIdAndGS( ReplicaInfo replicaInfo, String bpid, long newBlkId, long newGS) throws IOException { String blockFileName = Block.BLOCK_FILE_PREFIX + newBlkId; FsVolumeImpl v = (FsVolumeImpl) replicaInfo.getVolume(); final File tmpDir = v.getBlockPoolSlice(bpid).getTmpDir(); final File destDir = DatanodeUtil.idToBlockDir(tmpDir, newBlkId); final File dstBlockFile = new File(destDir, blockFileName); final File dstMetaFile = FsDatasetUtil.getMetaFile(dstBlockFile, newGS); return FsDatasetImpl.copyBlockFiles(replicaInfo, dstMetaFile, dstBlockFile, true, DFSUtilClient.getSmallBufferSize(conf), conf); }
File addFinalizedBlock(Block b, ReplicaInfo replicaInfo) throws IOException { File blockDir = DatanodeUtil.idToBlockDir(finalizedDir, b.getBlockId()); fileIoProvider.mkdirsWithExistsCheck(volume, blockDir); File blockFile = FsDatasetImpl.moveBlockFiles(b, replicaInfo, blockDir); File metaFile = FsDatasetUtil.getMetaFile(blockFile, b.getGenerationStamp()); if (dfsUsage instanceof CachingGetSpaceUsed) { ((CachingGetSpaceUsed) dfsUsage).incDfsUsed( b.getNumBytes() + metaFile.length()); } return blockFile; }
final File metaFile = FsDatasetUtil.getMetaFile(blockFile, genStamp); long blockFileLen = blockFile.length(); long metaFileLen = metaFile.length();
File memMetaFile = FsDatasetUtil.getMetaFile(diskFile, memBlockInfo.getGenerationStamp()); if (fileIoProvider.exists(vol, memMetaFile)) {
public ReplicaInPipeline convertTemporaryToRbw(ExtendedBlock b, ReplicaInfo temp) throws IOException { final long blockId = b.getBlockId(); final long expectedGs = b.getGenerationStamp(); final long visible = b.getNumBytes(); final long numBytes = temp.getNumBytes(); // move block files to the rbw directory BlockPoolSlice bpslice = getBlockPoolSlice(b.getBlockPoolId()); final File dest = FsDatasetImpl.moveBlockFiles(b.getLocalBlock(), temp, bpslice.getRbwDir()); // create RBW final LocalReplicaInPipeline rbw = new ReplicaBuilder(ReplicaState.RBW) .setBlockId(blockId) .setLength(numBytes) .setGenerationStamp(expectedGs) .setFsVolume(this) .setDirectoryToUse(dest.getParentFile()) .setWriterThread(Thread.currentThread()) .setBytesToReserve(0) .buildLocalReplicaInPipeline(); rbw.setBytesAcked(visible); // load last checksum and datalen final File destMeta = FsDatasetUtil.getMetaFile(dest, b.getGenerationStamp()); byte[] lastChunkChecksum = loadLastPartialChunkChecksum(dest, destMeta); rbw.setLastChecksumAndDataLen(numBytes, lastChunkChecksum); return rbw; }
/** * Copy the block and meta files for the given block to the given destination. * @return the new meta and block files. * @throws IOException */ static File[] copyBlockFiles(long blockId, long genStamp, File srcMeta, File srcFile, File destRoot, boolean calculateChecksum) throws IOException { final File destDir = DatanodeUtil.idToBlockDir(destRoot, blockId); final File dstFile = new File(destDir, srcFile.getName()); final File dstMeta = FsDatasetUtil.getMetaFile(dstFile, genStamp); return copyBlockFiles(srcMeta, srcFile, dstMeta, dstFile, calculateChecksum); }
/** * Copy the block and meta files for the given block to the given destination. * @return the new meta and block files. * @throws IOException */ static File[] copyBlockFiles(long blockId, long genStamp, File srcMeta, File srcFile, File destRoot, boolean calculateChecksum) throws IOException { final File destDir = DatanodeUtil.idToBlockDir(destRoot, blockId); final File dstFile = new File(destDir, srcFile.getName()); final File dstMeta = FsDatasetUtil.getMetaFile(dstFile, genStamp); return copyBlockFiles(srcMeta, srcFile, dstMeta, dstFile, calculateChecksum); }
/** Check the files of a replica. */ static void checkReplicaFiles(final ReplicaInfo r) throws IOException { //check replica's file final File f = r.getBlockFile(); if (!f.exists()) { throw new FileNotFoundException("File " + f + " not found, r=" + r); } if (r.getBytesOnDisk() != f.length()) { throw new IOException("File length mismatched. The length of " + f + " is " + f.length() + " but r=" + r); } //check replica's meta file final File metafile = FsDatasetUtil.getMetaFile(f, r.getGenerationStamp()); if (!metafile.exists()) { throw new IOException("Metafile " + metafile + " does not exist, r=" + r); } if (metafile.length() == 0) { throw new IOException("Metafile " + metafile + " is empty, r=" + r); } }
/** Check the files of a replica. */ static void checkReplicaFiles(final ReplicaInfo r) throws IOException { //check replica's file final File f = r.getBlockFile(); if (!f.exists()) { throw new FileNotFoundException("File " + f + " not found, r=" + r); } if (r.getBytesOnDisk() != f.length()) { throw new IOException("File length mismatched. The length of " + f + " is " + f.length() + " but r=" + r); } //check replica's meta file final File metafile = FsDatasetUtil.getMetaFile(f, r.getGenerationStamp()); if (!metafile.exists()) { throw new IOException("Metafile " + metafile + " does not exist, r=" + r); } if (metafile.length() == 0) { throw new IOException("Metafile " + metafile + " is empty, r=" + r); } }
@Override // FsDatasetSpi public LengthInputStream getMetaDataInputStream(ExtendedBlock b) throws IOException { File meta = FsDatasetUtil.getMetaFile(getBlockFile(b), b.getGenerationStamp()); if (meta == null || !meta.exists()) { return null; } if (isNativeIOAvailable) { return new LengthInputStream( NativeIO.getShareDeleteFileInputStream(meta), meta.length()); } return new LengthInputStream(new FileInputStream(meta), meta.length()); }
static File moveBlockFiles(Block b, File srcfile, File destdir) throws IOException { final File dstfile = new File(destdir, b.getBlockName()); final File srcmeta = FsDatasetUtil.getMetaFile(srcfile, b.getGenerationStamp()); final File dstmeta = FsDatasetUtil.getMetaFile(dstfile, b.getGenerationStamp()); try { NativeIO.renameTo(srcmeta, dstmeta); } catch (IOException e) { throw new IOException("Failed to move meta file for " + b + " from " + srcmeta + " to " + dstmeta, e); } try { NativeIO.renameTo(srcfile, dstfile); } catch (IOException e) { throw new IOException("Failed to move block file for " + b + " from " + srcfile + " to " + dstfile.getAbsolutePath(), e); } if (LOG.isDebugEnabled()) { LOG.debug("addFinalizedBlock: Moved " + srcmeta + " to " + dstmeta + " and " + srcfile + " to " + dstfile); } return dstfile; }
@Override // FsDatasetSpi public LengthInputStream getMetaDataInputStream(ExtendedBlock b) throws IOException { File meta = FsDatasetUtil.getMetaFile(getBlockFile(b), b.getGenerationStamp()); if (meta == null || !meta.exists()) { return null; } if (isNativeIOAvailable) { return new LengthInputStream( NativeIO.getShareDeleteFileInputStream(meta), meta.length()); } return new LengthInputStream(new FileInputStream(meta), meta.length()); }
File addBlock(Block b, File f) throws IOException { File blockDir = DatanodeUtil.idToBlockDir(finalizedDir, b.getBlockId()); if (!blockDir.exists()) { if (!blockDir.mkdirs()) { throw new IOException("Failed to mkdirs " + blockDir); } } File blockFile = FsDatasetImpl.moveBlockFiles(b, f, blockDir); File metaFile = FsDatasetUtil.getMetaFile(blockFile, b.getGenerationStamp()); dfsUsage.incDfsUsed(b.getNumBytes()+metaFile.length()); return blockFile; }
File addBlock(Block b, File f) throws IOException { File blockDir = DatanodeUtil.idToBlockDir(finalizedDir, b.getBlockId()); if (!blockDir.exists()) { if (!blockDir.mkdirs()) { throw new IOException("Failed to mkdirs " + blockDir); } } File blockFile = FsDatasetImpl.moveBlockFiles(b, f, blockDir); File metaFile = FsDatasetUtil.getMetaFile(blockFile, b.getGenerationStamp()); dfsUsage.incDfsUsed(b.getNumBytes()+metaFile.length()); return blockFile; }
static File moveBlockFiles(Block b, File srcfile, File destdir) throws IOException { final File dstfile = new File(destdir, b.getBlockName()); final File srcmeta = FsDatasetUtil.getMetaFile(srcfile, b.getGenerationStamp()); final File dstmeta = FsDatasetUtil.getMetaFile(dstfile, b.getGenerationStamp()); try { NativeIO.renameTo(srcmeta, dstmeta); } catch (IOException e) { throw new IOException("Failed to move meta file for " + b + " from " + srcmeta + " to " + dstmeta, e); } try { NativeIO.renameTo(srcfile, dstfile); } catch (IOException e) { throw new IOException("Failed to move block file for " + b + " from " + srcfile + " to " + dstfile.getAbsolutePath(), e); } if (LOG.isDebugEnabled()) { LOG.debug("addFinalizedBlock: Moved " + srcmeta + " to " + dstmeta + " and " + srcfile + " to " + dstfile); } return dstfile; }
public static File getMetaFile(FsDatasetSpi<?> fsd, String bpid, Block b) throws IOException { return FsDatasetUtil.getMetaFile(getBlockFile(fsd, bpid, b), b .getGenerationStamp()); }
private File[] copyReplicaWithNewBlockIdAndGS( ReplicaUnderRecovery replicaInfo, String bpid, long newBlkId, long newGS) throws IOException { String blockFileName = Block.BLOCK_FILE_PREFIX + newBlkId; FsVolumeImpl v = (FsVolumeImpl) replicaInfo.getVolume(); final File tmpDir = v.getBlockPoolSlice(bpid).getTmpDir(); final File destDir = DatanodeUtil.idToBlockDir(tmpDir, newBlkId); final File dstBlockFile = new File(destDir, blockFileName); final File dstMetaFile = FsDatasetUtil.getMetaFile(dstBlockFile, newGS); return copyBlockFiles(replicaInfo.getMetaFile(), replicaInfo.getBlockFile(), dstMetaFile, dstBlockFile, true); }
private File[] copyReplicaWithNewBlockIdAndGS( ReplicaUnderRecovery replicaInfo, String bpid, long newBlkId, long newGS) throws IOException { String blockFileName = Block.BLOCK_FILE_PREFIX + newBlkId; FsVolumeImpl v = (FsVolumeImpl) replicaInfo.getVolume(); final File tmpDir = v.getBlockPoolSlice(bpid).getTmpDir(); final File destDir = DatanodeUtil.idToBlockDir(tmpDir, newBlkId); final File dstBlockFile = new File(destDir, blockFileName); final File dstMetaFile = FsDatasetUtil.getMetaFile(dstBlockFile, newGS); return copyBlockFiles(replicaInfo.getMetaFile(), replicaInfo.getBlockFile(), dstMetaFile, dstBlockFile, true); }