File metaFile = FsDatasetUtil.findMetaFile(blkFile); block.setGenerationStamp( Block.getGenerationStamp(metaFile.getName()));
/** * Copy the block and meta files for the given block to the given destination. * @return the new meta and block files. * @throws IOException */ static File[] copyBlockFiles(long blockId, long genStamp, ReplicaInfo srcReplica, File destRoot, boolean calculateChecksum, int smallBufferSize, final Configuration conf) throws IOException { final File destDir = DatanodeUtil.idToBlockDir(destRoot, blockId); // blockName is same as the filename for the block final File dstFile = new File(destDir, srcReplica.getBlockName()); final File dstMeta = FsDatasetUtil.getMetaFile(dstFile, genStamp); return copyBlockFiles(srcReplica, dstMeta, dstFile, calculateChecksum, smallBufferSize, conf); }
if (isFinalized && FsDatasetUtil.isUnlinkTmpFile(file)) { file = recoverTempUnlinkedBlock(file); if (file == null) { // the original block still exists, so we cover it long genStamp = FsDatasetUtil.getGenerationStampFromFile( files, file, i); long blockId = Block.filename2id(file.getName());
@Override // FsDatasetSpi public synchronized Block getStoredBlock(String bpid, long blkid) throws IOException { File blockfile = getFile(bpid, blkid, false); if (blockfile == null) { return null; } final File metafile = FsDatasetUtil.findMetaFile(blockfile); final long gs = FsDatasetUtil.parseGenerationStamp(blockfile, metafile); return new Block(blkid, blockfile.length(), gs); }
/** * Recover an unlinked tmp file on datanode restart. If the original block * does not exist, then the tmp file is renamed to be the * original file name and the original name is returned; otherwise the tmp * file is deleted and null is returned. */ File recoverTempUnlinkedBlock(File unlinkedTmp) throws IOException { File blockFile = FsDatasetUtil.getOrigFile(unlinkedTmp); if (blockFile.exists()) { // If the original block file still exists, then no recovery is needed. if (!fileIoProvider.delete(volume, unlinkedTmp)) { throw new IOException("Unable to cleanup unlinked tmp file " + unlinkedTmp); } return null; } else { fileIoProvider.rename(volume, unlinkedTmp, blockFile); return blockFile; } }
BlockMetadataHeader.writeHeader(metaOut, checksum); metaOut.close(); FsDatasetUtil.computeChecksum( srcMeta, srcMeta, blockFile, smallBufferSize, conf); System.out.println(
@Override // FsDatasetSpi public synchronized Block getStoredBlock(String bpid, long blkid) throws IOException { File blockfile = getFile(bpid, blkid, false); if (blockfile == null) { return null; } final File metafile = FsDatasetUtil.findMetaFile(blockfile); final long gs = FsDatasetUtil.parseGenerationStamp(blockfile, metafile); return new Block(blkid, blockfile.length(), gs); }
/** * Recover an unlinked tmp file on datanode restart. If the original block * does not exist, then the tmp file is renamed to be the * original file name and the original name is returned; otherwise the tmp * file is deleted and null is returned. */ File recoverTempUnlinkedBlock(File unlinkedTmp) throws IOException { File blockFile = FsDatasetUtil.getOrigFile(unlinkedTmp); if (blockFile.exists()) { // If the original block file still exists, then no recovery is needed. if (!unlinkedTmp.delete()) { throw new IOException("Unable to cleanup unlinked tmp file " + unlinkedTmp); } return null; } else { if (!unlinkedTmp.renameTo(blockFile)) { throw new IOException("Unable to rename unlinked tmp file " + unlinkedTmp); } return blockFile; } }
FsDatasetUtil.computeChecksum(rbw.getMetaFile(), rbw.getMetaFile(), rbw.getBlockFile());
static File moveBlockFiles(Block b, ReplicaInfo replicaInfo, File destdir) throws IOException { final File dstfile = new File(destdir, b.getBlockName()); final File dstmeta = FsDatasetUtil.getMetaFile(dstfile, b.getGenerationStamp()); try { replicaInfo.renameMeta(dstmeta.toURI()); } catch (IOException e) { throw new IOException("Failed to move meta file for " + b + " from " + replicaInfo.getMetadataURI() + " to " + dstmeta, e); } try { replicaInfo.renameData(dstfile.toURI()); } catch (IOException e) { throw new IOException("Failed to move block file for " + b + " from " + replicaInfo.getBlockURI() + " to " + dstfile.getAbsolutePath(), e); } if (LOG.isDebugEnabled()) { LOG.debug("addFinalizedBlock: Moved " + replicaInfo.getMetadataURI() + " to " + dstmeta + " and " + replicaInfo.getBlockURI() + " to " + dstfile); } return dstfile; }
if (isFinalized && FsDatasetUtil.isUnlinkTmpFile(file)) { file = recoverTempUnlinkedBlock(file); if (file == null) { // the original block still exists, so we cover it continue; long genStamp = FsDatasetUtil.getGenerationStampFromFile( files, file); long blockId = Block.filename2id(file.getName());
public boolean changeGenStampOfBlock(int dnIndex, ExtendedBlock blk, long newGenStamp) throws IOException { File blockFile = getBlockFile(dnIndex, blk); File metaFile = FsDatasetUtil.findMetaFile(blockFile); return metaFile.renameTo(new File(DatanodeUtil.getMetaName( blockFile.getAbsolutePath(), newGenStamp))); }
/** * Recover an unlinked tmp file on datanode restart. If the original block * does not exist, then the tmp file is renamed to be the * original file name and the original name is returned; otherwise the tmp * file is deleted and null is returned. */ File recoverTempUnlinkedBlock(File unlinkedTmp) throws IOException { File blockFile = FsDatasetUtil.getOrigFile(unlinkedTmp); if (blockFile.exists()) { // If the original block file still exists, then no recovery is needed. if (!unlinkedTmp.delete()) { throw new IOException("Unable to cleanup unlinked tmp file " + unlinkedTmp); } return null; } else { if (!unlinkedTmp.renameTo(blockFile)) { throw new IOException("Unable to rename unlinked tmp file " + unlinkedTmp); } return blockFile; } }
private File[] copyReplicaWithNewBlockIdAndGS( ReplicaInfo replicaInfo, String bpid, long newBlkId, long newGS) throws IOException { String blockFileName = Block.BLOCK_FILE_PREFIX + newBlkId; FsVolumeImpl v = (FsVolumeImpl) replicaInfo.getVolume(); final File tmpDir = v.getBlockPoolSlice(bpid).getTmpDir(); final File destDir = DatanodeUtil.idToBlockDir(tmpDir, newBlkId); final File dstBlockFile = new File(destDir, blockFileName); final File dstMetaFile = FsDatasetUtil.getMetaFile(dstBlockFile, newGS); return FsDatasetImpl.copyBlockFiles(replicaInfo, dstMetaFile, dstBlockFile, true, DFSUtilClient.getSmallBufferSize(conf), conf); }
if (isFinalized && FsDatasetUtil.isUnlinkTmpFile(file)) { file = recoverTempUnlinkedBlock(file); if (file == null) { // the original block still exists, so we cover it continue; long genStamp = FsDatasetUtil.getGenerationStampFromFile( files, file); long blockId = Block.filename2id(file.getName());
File addFinalizedBlock(Block b, ReplicaInfo replicaInfo) throws IOException { File blockDir = DatanodeUtil.idToBlockDir(finalizedDir, b.getBlockId()); fileIoProvider.mkdirsWithExistsCheck(volume, blockDir); File blockFile = FsDatasetImpl.moveBlockFiles(b, replicaInfo, blockDir); File metaFile = FsDatasetUtil.getMetaFile(blockFile, b.getGenerationStamp()); if (dfsUsage instanceof CachingGetSpaceUsed) { ((CachingGetSpaceUsed) dfsUsage).incDfsUsed( b.getNumBytes() + metaFile.length()); } return blockFile; }
final File metaFile = FsDatasetUtil.getMetaFile(blockFile, genStamp); long blockFileLen = blockFile.length(); long metaFileLen = metaFile.length();
File memMetaFile = FsDatasetUtil.getMetaFile(diskFile, memBlockInfo.getGenerationStamp()); if (fileIoProvider.exists(vol, memMetaFile)) {
public ReplicaInPipeline convertTemporaryToRbw(ExtendedBlock b, ReplicaInfo temp) throws IOException { final long blockId = b.getBlockId(); final long expectedGs = b.getGenerationStamp(); final long visible = b.getNumBytes(); final long numBytes = temp.getNumBytes(); // move block files to the rbw directory BlockPoolSlice bpslice = getBlockPoolSlice(b.getBlockPoolId()); final File dest = FsDatasetImpl.moveBlockFiles(b.getLocalBlock(), temp, bpslice.getRbwDir()); // create RBW final LocalReplicaInPipeline rbw = new ReplicaBuilder(ReplicaState.RBW) .setBlockId(blockId) .setLength(numBytes) .setGenerationStamp(expectedGs) .setFsVolume(this) .setDirectoryToUse(dest.getParentFile()) .setWriterThread(Thread.currentThread()) .setBytesToReserve(0) .buildLocalReplicaInPipeline(); rbw.setBytesAcked(visible); // load last checksum and datalen final File destMeta = FsDatasetUtil.getMetaFile(dest, b.getGenerationStamp()); byte[] lastChunkChecksum = loadLastPartialChunkChecksum(dest, destMeta); rbw.setLastChecksumAndDataLen(numBytes, lastChunkChecksum); return rbw; }
/** * Copy the block and meta files for the given block to the given destination. * @return the new meta and block files. * @throws IOException */ static File[] copyBlockFiles(long blockId, long genStamp, File srcMeta, File srcFile, File destRoot, boolean calculateChecksum) throws IOException { final File destDir = DatanodeUtil.idToBlockDir(destRoot, blockId); final File dstFile = new File(destDir, srcFile.getName()); final File dstMeta = FsDatasetUtil.getMetaFile(dstFile, genStamp); return copyBlockFiles(srcMeta, srcFile, dstMeta, dstFile, calculateChecksum); }