/** * Delete the block file and meta file from the disk synchronously, adjust * dfsUsed statistics accordingly. */ void deleteSync(FsVolumeReference volumeRef, ReplicaInfo replicaToDelete, ExtendedBlock block, String trashDirectory) { LOG.info("Deleting " + block.getLocalBlock() + " replica " + replicaToDelete); ReplicaFileDeleteTask deletionTask = new ReplicaFileDeleteTask(volumeRef, replicaToDelete, block, trashDirectory); deletionTask.run(); }
private void buildContinuousBlockMovingInfos(LocatedBlock blockInfo, DatanodeInfo sourceNode, StorageType sourceStorageType, DatanodeInfo targetNode, StorageType targetStorageType, List<BlockMovingInfo> blkMovingInfos) { Block blk = ExtendedBlock.getLocalBlock(blockInfo.getBlock()); BlockMovingInfo blkMovingInfo = new BlockMovingInfo(blk, sourceNode, targetNode, sourceStorageType, targetStorageType); blkMovingInfos.add(blkMovingInfo); }
static boolean completeFile(FSNamesystem fsn, FSPermissionChecker pc, final String srcArg, String holder, ExtendedBlock last, long fileId) throws IOException { String src = srcArg; if (NameNode.stateChangeLog.isDebugEnabled()) { NameNode.stateChangeLog.debug("DIR* NameSystem.completeFile: " + src + " for " + holder); } checkBlock(fsn, last); INodesInPath iip = fsn.dir.resolvePath(pc, src, fileId); return completeFileInternal(fsn, iip, holder, ExtendedBlock.getLocalBlock(last), fileId); }
/** * Delete the block file and meta file from the disk asynchronously, adjust * dfsUsed statistics accordingly. */ void deleteAsync(FsVolumeReference volumeRef, ReplicaInfo replicaToDelete, ExtendedBlock block, String trashDirectory) { LOG.info("Scheduling " + block.getLocalBlock() + " replica " + replicaToDelete + " for deletion"); ReplicaFileDeleteTask deletionTask = new ReplicaFileDeleteTask( volumeRef, replicaToDelete, block, trashDirectory); execute(((FsVolumeImpl) volumeRef.getVolume()), deletionTask); }
@Override public String toString() { // Called in AsyncDiskService.execute for displaying error messages. return "deletion of block " + block.getBlockPoolId() + " " + block.getLocalBlock() + " with block file " + replicaToDelete.getBlockURI() + " and meta file " + replicaToDelete.getMetadataURI() + " from volume " + volume; }
@Override public FsVolumeImpl getVolume(final ExtendedBlock b) { try (AutoCloseableLock lock = datasetLock.acquire()) { final ReplicaInfo r = volumeMap.get(b.getBlockPoolId(), b.getLocalBlock()); return r != null ? (FsVolumeImpl) r.getVolume() : null; } }
/** * Get the meta info of a block stored in volumeMap. To find a block, * block pool Id, block Id and generation stamp must match. * @param b extended block * @return the meta replica information * @throws ReplicaNotFoundException if no entry is in the map or * there is a generation stamp mismatch */ ReplicaInfo getReplicaInfo(ExtendedBlock b) throws ReplicaNotFoundException { ReplicaInfo info = volumeMap.get(b.getBlockPoolId(), b.getLocalBlock()); if (info == null) { if (volumeMap.get(b.getBlockPoolId(), b.getLocalBlock().getBlockId()) == null) { throw new ReplicaNotFoundException( ReplicaNotFoundException.NON_EXISTENT_REPLICA + b); } else { throw new ReplicaNotFoundException( ReplicaNotFoundException.UNEXPECTED_GS_REPLICA + b); } } return info; }
throws IOException { checkOperation(OperationCategory.WRITE); LOG.info("updatePipeline(" + oldBlock.getLocalBlock() + ", newGS=" + newBlock.getGenerationStamp() + ", newLength=" + newBlock.getNumBytes() LOG.info("updatePipeline(" + oldBlock.getLocalBlock() + " => " + newBlock.getLocalBlock() + ") success");
@Override // FsDatasetSpi public boolean contains(final ExtendedBlock block) { try (AutoCloseableLock lock = datasetLock.acquire()) { final long blockId = block.getLocalBlock().getBlockId(); final String bpid = block.getBlockPoolId(); final ReplicaInfo r = volumeMap.get(bpid, blockId); return (r != null && r.blockDataExists()); } }
public ReplicaInPipeline createTemporary(ExtendedBlock b) throws IOException { // create a temporary file to hold block in the designated volume File f = createTmpFile(b.getBlockPoolId(), b.getLocalBlock()); LocalReplicaInPipeline newReplicaInfo = new ReplicaBuilder(ReplicaState.TEMPORARY) .setBlockId(b.getBlockId()) .setGenerationStamp(b.getGenerationStamp()) .setDirectoryToUse(f.getParentFile()) .setBytesToReserve(b.getLocalBlock().getNumBytes()) .setFsVolume(this) .buildLocalReplicaInPipeline(); return newReplicaInfo; }
private void notifyNamenodeBlock(ExtendedBlock block, BlockStatus status, String delHint, String storageUuid, boolean isOnTransientStorage) { checkBlock(block); final ReceivedDeletedBlockInfo info = new ReceivedDeletedBlockInfo( block.getLocalBlock(), status, delHint); final DatanodeStorage storage = dn.getFSDataset().getStorage(storageUuid); for (BPServiceActor actor : bpServices) { actor.getIbrManager().notifyNamenodeBlock(info, storage, isOnTransientStorage); } }
@Override public void run() { final long blockLength = replicaToDelete.getBlockDataLength(); final long metaLength = replicaToDelete.getMetadataLength(); boolean result; result = (trashDirectory == null) ? deleteFiles() : moveFiles(); if (!result) { LOG.warn("Unexpected error trying to " + (trashDirectory == null ? "delete" : "move") + " block " + block.getBlockPoolId() + " " + block.getLocalBlock() + " at file " + replicaToDelete.getBlockURI() + ". Ignored."); } else { if(block.getLocalBlock().getNumBytes() != BlockCommand.NO_ACK){ datanode.notifyNamenodeDeletedBlock(block, volume.getStorageID()); } volume.onBlockFileDeletion(block.getBlockPoolId(), blockLength); volume.onMetaFileDeletion(block.getBlockPoolId(), metaLength); LOG.info("Deleted " + block.getBlockPoolId() + " " + block.getLocalBlock() + " URI " + replicaToDelete.getBlockURI()); } updateDeletedBlockId(block); IOUtils.cleanup(null, volumeRef); } }
BlockInfo storedBlock = getStoredBlock(ExtendedBlock.getLocalBlock(block)); if (storedBlock == null) { throw new IOException(block + " does not exist.");
@Override // FsDatasetSpi public ReplicaRecoveryInfo initReplicaRecovery(RecoveringBlock rBlock) throws IOException { return initReplicaRecovery(rBlock.getBlock().getBlockPoolId(), volumeMap, rBlock.getBlock().getLocalBlock(), rBlock.getNewGenerationStamp(), datanode.getDnConf().getXceiverStopTimeout()); }
/** * Remove the temporary block file (if any) */ @Override // FsDatasetSpi public void unfinalizeBlock(ExtendedBlock b) throws IOException { try (AutoCloseableLock lock = datasetLock.acquire()) { ReplicaInfo replicaInfo = volumeMap.get(b.getBlockPoolId(), b.getLocalBlock()); if (replicaInfo != null && replicaInfo.getState() == ReplicaState.TEMPORARY) { // remove from volumeMap volumeMap.remove(b.getBlockPoolId(), b.getLocalBlock()); // delete the on-disk temp file if (delBlockFromDisk(replicaInfo)) { LOG.warn("Block " + b + " unfinalized and removed. "); } if (replicaInfo.getVolume().isTransientStorage()) { ramDiskReplicaTracker.discardReplica(b.getBlockPoolId(), b.getBlockId(), true); } } } }
public ReplicaInPipeline createRbw(ExtendedBlock b) throws IOException { File f = createRbwFile(b.getBlockPoolId(), b.getLocalBlock()); LocalReplicaInPipeline newReplicaInfo = new ReplicaBuilder(ReplicaState.RBW) .setBlockId(b.getBlockId()) .setGenerationStamp(b.getGenerationStamp()) .setFsVolume(this) .setDirectoryToUse(f.getParentFile()) .setBytesToReserve(b.getNumBytes()) .buildLocalReplicaInPipeline(); return newReplicaInfo; }
@Override // FsDatasetSpi public InputStream getBlockInputStream(ExtendedBlock b, long seekOffset) throws IOException { ReplicaInfo info; synchronized(this) { info = volumeMap.get(b.getBlockPoolId(), b.getLocalBlock()); } if (info != null && info.getVolume().isTransientStorage()) { ramDiskReplicaTracker.touch(b.getBlockPoolId(), b.getBlockId()); datanode.getMetrics().incrRamDiskBlocksReadHits(); } if(info != null && info.blockDataExists()) { return info.getDataInputStream(seekOffset); } else { throw new IOException("No data exists for block " + b); } }
static void abandonBlock( FSDirectory fsd, FSPermissionChecker pc, ExtendedBlock b, long fileId, String src, String holder) throws IOException { final INodesInPath iip = fsd.resolvePath(pc, src, fileId); src = iip.getPath(); FSNamesystem fsn = fsd.getFSNamesystem(); final INodeFile file = fsn.checkLease(iip, holder, fileId); Preconditions.checkState(file.isUnderConstruction()); if (file.getBlockType() == BlockType.STRIPED) { return; // do not abandon block for striped file } Block localBlock = ExtendedBlock.getLocalBlock(b); fsd.writeLock(); try { // Remove the block from the pending creates list if (!unprotectedRemoveBlock(fsd, src, iip, file, localBlock)) { return; } } finally { fsd.writeUnlock(); } persistBlocks(fsd, src, file, false); }
DBlock newDBlock(LocatedBlock lb, List<MLocation> locations, ErasureCodingPolicy ecPolicy) { Block blk = lb.getBlock().getLocalBlock(); DBlock db; if (lb.isStriped()) { LocatedStripedBlock lsb = (LocatedStripedBlock) lb; byte[] indices = new byte[lsb.getBlockIndices().length]; for (int i = 0; i < indices.length; i++) { indices[i] = (byte) lsb.getBlockIndices()[i]; } db = new DBlockStriped(blk, indices, (short) ecPolicy.getNumDataUnits(), ecPolicy.getCellSize()); } else { db = new DBlock(blk); } for(MLocation ml : locations) { StorageGroup source = storages.getSource(ml); if (source != null) { db.addLocation(source); } } return db; }
public ReplicaInPipeline convertTemporaryToRbw(ExtendedBlock b, ReplicaInfo temp) throws IOException { final long blockId = b.getBlockId(); final long expectedGs = b.getGenerationStamp(); final long visible = b.getNumBytes(); final long numBytes = temp.getNumBytes(); // move block files to the rbw directory BlockPoolSlice bpslice = getBlockPoolSlice(b.getBlockPoolId()); final File dest = FsDatasetImpl.moveBlockFiles(b.getLocalBlock(), temp, bpslice.getRbwDir()); // create RBW final LocalReplicaInPipeline rbw = new ReplicaBuilder(ReplicaState.RBW) .setBlockId(blockId) .setLength(numBytes) .setGenerationStamp(expectedGs) .setFsVolume(this) .setDirectoryToUse(dest.getParentFile()) .setWriterThread(Thread.currentThread()) .setBytesToReserve(0) .buildLocalReplicaInPipeline(); rbw.setBytesAcked(visible); // load last checksum and datalen final File destMeta = FsDatasetUtil.getMetaFile(dest, b.getGenerationStamp()); byte[] lastChunkChecksum = loadLastPartialChunkChecksum(dest, destMeta); rbw.setLastChecksumAndDataLen(numBytes, lastChunkChecksum); return rbw; }