@VisibleForTesting static ReplicaInfo selectReplicaToDelete(final ReplicaInfo replica1, final ReplicaInfo replica2) { ReplicaInfo replicaToKeep; ReplicaInfo replicaToDelete; // it's the same block so don't ever delete it, even if GS or size // differs. caller should keep the one it just discovered on disk if (replica1.getBlockURI().equals(replica2.getBlockURI())) { return null; } if (replica1.getGenerationStamp() != replica2.getGenerationStamp()) { replicaToKeep = replica1.getGenerationStamp() > replica2.getGenerationStamp() ? replica1 : replica2; } else if (replica1.getNumBytes() != replica2.getNumBytes()) { replicaToKeep = replica1.getNumBytes() > replica2.getNumBytes() ? replica1 : replica2; } else if (replica1.getVolume().isTransientStorage() && !replica2.getVolume().isTransientStorage()) { replicaToKeep = replica2; } else { replicaToKeep = replica1; } replicaToDelete = (replicaToKeep == replica1) ? replica2 : replica1; if (LOG.isDebugEnabled()) { LOG.debug("resolveDuplicateReplicas decide to keep " + replicaToKeep + ". Will try to delete " + replicaToDelete); } return replicaToDelete; }
private boolean moveFiles() { if (trashDirectory == null) { LOG.error("Trash dir for replica " + replicaToDelete + " is null"); return false; } File trashDirFile = new File(trashDirectory); try { volume.getFileIoProvider().mkdirsWithExistsCheck( volume, trashDirFile); } catch (IOException e) { return false; } if (LOG.isDebugEnabled()) { LOG.debug("Moving files " + replicaToDelete.getBlockURI() + " and " + replicaToDelete.getMetadataURI() + " to trash."); } final String blockName = replicaToDelete.getBlockName(); final long genstamp = replicaToDelete.getGenerationStamp(); File newBlockFile = new File(trashDirectory, blockName); File newMetaFile = new File(trashDirectory, DatanodeUtil.getMetaName(blockName, genstamp)); try { return (replicaToDelete.renameData(newBlockFile.toURI()) && replicaToDelete.renameMeta(newMetaFile.toURI())); } catch (IOException e) { LOG.error("Error moving files to trash: " + replicaToDelete, e); } return false; }
@Override // FsDatasetSpi public Block getStoredBlock(String bpid, long blkid) throws IOException { try (AutoCloseableLock lock = datasetLock.acquire()) { ReplicaInfo r = volumeMap.get(bpid, blkid); if (r == null) { return null; } return new Block(blkid, r.getBytesOnDisk(), r.getGenerationStamp()); } }
/** * Get the meta information of the replica that matches both block id * and generation stamp * @param bpid block pool id * @param block block with its id as the key * @return the replica's meta information * @throws IllegalArgumentException if the input block or block pool is null */ ReplicaInfo get(String bpid, Block block) { checkBlockPool(bpid); checkBlock(block); ReplicaInfo replicaInfo = get(bpid, block.getBlockId()); if (replicaInfo != null && block.getGenerationStamp() == replicaInfo.getGenerationStamp()) { return replicaInfo; } return null; }
genstamp = info.getGenerationStamp(); volumeExecutor = volume.getCacheExecutor();
/** * Move a persisted replica from lazypersist directory to a subdirectory * under finalized. */ ReplicaInfo activateSavedReplica(ReplicaInfo replicaInfo, RamDiskReplica replicaState) throws IOException { File metaFile = replicaState.getSavedMetaFile(); File blockFile = replicaState.getSavedBlockFile(); final long blockId = replicaInfo.getBlockId(); final File blockDir = DatanodeUtil.idToBlockDir(finalizedDir, blockId); final File targetBlockFile = new File(blockDir, blockFile.getName()); final File targetMetaFile = new File(blockDir, metaFile.getName()); fileIoProvider.moveFile(volume, blockFile, targetBlockFile); FsDatasetImpl.LOG.info("Moved " + blockFile + " to " + targetBlockFile); fileIoProvider.moveFile(volume, metaFile, targetMetaFile); FsDatasetImpl.LOG.info("Moved " + metaFile + " to " + targetMetaFile); ReplicaInfo newReplicaInfo = new ReplicaBuilder(ReplicaState.FINALIZED) .setBlockId(blockId) .setLength(replicaInfo.getBytesOnDisk()) .setGenerationStamp(replicaInfo.getGenerationStamp()) .setFsVolume(replicaState.getLazyPersistVolume()) .setDirectoryToUse(targetBlockFile.getParentFile()) .build(); return newReplicaInfo; }
/** * Remove the replica's meta information from the map that matches * the input block's id and generation stamp * @param bpid block pool id * @param block block with its id as the key * @return the removed replica's meta information * @throws IllegalArgumentException if the input block is null */ ReplicaInfo remove(String bpid, Block block) { checkBlockPool(bpid); checkBlock(block); try (AutoCloseableLock l = lock.acquire()) { FoldedTreeSet<ReplicaInfo> set = map.get(bpid); if (set != null) { ReplicaInfo replicaInfo = set.get(block.getBlockId(), LONG_AND_BLOCK_COMPARATOR); if (replicaInfo != null && block.getGenerationStamp() == replicaInfo.getGenerationStamp()) { return set.removeAndGet(replicaInfo); } } } return null; }
public ReplicaInfo moveBlockToTmpLocation(ExtendedBlock block, ReplicaInfo replicaInfo, int smallBufferSize, Configuration conf) throws IOException { File[] blockFiles = FsDatasetImpl.copyBlockFiles(block.getBlockId(), block.getGenerationStamp(), replicaInfo, getTmpDir(block.getBlockPoolId()), replicaInfo.isOnTransientStorage(), smallBufferSize, conf); ReplicaInfo newReplicaInfo = new ReplicaBuilder(ReplicaState.TEMPORARY) .setBlockId(replicaInfo.getBlockId()) .setGenerationStamp(replicaInfo.getGenerationStamp()) .setFsVolume(this) .setDirectoryToUse(blockFiles[0].getParentFile()) .setBytesToReserve(0) .build(); newReplicaInfo.setNumBytes(blockFiles[1].length()); return newReplicaInfo; }
if (replica.getGenerationStamp() < block.getGenerationStamp()) { throw new IOException( "replica.getGenerationStamp() < block.getGenerationStamp(), block=" if (replica.getGenerationStamp() >= recoveryId) { throw new IOException("THIS IS NOT SUPPOSED TO HAPPEN:" + " replica.getGenerationStamp() >= recoveryId = " + recoveryId
+ " from " + memBlockInfo.getGenerationStamp() + " to " + diskGS); memBlockInfo.setGenerationStamp(diskGS); if (memBlockInfo.getGenerationStamp() != diskGS) { File memMetaFile = FsDatasetUtil.getMetaFile(diskFile, memBlockInfo.getGenerationStamp()); if (fileIoProvider.exists(vol, memMetaFile)) { String warningPrefix = "Metadata file in memory " + " from " + memBlockInfo.getGenerationStamp() + " to " + gs);
} else if (info.getGenStamp() != memBlock.getGenerationStamp() || info.getBlockLength() != memBlock.getNumBytes()) {
if (!copyTruncate) { assert finalized.getBlockId() == oldBlock.getBlockId() && finalized.getGenerationStamp() == recoveryId && finalized.getNumBytes() == newlength : "Replica information mismatched: oldBlock=" + oldBlock } else { assert finalized.getBlockId() == oldBlock.getBlockId() && finalized.getGenerationStamp() == oldBlock.getGenerationStamp() && finalized.getNumBytes() == oldBlock.getNumBytes() : "Finalized and old information mismatched: oldBlock=" + oldBlock
try (AutoCloseableLock lock = datasetLock.acquire()) { if (volumeMap.get(bpid, replicaInfo.getBlockId()).getGenerationStamp() > replicaInfo.getGenerationStamp()) { throw new IOException("Generation Stamp should be monotonically " + "increased.");
if (temp.getGenerationStamp() != expectedGs) { throw new ReplicaAlreadyExistsException( "temp.getGenerationStamp() != expectedGs = " + expectedGs
replicaInfo.getGenerationStamp(), block.getCreationTime(), replicaInfo, targetReference);
long replicaGenerationStamp = replicaInfo.getGenerationStamp(); if (replicaGenerationStamp < b.getGenerationStamp() || replicaGenerationStamp > newGS) {
if (((currentReplicaInfo.getGenerationStamp() >= b .getGenerationStamp()) || (!isTransfer && !isInPipeline)) && !isReplicaProvided(currentReplicaInfo)) {
public ReplicaRecoveryInfo createInfo() { return new ReplicaRecoveryInfo(original.getBlockId(), original.getBytesOnDisk(), original.getGenerationStamp(), original.getState()); } }
private void verifyGenStamp(long blockId, long genStamp) { final ReplicaInfo memBlock; memBlock = FsDatasetTestUtil.fetchReplicaInfo(fds, bpid, blockId); assertNotNull(memBlock); assertEquals(genStamp, memBlock.getGenerationStamp()); }
private static void assertEquals(ReplicaInfo originalInfo, ReplicaRecoveryInfo recoveryInfo) { Assert.assertEquals(originalInfo.getBlockId(), recoveryInfo.getBlockId()); Assert.assertEquals(originalInfo.getGenerationStamp(), recoveryInfo.getGenerationStamp()); Assert.assertEquals(originalInfo.getBytesOnDisk(), recoveryInfo.getNumBytes()); Assert.assertEquals(originalInfo.getState(), recoveryInfo.getOriginalReplicaState()); }