@VisibleForTesting static ReplicaInfo selectReplicaToDelete(final ReplicaInfo replica1, final ReplicaInfo replica2) { ReplicaInfo replicaToKeep; ReplicaInfo replicaToDelete; // it's the same block so don't ever delete it, even if GS or size // differs. caller should keep the one it just discovered on disk if (replica1.getBlockURI().equals(replica2.getBlockURI())) { return null; } if (replica1.getGenerationStamp() != replica2.getGenerationStamp()) { replicaToKeep = replica1.getGenerationStamp() > replica2.getGenerationStamp() ? replica1 : replica2; } else if (replica1.getNumBytes() != replica2.getNumBytes()) { replicaToKeep = replica1.getNumBytes() > replica2.getNumBytes() ? replica1 : replica2; } else if (replica1.getVolume().isTransientStorage() && !replica2.getVolume().isTransientStorage()) { replicaToKeep = replica2; } else { replicaToKeep = replica1; } replicaToDelete = (replicaToKeep == replica1) ? replica2 : replica1; if (LOG.isDebugEnabled()) { LOG.debug("resolveDuplicateReplicas decide to keep " + replicaToKeep + ". Will try to delete " + replicaToDelete); } return replicaToDelete; }
@Override //Object public String toString() { return getClass().getSimpleName() + ", " + super.toString() + ", " + getState() + "\n getNumBytes() = " + getNumBytes() + "\n getBytesOnDisk() = " + getBytesOnDisk() + "\n getVisibleLength()= " + getVisibleLength() + "\n getVolume() = " + getVolume() + "\n getBlockURI() = " + getBlockURI(); }
ReplicaNotFoundException.UNFINALIZED_REPLICA + b); if (replicaInfo.getNumBytes() != expectedBlockLen) { throw new IOException("Corrupted replica " + replicaInfo + " with a length of " + replicaInfo.getNumBytes() + " expected length is " + expectedBlockLen);
releaseLockedMemory( replicaInfo.getOriginalBytesReserved() - replicaInfo.getNumBytes(), false); ramDiskReplicaTracker.addReplica( bpid, replicaInfo.getBlockId(), v, replicaInfo.getNumBytes()); datanode.getMetrics().addRamDiskBytesWrite(replicaInfo.getNumBytes());
if (rur.getNumBytes() < newlength) { throw new IOException("rur.getNumBytes() < newlength = " + newlength + ", rur=" + rur); if (rur.getNumBytes() > newlength) { if(!copyOnTruncate) { rur.breakHardLinksIfNeeded();
public ReplicaInPipeline append(String bpid, ReplicaInfo replicaInfo, long newGS, long estimateBlockLen) throws IOException { long bytesReserved = estimateBlockLen - replicaInfo.getNumBytes(); if (getAvailable() < bytesReserved) { throw new DiskOutOfSpaceException("Insufficient space for appending to " LocalReplicaInPipeline newReplicaInfo = new ReplicaBuilder(ReplicaState.RBW) .setBlockId(replicaInfo.getBlockId()) .setLength(replicaInfo.getNumBytes()) .setGenerationStamp(newGS) .setFsVolume(this)
ReplicaNotFoundException.UNFINALIZED_REPLICA + block); if (replicaInfo.getNumBytes() != block.getNumBytes()) { throw new IOException("Corrupted replica " + replicaInfo + " with a length of " + replicaInfo.getNumBytes() + " expected length is " + block.getNumBytes());
public ReplicaInPipeline updateRURCopyOnTruncate(ReplicaInfo rur, String bpid, long newBlockId, long recoveryId, long newlength) throws IOException { rur.breakHardLinksIfNeeded(); File[] copiedReplicaFiles = copyReplicaWithNewBlockIdAndGS(rur, bpid, newBlockId, recoveryId); File blockFile = copiedReplicaFiles[1]; File metaFile = copiedReplicaFiles[0]; LocalReplica.truncateBlock(rur.getVolume(), blockFile, metaFile, rur.getNumBytes(), newlength, fileIoProvider); LocalReplicaInPipeline newReplicaInfo = new ReplicaBuilder(ReplicaState.RBW) .setBlockId(newBlockId) .setGenerationStamp(recoveryId) .setFsVolume(this) .setDirectoryToUse(blockFile.getParentFile()) .setBytesToReserve(newlength) .buildLocalReplicaInPipeline(); // In theory, this rbw replica needs to reload last chunk checksum, // but it is immediately converted to finalized state within the same lock, // so no need to update it. return newReplicaInfo; }
if (vol.isTransientStorage()) { long lockedBytesReserved = cacheManager.reserve(diskBlockInfo.getNumBytes()) > 0 ? diskBlockInfo.getNumBytes() : 0; ramDiskReplicaTracker.addReplica( bpid, blockId, (FsVolumeImpl) vol, lockedBytesReserved); if (memBlockInfo.getNumBytes() != memBlockInfo.getBlockDataLength()) { + memBlockInfo.getNumBytes() + " to " + memBlockInfo.getBlockDataLength()); memBlockInfo.setNumBytes(memBlockInfo.getBlockDataLength());
|| info.getBlockLength() != memBlock.getNumBytes()) {
assert finalized.getBlockId() == oldBlock.getBlockId() && finalized.getGenerationStamp() == recoveryId && finalized.getNumBytes() == newlength : "Replica information mismatched: oldBlock=" + oldBlock + ", recoveryId=" + recoveryId + ", newlength=" + newlength assert finalized.getBlockId() == oldBlock.getBlockId() && finalized.getGenerationStamp() == oldBlock.getGenerationStamp() && finalized.getNumBytes() == oldBlock.getNumBytes() : "Finalized and old information mismatched: oldBlock=" + oldBlock + ", genStamp=" + oldBlock.getGenerationStamp()
final long numBytes = temp.getNumBytes(); if (numBytes < visible) { throw new IOException(numBytes + " = numBytes < visible = "
StorageType.DEFAULT, null, replicaInfo.getNumBytes()); targetVolume = (FsVolumeImpl) targetReference.getVolume();
public ReplicaInPipeline convertTemporaryToRbw(ExtendedBlock b, ReplicaInfo temp) throws IOException { final long blockId = b.getBlockId(); final long expectedGs = b.getGenerationStamp(); final long visible = b.getNumBytes(); final long numBytes = temp.getNumBytes(); // move block files to the rbw directory BlockPoolSlice bpslice = getBlockPoolSlice(b.getBlockPoolId()); final File dest = FsDatasetImpl.moveBlockFiles(b.getLocalBlock(), temp, bpslice.getRbwDir()); // create RBW final LocalReplicaInPipeline rbw = new ReplicaBuilder(ReplicaState.RBW) .setBlockId(blockId) .setLength(numBytes) .setGenerationStamp(expectedGs) .setFsVolume(this) .setDirectoryToUse(dest.getParentFile()) .setWriterThread(Thread.currentThread()) .setBytesToReserve(0) .buildLocalReplicaInPipeline(); rbw.setBytesAcked(visible); // load last checksum and datalen final File destMeta = FsDatasetUtil.getMetaFile(dest, b.getGenerationStamp()); byte[] lastChunkChecksum = loadLastPartialChunkChecksum(dest, destMeta); rbw.setLastChecksumAndDataLen(numBytes, lastChunkChecksum); return rbw; }
long replicaLen = replicaInfo.getNumBytes(); if (replicaInfo.getState() == ReplicaState.RBW) { ReplicaInPipeline rbw = (ReplicaInPipeline) replicaInfo;
@VisibleForTesting static ReplicaInfo selectReplicaToDelete(final ReplicaInfo replica1, final ReplicaInfo replica2) { ReplicaInfo replicaToKeep; ReplicaInfo replicaToDelete; // it's the same block so don't ever delete it, even if GS or size // differs. caller should keep the one it just discovered on disk if (replica1.getBlockFile().equals(replica2.getBlockFile())) { return null; } if (replica1.getGenerationStamp() != replica2.getGenerationStamp()) { replicaToKeep = replica1.getGenerationStamp() > replica2.getGenerationStamp() ? replica1 : replica2; } else if (replica1.getNumBytes() != replica2.getNumBytes()) { replicaToKeep = replica1.getNumBytes() > replica2.getNumBytes() ? replica1 : replica2; } else if (replica1.getVolume().isTransientStorage() && !replica2.getVolume().isTransientStorage()) { replicaToKeep = replica2; } else { replicaToKeep = replica1; } replicaToDelete = (replicaToKeep == replica1) ? replica2 : replica1; if (LOG.isDebugEnabled()) { LOG.debug("resolveDuplicateReplicas decide to keep " + replicaToKeep + ". Will try to delete " + replicaToDelete); } return replicaToDelete; }
@Override //Object public String toString() { return getClass().getSimpleName() + ", " + super.toString() + ", " + getState() + "\n getNumBytes() = " + getNumBytes() + "\n getBytesOnDisk() = " + getBytesOnDisk() + "\n getVisibleLength()= " + getVisibleLength() + "\n getVolume() = " + getVolume() + "\n getBlockFile() = " + getBlockFile(); }
@Override //Object public String toString() { return getClass().getSimpleName() + ", " + super.toString() + ", " + getState() + "\n getNumBytes() = " + getNumBytes() + "\n getBytesOnDisk() = " + getBytesOnDisk() + "\n getVisibleLength()= " + getVisibleLength() + "\n getVolume() = " + getVolume() + "\n getBlockFile() = " + getBlockFile(); }
private void verifyAddition(long blockId, long genStamp, long size) { final ReplicaInfo replicainfo; replicainfo = fds.fetchReplicaInfo(blockId); assertNotNull(replicainfo); // Added block has the same file as the one created by the test File file = new File(getBlockFile(blockId)); assertEquals(file.getName(), fds.findBlockFile(blockId).getName()); // Generation stamp is same as that of created file assertEquals(genStamp, replicainfo.getGenerationStamp()); // File size matches assertEquals(size, replicainfo.getNumBytes()); }
private void verifyAddition(long blockId, long genStamp, long size) { final ReplicaInfo replicainfo; replicainfo = FsDatasetTestUtil.fetchReplicaInfo(fds, bpid, blockId); assertNotNull(replicainfo); // Added block has the same file as the one created by the test File file = new File(getBlockFile(blockId)); assertEquals(file.getName(), FsDatasetTestUtil.getFile(fds, bpid, blockId).getName()); // Generation stamp is same as that of created file assertEquals(genStamp, replicainfo.getGenerationStamp()); // File size matches assertEquals(size, replicainfo.getNumBytes()); }