@Override public boolean isOnTransientStorage() { return volume.isTransientStorage(); }
/** @return is writing to a transient storage? */ public boolean isTransientStorage() { return volume.isTransientStorage(); }
@VisibleForTesting static ReplicaInfo selectReplicaToDelete(final ReplicaInfo replica1, final ReplicaInfo replica2) { ReplicaInfo replicaToKeep; ReplicaInfo replicaToDelete; // it's the same block so don't ever delete it, even if GS or size // differs. caller should keep the one it just discovered on disk if (replica1.getBlockURI().equals(replica2.getBlockURI())) { return null; } if (replica1.getGenerationStamp() != replica2.getGenerationStamp()) { replicaToKeep = replica1.getGenerationStamp() > replica2.getGenerationStamp() ? replica1 : replica2; } else if (replica1.getNumBytes() != replica2.getNumBytes()) { replicaToKeep = replica1.getNumBytes() > replica2.getNumBytes() ? replica1 : replica2; } else if (replica1.getVolume().isTransientStorage() && !replica2.getVolume().isTransientStorage()) { replicaToKeep = replica2; } else { replicaToKeep = replica1; } replicaToDelete = (replicaToKeep == replica1) ? replica2 : replica1; if (LOG.isDebugEnabled()) { LOG.debug("resolveDuplicateReplicas decide to keep " + replicaToKeep + ". Will try to delete " + replicaToDelete); } return replicaToDelete; }
@Override // FsDatasetSpi public InputStream getBlockInputStream(ExtendedBlock b, long seekOffset) throws IOException { ReplicaInfo info; synchronized(this) { info = volumeMap.get(b.getBlockPoolId(), b.getLocalBlock()); } if (info != null && info.getVolume().isTransientStorage()) { ramDiskReplicaTracker.touch(b.getBlockPoolId(), b.getBlockId()); datanode.getMetrics().incrRamDiskBlocksReadHits(); } if(info != null && info.blockDataExists()) { return info.getDataInputStream(seekOffset); } else { throw new IOException("No data exists for block " + b); } }
@Override public boolean isOnTransientStorage() { return volume.isTransientStorage(); } }
@Override public boolean isOnTransientStorage() { return volume.isTransientStorage(); } }
if (source.isTransientStorage() || dest.isTransientStorage()) { final String errMsg = "Disk Balancer - Unable to support " + "transient storage type.";
/** * Remove the temporary block file (if any) */ @Override // FsDatasetSpi public void unfinalizeBlock(ExtendedBlock b) throws IOException { try (AutoCloseableLock lock = datasetLock.acquire()) { ReplicaInfo replicaInfo = volumeMap.get(b.getBlockPoolId(), b.getLocalBlock()); if (replicaInfo != null && replicaInfo.getState() == ReplicaState.TEMPORARY) { // remove from volumeMap volumeMap.remove(b.getBlockPoolId(), b.getLocalBlock()); // delete the on-disk temp file if (delBlockFromDisk(replicaInfo)) { LOG.warn("Block " + b + " unfinalized and removed. "); } if (replicaInfo.getVolume().isTransientStorage()) { ramDiskReplicaTracker.discardReplica(b.getBlockPoolId(), b.getBlockId(), true); } } } }
if (vol.isTransientStorage()) { ramDiskReplicaTracker.discardReplica(bpid, blockId, true); .build(); volumeMap.add(bpid, diskBlockInfo); if (vol.isTransientStorage()) { long lockedBytesReserved = cacheManager.reserve(diskBlockInfo.getNumBytes()) > 0 ?
replicaInfo.getVolume().isTransientStorage()) {
if (newReplica.getVolume().isTransientStorage()) { lazyWriteReplicaMap.addReplica(bpid, blockId, (FsVolumeImpl) newReplica.getVolume(), 0);
replicaInfo = getReplicaInfo(replicaState.getBlockPoolId(), replicaState.getBlockId()); Preconditions.checkState(replicaInfo.getVolume().isTransientStorage()); ramDiskReplicaTracker.discardReplica(replicaState.getBlockPoolId(), replicaState.getBlockId(), false);
/** * Get the list of finalized blocks from in-memory blockmap for a block pool. */ @Override public synchronized List<FinalizedReplica> getFinalizedBlocksOnPersistentStorage(String bpid) { ArrayList<FinalizedReplica> finalized = new ArrayList<FinalizedReplica>(volumeMap.size(bpid)); for (ReplicaInfo b : volumeMap.replicas(bpid)) { if(!b.getVolume().isTransientStorage() && b.getState() == ReplicaState.FINALIZED) { finalized.add(new FinalizedReplica((FinalizedReplica)b)); } } return finalized; }
/** * Get the list of finalized blocks from in-memory blockmap for a block pool. */ @Override public synchronized List<FinalizedReplica> getFinalizedBlocksOnPersistentStorage(String bpid) { ArrayList<FinalizedReplica> finalized = new ArrayList<FinalizedReplica>(volumeMap.size(bpid)); for (ReplicaInfo b : volumeMap.replicas(bpid)) { if(!b.getVolume().isTransientStorage() && b.getState() == ReplicaState.FINALIZED) { finalized.add(new FinalizedReplica((FinalizedReplica)b)); } } return finalized; }
private void verifyStorageType(long blockId, boolean expectTransient) { final ReplicaInfo memBlock; memBlock = FsDatasetTestUtil.fetchReplicaInfo(fds, bpid, blockId); assertNotNull(memBlock); assertThat(memBlock.getVolume().isTransientStorage(), is(expectTransient)); }
/** * Turn the block identifier into a filename * @param bpid Block pool Id * @param blockId a block's id * @return on disk data file path; null if the replica does not exist */ File getFile(final String bpid, final long blockId, boolean touch) { ReplicaInfo info = volumeMap.get(bpid, blockId); if (info != null) { if (touch && info.getVolume().isTransientStorage()) { ramDiskReplicaTracker.touch(bpid, blockId); datanode.getMetrics().incrRamDiskBlocksReadHits(); } return info.getBlockFile(); } return null; }
/** * Turn the block identifier into a filename * @param bpid Block pool Id * @param blockId a block's id * @return on disk data file path; null if the replica does not exist */ File getFile(final String bpid, final long blockId, boolean touch) { ReplicaInfo info = volumeMap.get(bpid, blockId); if (info != null) { if (touch && info.getVolume().isTransientStorage()) { ramDiskReplicaTracker.touch(bpid, blockId); datanode.getMetrics().incrRamDiskBlocksReadHits(); } return info.getBlockFile(); } return null; }
/** * Remove the temporary block file (if any) */ @Override // FsDatasetSpi public synchronized void unfinalizeBlock(ExtendedBlock b) throws IOException { ReplicaInfo replicaInfo = volumeMap.get(b.getBlockPoolId(), b.getLocalBlock()); if (replicaInfo != null && replicaInfo.getState() == ReplicaState.TEMPORARY) { // remove from volumeMap volumeMap.remove(b.getBlockPoolId(), b.getLocalBlock()); // delete the on-disk temp file if (delBlockFromDisk(replicaInfo.getBlockFile(), replicaInfo.getMetaFile(), b.getLocalBlock())) { LOG.warn("Block " + b + " unfinalized and removed. " ); } if (replicaInfo.getVolume().isTransientStorage()) { ramDiskReplicaTracker.discardReplica(b.getBlockPoolId(), b.getBlockId(), true); } } }
protected final boolean verifyDeletedBlocks(LocatedBlocks locatedBlocks) throws IOException, InterruptedException { LOG.info("Verifying replica has no saved copy after deletion."); triggerBlockReport(); while( DataNodeTestUtils.getPendingAsyncDeletions(cluster.getDataNodes().get(0)) > 0L){ Thread.sleep(1000); } final String bpid = cluster.getNamesystem().getBlockPoolId(); List<? extends FsVolumeSpi> volumes = cluster.getDataNodes().get(0).getFSDataset().getVolumes(); // Make sure deleted replica does not have a copy on either finalized dir of // transient volume or finalized dir of non-transient volume for (FsVolumeSpi v : volumes) { FsVolumeImpl volume = (FsVolumeImpl) v; File targetDir = (v.isTransientStorage()) ? volume.getBlockPoolSlice(bpid).getFinalizedDir() : volume.getBlockPoolSlice(bpid).getLazypersistDir(); if (verifyBlockDeletedFromDir(targetDir, locatedBlocks) == false) { return false; } } return true; }
/** * Remove the temporary block file (if any) */ @Override // FsDatasetSpi public synchronized void unfinalizeBlock(ExtendedBlock b) throws IOException { ReplicaInfo replicaInfo = volumeMap.get(b.getBlockPoolId(), b.getLocalBlock()); if (replicaInfo != null && replicaInfo.getState() == ReplicaState.TEMPORARY) { // remove from volumeMap volumeMap.remove(b.getBlockPoolId(), b.getLocalBlock()); // delete the on-disk temp file if (delBlockFromDisk(replicaInfo.getBlockFile(), replicaInfo.getMetaFile(), b.getLocalBlock())) { LOG.warn("Block " + b + " unfinalized and removed. " ); } if (replicaInfo.getVolume().isTransientStorage()) { ramDiskReplicaTracker.discardReplica(b.getBlockPoolId(), b.getBlockId(), true); } } }