private boolean readReplicasFromCache(ReplicaMap volumeMap, final RamDiskReplicaTracker lazyWriteReplicaMap) { ReplicaMap tmpReplicaMap = new ReplicaMap(new AutoCloseableLock()); File replicaFile = new File(currentDir, REPLICA_CACHE_FILE); tmpReplicaMap.replicas(bpid).iterator(); iter.hasNext(); ) { ReplicaInfo info = iter.next(); volumeMap.add(bpid, info);
final FsVolumeImpl v; try (AutoCloseableLock lock = datasetLock.acquire()) { final ReplicaInfo info = volumeMap.get(bpid, invalidBlks[i]); if (info == null) { ReplicaInfo infoByBlockId = volumeMap.get(bpid, invalidBlks[i].getBlockId()); if (infoByBlockId == null) { + " is not backed by a local file"); removing = volumeMap.remove(bpid, invalidBlks[i]); addDeletingBlock(bpid, removing.getBlockId()); if (LOG.isDebugEnabled()) {
void stopAllDataxceiverThreads(FsVolumeImpl volume) { try (AutoCloseableLock lock = datasetLock.acquire()) { for (String blockPoolId : volumeMap.getBlockPoolList()) { Collection<ReplicaInfo> replicas = volumeMap.replicas(blockPoolId); for (ReplicaInfo replicaInfo : replicas) { if ((replicaInfo.getState() == ReplicaState.TEMPORARY || replicaInfo.getState() == ReplicaState.RBW) && replicaInfo.getVolume().equals(volume)) { ReplicaInPipeline replicaInPipeline = (ReplicaInPipeline) replicaInfo; replicaInPipeline.interruptThread(); } } } } } }
/** * Get the meta information of the replica that matches both block id * and generation stamp * @param bpid block pool id * @param block block with its id as the key * @return the replica's meta information * @throws IllegalArgumentException if the input block or block pool is null */ ReplicaInfo get(String bpid, Block block) { checkBlockPool(bpid); checkBlock(block); ReplicaInfo replicaInfo = get(bpid, block.getBlockId()); if (replicaInfo != null && block.getGenerationStamp() == replicaInfo.getGenerationStamp()) { return replicaInfo; } return null; }
ProvidedBlockPoolSlice(String bpid, ProvidedVolumeImpl volume, Configuration conf) { this.providedVolume = volume; bpVolumeMap = new ReplicaMap(new AutoCloseableLock()); Class<? extends BlockAliasMap> fmt = conf.getClass(DFSConfigKeys.DFS_PROVIDED_ALIASMAP_CLASS, TextFileRegionAliasMap.class, BlockAliasMap.class); aliasMap = ReflectionUtils.newInstance(fmt, conf); this.conf = conf; this.bpid = bpid; this.df = new ProvidedVolumeDF(); bpVolumeMap.initBlockPool(bpid); this.numRetries = conf.getInt(DFS_PROVIDED_ALIASMAP_LOAD_RETRIES, 0); LOG.info("Created alias map using class: " + aliasMap.getClass()); }
/** * Gets a list of references to the finalized blocks for the given block pool. * <p> * Callers of this function should call * {@link FsDatasetSpi#acquireDatasetLock} to avoid blocks' status being * changed during list iteration. * </p> * @return a list of references to the finalized blocks for the given block * pool. */ @Override public List<ReplicaInfo> getFinalizedBlocks(String bpid) { try (AutoCloseableLock lock = datasetLock.acquire()) { final List<ReplicaInfo> finalized = new ArrayList<ReplicaInfo>( volumeMap.size(bpid)); for (ReplicaInfo b : volumeMap.replicas(bpid)) { if (b.getState() == ReplicaState.FINALIZED) { finalized.add(b); } } return finalized; } }
ReplicaInfo memBlockInfo; try (AutoCloseableLock lock = datasetLock.acquire()) { memBlockInfo = volumeMap.get(bpid, blockId); if (memBlockInfo != null && memBlockInfo.getState() != ReplicaState.FINALIZED) { .build(); volumeMap.add(bpid, diskBlockInfo); LOG.warn("Added missing block to memory " + diskBlockInfo); } else { volumeMap.remove(bpid, blockId); LOG.warn("Deleting missing provided block " + memBlockInfo); volumeMap.remove(bpid, blockId); if (vol.isTransientStorage()) { ramDiskReplicaTracker.discardReplica(bpid, blockId, true); .setDirectoryToUse(diskFile.getParentFile()) .build(); volumeMap.add(bpid, diskBlockInfo); if (vol.isTransientStorage()) { long lockedBytesReserved =
/** * Add a replica's meta information into the map * * @param bpid block pool id * @param replicaInfo a replica's meta information * @return previous meta information of the replica * @throws IllegalArgumentException if the input parameter is null */ ReplicaInfo add(String bpid, ReplicaInfo replicaInfo) { checkBlockPool(bpid); checkBlock(replicaInfo); try (AutoCloseableLock l = lock.acquire()) { FoldedTreeSet<ReplicaInfo> set = map.get(bpid); if (set == null) { // Add an entry for block pool if it does not exist already set = new FoldedTreeSet<>(); map.put(bpid, set); } return set.addOrReplace(replicaInfo); } }
/** * @deprecated use {@link #fetchReplicaInfo(String, long)} instead. */ @Override // FsDatasetSpi @Deprecated public ReplicaInfo getReplica(String bpid, long blockId) { return volumeMap.get(bpid, blockId); }
(replicaToDelete != replica1) ? replica1 : replica2; volumeMap.add(bpid, replicaToKeep); if (replicaToDelete != null) { deleteReplica(replicaToDelete);
private void addVolume(Collection<StorageLocation> dataLocations, Storage.StorageDirectory sd) throws IOException { final File dir = sd.getCurrentDir(); final StorageType storageType = getStorageTypeFromLocations(dataLocations, sd.getRoot()); // If IOException raises from FsVolumeImpl() or getVolumeMap(), there is // nothing needed to be rolled back to make various data structures, e.g., // storageMap and asyncDiskService, consistent. FsVolumeImpl fsVolume = new FsVolumeImpl( this, sd.getStorageUuid(), dir, this.conf, storageType); FsVolumeReference ref = fsVolume.obtainReference(); ReplicaMap tempVolumeMap = new ReplicaMap(this); fsVolume.getVolumeMap(tempVolumeMap, ramDiskReplicaTracker); synchronized (this) { volumeMap.addAll(tempVolumeMap); storageMap.put(sd.getStorageUuid(), new DatanodeStorage(sd.getStorageUuid(), DatanodeStorage.State.NORMAL, storageType)); asyncDiskService.addVolume(sd.getCurrentDir()); volumes.addVolume(ref); } LOG.info("Added volume - " + dir + ", StorageType: " + storageType); }
final long gs = 7777L; final long length = 22L; final ReplicaMap map = new ReplicaMap(this); String bpid = "BP-TEST"; final Block[] blocks = new Block[5]; for(int i = 0; i < blocks.length; i++) { blocks[i] = new Block(firstblockid + i, length, gs); map.add(bpid, createReplicaInfo(blocks[i])); final ReplicaInfo originalInfo = map.get(bpid, b); assertEquals(originalInfo, recoveryInfo); final ReplicaUnderRecovery updatedInfo = (ReplicaUnderRecovery)map.get(bpid, b); Assert.assertEquals(originalInfo.getBlockId(), updatedInfo.getBlockId()); Assert.assertEquals(recoveryid, updatedInfo.getRecoveryID()); assertEquals(originalInfo, recoveryInfo2); final ReplicaUnderRecovery updatedInfo2 = (ReplicaUnderRecovery)map.get(bpid, b); Assert.assertEquals(originalInfo.getBlockId(), updatedInfo2.getBlockId()); Assert.assertEquals(recoveryid2, updatedInfo2.getRecoveryID());
void cleanUpBlockPool(String bpid) { checkBlockPool(bpid); try (AutoCloseableLock l = lock.acquire()) { map.remove(bpid); } }
public boolean isEmpty() { return bpVolumeMap.replicas(bpid).size() == 0; }
final FsVolumeImpl fsVolume = createFsVolume(sd.getStorageUuid(), sd, location); final ReplicaMap tempVolumeMap = new ReplicaMap(new AutoCloseableLock()); ArrayList<IOException> exceptions = Lists.newArrayList();
@Test public void testRemove() { // Test 1: null argument throws invalid argument exception try { map.remove(bpid, null); fail("Expected exception not thrown"); } catch (IllegalArgumentException expected) { } // Test 2: remove failure - generation stamp mismatch Block b = new Block(block); b.setGenerationStamp(0); assertNull(map.remove(bpid, b)); // Test 3: remove failure - blockID mismatch b.setGenerationStamp(block.getGenerationStamp()); b.setBlockId(0); assertNull(map.remove(bpid, b)); // Test 4: remove success assertNotNull(map.remove(bpid, block)); // Test 5: remove failure - invalid blockID assertNull(map.remove(bpid, 0)); // Test 6: remove success map.add(bpid, new FinalizedReplica(block, null, null)); assertNotNull(map.remove(bpid, block.getBlockId())); } }
for (String bpid : dataset.volumeMap.getBlockPoolList()) { totalNumReplicas += dataset.volumeMap.size(bpid);
/** * Activate a volume to serve requests. * @throws IOException if the storage UUID already exists. */ private void activateVolume( ReplicaMap replicaMap, Storage.StorageDirectory sd, StorageType storageType, FsVolumeReference ref) throws IOException { try (AutoCloseableLock lock = datasetLock.acquire()) { DatanodeStorage dnStorage = storageMap.get(sd.getStorageUuid()); if (dnStorage != null) { final String errorMsg = String.format( "Found duplicated storage UUID: %s in %s.", sd.getStorageUuid(), sd.getVersionFile()); LOG.error(errorMsg); throw new IOException(errorMsg); } volumeMap.addAll(replicaMap); storageMap.put(sd.getStorageUuid(), new DatanodeStorage(sd.getStorageUuid(), DatanodeStorage.State.NORMAL, storageType)); asyncDiskService.addVolume((FsVolumeImpl) ref.getVolume()); volumes.addVolume(ref); } }
static ReplicaRecoveryInfo initReplicaRecoveryImpl(String bpid, ReplicaMap map, Block block, long recoveryId) throws IOException, MustStopExistingWriter { final ReplicaInfo replica = map.get(bpid, block.getBlockId()); LOG.info("initReplicaRecovery: " + block + ", recoveryId=" + recoveryId + ", replica=" + replica); rur = new ReplicaBuilder(ReplicaState.RUR) .from(replica).setRecoveryId(recoveryId).build(); map.add(bpid, rur); LOG.info("initReplicaRecovery: changing replica state for " + block + " from " + replica.getState()