@VisibleForTesting public File getFinalizedDir(String bpid) throws IOException { return getBlockPoolSlice(bpid).getFinalizedDir(); }
protected File getLazyPersistDir(String bpid) throws IOException { return getBlockPoolSlice(bpid).getLazypersistDir(); }
selectReplicaToDelete(replica1, replica2); final ReplicaInfo replicaToKeep = (replicaToDelete != replica1) ? replica1 : replica2; deleteReplica(replicaToDelete);
private void decDfsUsedAndNumBlocks(String bpid, long value, boolean blockFileDeleted) { // BlockPoolSlice map is thread safe, and update the space used or // number of blocks are atomic operations, so it doesn't require to // hold the dataset lock. BlockPoolSlice bp = bpSlices.get(bpid); if (bp != null) { bp.decDfsUsed(value); if (blockFileDeleted) { bp.decrNumBlocks(); } } }
for (File file : files) { if (file.isDirectory()) { addToReplicasMap(volumeMap, file, lazyWriteReplicaMap, isFinalized); file = recoverTempUnlinkedBlock(file); if (file == null) { // the original block still exists, so we cover it validateIntegrityAndSetLength(file, genStamp), genStamp, volume, file.getParentFile(), null, 0); loadRwr = false; validateIntegrityAndSetLength(file, genStamp), genStamp, volume, file.getParentFile()); newReplica = resolveDuplicateReplicas(newReplica, oldReplica, volumeMap);
void getVolumeMap(ReplicaMap volumeMap, final RamDiskReplicaTracker lazyWriteReplicaMap) throws IOException { // Recover lazy persist replicas, they will be added to the volumeMap // when we scan the finalized directory. if (lazypersistDir.exists()) { int numRecovered = moveLazyPersistReplicasToFinalized(lazypersistDir); FsDatasetImpl.LOG.info( "Recovered " + numRecovered + " replicas from " + lazypersistDir); } // add finalized replicas addToReplicasMap(volumeMap, finalizedDir, lazyWriteReplicaMap, true); // add rbw replicas addToReplicasMap(volumeMap, rbwDir, lazyWriteReplicaMap, false); }
/** * RBW files. They get moved to the finalized block directory when * the block is finalized. */ File createRbwFile(String bpid, Block b) throws IOException { checkReference(); reserveSpaceForReplica(b.getNumBytes()); try { return getBlockPoolSlice(bpid).createRbwFile(b); } catch (IOException exception) { releaseReservedSpace(b.getNumBytes()); throw exception; } }
protected final boolean verifyDeletedBlocks(LocatedBlocks locatedBlocks) throws IOException, InterruptedException { LOG.info("Verifying replica has no saved copy after deletion."); triggerBlockReport(); while( DataNodeTestUtils.getPendingAsyncDeletions(cluster.getDataNodes().get(0)) > 0L){ Thread.sleep(1000); } final String bpid = cluster.getNamesystem().getBlockPoolId(); List<? extends FsVolumeSpi> volumes = cluster.getDataNodes().get(0).getFSDataset().getVolumes(); // Make sure deleted replica does not have a copy on either finalized dir of // transient volume or finalized dir of non-transient volume for (FsVolumeSpi v : volumes) { FsVolumeImpl volume = (FsVolumeImpl) v; File targetDir = (v.isTransientStorage()) ? volume.getBlockPoolSlice(bpid).getFinalizedDir() : volume.getBlockPoolSlice(bpid).getLazypersistDir(); if (verifyBlockDeletedFromDir(targetDir, locatedBlocks) == false) { return false; } } return true; }
public ReplicaInfo activateSavedReplica(String bpid, ReplicaInfo replicaInfo, RamDiskReplica replicaState) throws IOException { return getBlockPoolSlice(bpid).activateSavedReplica(replicaInfo, replicaState); } }
@Override public VolumeCheckResult check(VolumeCheckContext ignored) throws DiskErrorException { // TODO:FEDERATION valid synchronization for(BlockPoolSlice s : bpSlices.values()) { s.checkDirs(); } return VolumeCheckResult.HEALTHY; }
void addBlockPool(String bpid, Configuration c, Timer timer) throws IOException { File bpdir = new File(currentDir, bpid); BlockPoolSlice bp; if (timer == null) { bp = new BlockPoolSlice(bpid, this, bpdir, c, new Timer()); } else { bp = new BlockPoolSlice(bpid, this, bpdir, c, timer); } bpSlices.put(bpid, bp); }
/** * Temporary files. They get moved to the finalized block directory when * the block is finalized. */ File createTmpFile(String bpid, Block b) throws IOException { checkReference(); reserveSpaceForReplica(b.getNumBytes()); try { return getBlockPoolSlice(bpid).createTmpFile(b); } catch (IOException exception) { releaseReservedSpace(b.getNumBytes()); throw exception; } }
void decDfsUsed(String bpid, long value) { synchronized(dataset) { BlockPoolSlice bp = bpSlices.get(bpid); if (bp != null) { bp.decDfsUsed(value); } } }
@Override protected void compute() { try { addToReplicasMap(volumeMap, dir, lazyWriteReplicaMap, isFinalized, exceptions, subTaskQueue); } catch (IOException e) { LOG.warn("Caught exception while adding replicas from " + volume + " in subtask. Will throw later.", e); exceptions.add(e); } } }
@Test public void testDuplicateReplicaResolution() throws IOException { FsVolumeImpl fsv1 = Mockito.mock(FsVolumeImpl.class); FsVolumeImpl fsv2 = Mockito.mock(FsVolumeImpl.class); File f1 = new File("d1/block"); File f2 = new File("d2/block"); ReplicaInfo replicaOlder = new FinalizedReplica(1,1,1,fsv1,f1); ReplicaInfo replica = new FinalizedReplica(1,2,2,fsv1,f1); ReplicaInfo replicaSame = new FinalizedReplica(1,2,2,fsv1,f1); ReplicaInfo replicaNewer = new FinalizedReplica(1,3,3,fsv1,f1); ReplicaInfo replicaOtherOlder = new FinalizedReplica(1,1,1,fsv2,f2); ReplicaInfo replicaOtherSame = new FinalizedReplica(1,2,2,fsv2,f2); ReplicaInfo replicaOtherNewer = new FinalizedReplica(1,3,3,fsv2,f2); // equivalent path so don't remove either assertNull(BlockPoolSlice.selectReplicaToDelete(replicaSame, replica)); assertNull(BlockPoolSlice.selectReplicaToDelete(replicaOlder, replica)); assertNull(BlockPoolSlice.selectReplicaToDelete(replicaNewer, replica)); // keep latest found replica assertSame(replica, BlockPoolSlice.selectReplicaToDelete(replicaOtherSame, replica)); assertSame(replicaOtherOlder, BlockPoolSlice.selectReplicaToDelete(replicaOtherOlder, replica)); assertSame(replica, BlockPoolSlice.selectReplicaToDelete(replicaOtherNewer, replica)); }
for (File file : files) { if (file.isDirectory()) { addToReplicasMap(volumeMap, file, lazyWriteReplicaMap, isFinalized); file = recoverTempUnlinkedBlock(file); if (file == null) { // the original block still exists, so we cover it validateIntegrityAndSetLength(file, genStamp), genStamp, volume, file.getParentFile(), null, 0); loadRwr = false; validateIntegrityAndSetLength(file, genStamp), genStamp, volume, file.getParentFile()); newReplica = resolveDuplicateReplicas(newReplica, oldReplica, volumeMap);
void getVolumeMap(ReplicaMap volumeMap, final RamDiskReplicaTracker lazyWriteReplicaMap) throws IOException { // Recover lazy persist replicas, they will be added to the volumeMap // when we scan the finalized directory. if (lazypersistDir.exists()) { int numRecovered = moveLazyPersistReplicasToFinalized(lazypersistDir); FsDatasetImpl.LOG.info( "Recovered " + numRecovered + " replicas from " + lazypersistDir); } // add finalized replicas addToReplicasMap(volumeMap, finalizedDir, lazyWriteReplicaMap, true); // add rbw replicas addToReplicasMap(volumeMap, rbwDir, lazyWriteReplicaMap, false); }
/** * RBW files. They get moved to the finalized block directory when * the block is finalized. */ File createRbwFile(String bpid, Block b) throws IOException { checkReference(); reserveSpaceForRbw(b.getNumBytes()); try { return getBlockPoolSlice(bpid).createRbwFile(b); } catch (IOException exception) { releaseReservedSpace(b.getNumBytes()); throw exception; } }
File newBlockFile = bpSlice.activateSavedReplica( replicaInfo, replicaState.getSavedMetaFile(), replicaState.getSavedBlockFile());
void checkDirs() throws DiskErrorException { // TODO:FEDERATION valid synchronization for(BlockPoolSlice s : bpSlices.values()) { s.checkDirs(); } }