FsVolumeImpl build() throws IOException { if (sd.getStorageLocation().getStorageType() == StorageType.PROVIDED) { return new ProvidedVolumeImpl(dataset, storageID, sd, fileIoProvider != null ? fileIoProvider : new FileIoProvider(null, null), conf); } if (null == usage) { // set usage unless overridden by unit tests usage = new DF(sd.getCurrentDir().getParentFile(), conf); } return new FsVolumeImpl( dataset, storageID, sd, fileIoProvider != null ? fileIoProvider : new FileIoProvider(null, null), conf, usage); } }
@VisibleForTesting public FsVolumeImpl createFsVolume(String storageUuid, File currentDir, StorageType storageType) throws IOException { return new FsVolumeImpl(this, storageUuid, currentDir, conf, storageType); }
@VisibleForTesting public FsVolumeImpl createFsVolume(String storageUuid, File currentDir, StorageType storageType) throws IOException { return new FsVolumeImpl(this, storageUuid, currentDir, conf, storageType); }
@Test public void testCheckDirsWithClosedVolume() throws IOException { FsVolumeList volumeList = new FsVolumeList( Collections.<VolumeFailureInfo>emptyList(), blockScanner, blockChooser); List<FsVolumeImpl> volumes = new ArrayList<>(); for (int i = 0; i < 3; i++) { File curDir = new File(baseDir, "volume-" + i); curDir.mkdirs(); FsVolumeImpl volume = new FsVolumeImpl(dataset, "storage-id", curDir, conf, StorageType.DEFAULT); volumes.add(volume); volumeList.addVolume(volume.obtainReference()); } // Close the 2nd volume. volumes.get(1).closeAndWait(); // checkDirs() should ignore the 2nd volume since it is closed. volumeList.checkDirs(); }
FsVolumeImpl volume = new FsVolumeImpl(dataset, "storage-id", volDir, conf, StorageType.RAM_DISK); assertEquals("", 100L, volume.getReserved()); DFSConfigKeys.DFS_DATANODE_DU_RESERVED_KEY + "." + StringUtils.toLowerCase(StorageType.SSD.toString()), 2L); FsVolumeImpl volume1 = new FsVolumeImpl(dataset, "storage-id", volDir, conf, StorageType.RAM_DISK); assertEquals("", 1L, volume1.getReserved()); FsVolumeImpl volume2 = new FsVolumeImpl(dataset, "storage-id", volDir, conf, StorageType.SSD); assertEquals("", 2L, volume2.getReserved()); FsVolumeImpl volume3 = new FsVolumeImpl(dataset, "storage-id", volDir, conf, StorageType.DISK); assertEquals("", 100L, volume3.getReserved()); FsVolumeImpl volume4 = new FsVolumeImpl(dataset, "storage-id", volDir, conf, StorageType.DEFAULT); assertEquals("", 100L, volume4.getReserved());
long reservedForReplicas = 50L; conf.setLong(DFSConfigKeys.DFS_DATANODE_DU_RESERVED_KEY, duReserved); FsVolumeImpl volume = new FsVolumeImpl(dataset, "storage-id", volDir, conf, StorageType.DEFAULT); FsVolumeImpl spyVolume = Mockito.spy(volume);
@Test public void testReleaseVolumeRefIfNoBlockScanner() throws IOException { FsVolumeList volumeList = new FsVolumeList( Collections.<VolumeFailureInfo>emptyList(), null, blockChooser); File volDir = new File(baseDir, "volume-0"); volDir.mkdirs(); FsVolumeImpl volume = new FsVolumeImpl(dataset, "storage-id", volDir, conf, StorageType.DEFAULT); FsVolumeReference ref = volume.obtainReference(); volumeList.addVolume(ref); try { ref.close(); fail("Should throw exception because the reference is closed in " + "VolumeList#addVolume()."); } catch (IllegalStateException e) { } }
@Test public void testGetNextVolumeWithClosedVolume() throws IOException { FsVolumeList volumeList = new FsVolumeList( Collections.<VolumeFailureInfo>emptyList(), blockScanner, blockChooser); List<FsVolumeImpl> volumes = new ArrayList<>(); for (int i = 0; i < 3; i++) { File curDir = new File(baseDir, "nextvolume-" + i); curDir.mkdirs(); FsVolumeImpl volume = new FsVolumeImpl(dataset, "storage-id", curDir, conf, StorageType.DEFAULT); volume.setCapacityForTesting(1024 * 1024 * 1024); volumes.add(volume); volumeList.addVolume(volume.obtainReference()); } // Close the second volume. volumes.get(1).closeAndWait(); for (int i = 0; i < 10; i++) { try (FsVolumeReference ref = volumeList.getNextVolume(StorageType.DEFAULT, 128)) { // volume No.2 will not be chosen. assertNotEquals(ref.getVolume(), volumes.get(1)); } } }
private void addVolume(Collection<StorageLocation> dataLocations, Storage.StorageDirectory sd) throws IOException { final File dir = sd.getCurrentDir(); final StorageType storageType = getStorageTypeFromLocations(dataLocations, sd.getRoot()); // If IOException raises from FsVolumeImpl() or getVolumeMap(), there is // nothing needed to be rolled back to make various data structures, e.g., // storageMap and asyncDiskService, consistent. FsVolumeImpl fsVolume = new FsVolumeImpl( this, sd.getStorageUuid(), dir, this.conf, storageType); FsVolumeReference ref = fsVolume.obtainReference(); ReplicaMap tempVolumeMap = new ReplicaMap(this); fsVolume.getVolumeMap(tempVolumeMap, ramDiskReplicaTracker); synchronized (this) { volumeMap.addAll(tempVolumeMap); storageMap.put(sd.getStorageUuid(), new DatanodeStorage(sd.getStorageUuid(), DatanodeStorage.State.NORMAL, storageType)); asyncDiskService.addVolume(sd.getCurrentDir()); volumes.addVolume(ref); } LOG.info("Added volume - " + dir + ", StorageType: " + storageType); }
private void addVolume(Collection<StorageLocation> dataLocations, Storage.StorageDirectory sd) throws IOException { final File dir = sd.getCurrentDir(); final StorageType storageType = getStorageTypeFromLocations(dataLocations, sd.getRoot()); // If IOException raises from FsVolumeImpl() or getVolumeMap(), there is // nothing needed to be rolled back to make various data structures, e.g., // storageMap and asyncDiskService, consistent. FsVolumeImpl fsVolume = new FsVolumeImpl( this, sd.getStorageUuid(), dir, this.conf, storageType); FsVolumeReference ref = fsVolume.obtainReference(); ReplicaMap tempVolumeMap = new ReplicaMap(this); fsVolume.getVolumeMap(tempVolumeMap, ramDiskReplicaTracker); synchronized (this) { volumeMap.addAll(tempVolumeMap); storageMap.put(sd.getStorageUuid(), new DatanodeStorage(sd.getStorageUuid(), DatanodeStorage.State.NORMAL, storageType)); asyncDiskService.addVolume(sd.getCurrentDir()); volumes.addVolume(ref); } LOG.info("Added volume - " + dir + ", StorageType: " + storageType); }