private static File getBlockPoolCurrentDir(String bpid, StorageLocation location) { if (location == null || location.getStorageType() == StorageType.PROVIDED) { return null; } else { return new File(location.getBpURI(bpid, STORAGE_DIR_CURRENT)); } }
private static File getStorageLocationFile(StorageLocation location) { if (location == null || location.getStorageType() == StorageType.PROVIDED) { return null; } try { return new File(location.getUri()); } catch (IllegalArgumentException e) { //if location does not refer to a File return null; } }
public boolean matchesStorageDirectory(StorageDirectory sd, String bpid) throws IOException { if (sd.getStorageLocation().getStorageType() == StorageType.PROVIDED && storageType == StorageType.PROVIDED) { return matchesStorageDirectory(sd); } if (sd.getStorageLocation().getStorageType() == StorageType.PROVIDED || storageType == StorageType.PROVIDED) { // only one PROVIDED storage directory can exist; so this cannot match! return false; } // both storage directories are local return this.getBpURI(bpid, Storage.STORAGE_DIR_CURRENT).normalize() .equals(sd.getRoot().toURI().normalize()); }
ProvidedVolumeImpl(FsDatasetImpl dataset, String storageID, StorageDirectory sd, FileIoProvider fileIoProvider, Configuration conf) throws IOException { super(dataset, storageID, sd, fileIoProvider, conf, null); assert getStorageLocation().getStorageType() == StorageType.PROVIDED: "Only provided storages must use ProvidedVolume"; baseURI = getStorageLocation().getUri(); df = new ProvidedVolumeDF(); remoteFS = FileSystem.get(baseURI, conf); }
newLocation.getNormalizedUri().toString()); if (oldLocation != null && oldLocation.getStorageType() != newLocation.getStorageType()) { throw new IOException("Changing storage type is not allowed.");
FsVolumeImpl(FsDatasetImpl dataset, String storageID, StorageDirectory sd, FileIoProvider fileIoProvider, Configuration conf, DF usage) throws IOException { if (sd.getStorageLocation() == null) { throw new IOException("StorageLocation specified for storage directory " + sd + " is null"); } this.dataset = dataset; this.storageID = storageID; this.reservedForReplicas = new AtomicLong(0L); this.storageLocation = sd.getStorageLocation(); this.currentDir = sd.getCurrentDir(); this.storageType = storageLocation.getStorageType(); this.configuredCapacity = -1; this.usage = usage; if (currentDir != null) { File parent = currentDir.getParentFile(); cacheExecutor = initializeCacheExecutor(parent); this.metrics = DataNodeVolumeMetrics.create(conf, parent.getPath()); } else { cacheExecutor = null; this.metrics = null; } this.conf = conf; this.fileIoProvider = fileIoProvider; this.reserved = new ReservedSpaceCalculator.Builder(conf) .setUsage(usage).setStorageType(storageType).build(); }
location.getStorageType() == StorageType.PROVIDED) {
/** Create an ID for this storage. * @return true if a new storage ID was generated. * */ public static boolean createStorageID( StorageDirectory sd, boolean regenerateStorageIds, Configuration conf) { final String oldStorageID = sd.getStorageUuid(); if (sd.getStorageLocation() != null && sd.getStorageLocation().getStorageType() == StorageType.PROVIDED) { // Only one provided storage id is supported. // TODO support multiple provided storage ids sd.setStorageUuid(conf.get(DFSConfigKeys.DFS_PROVIDER_STORAGEUUID, DFSConfigKeys.DFS_PROVIDER_STORAGEUUID_DEFAULT)); return false; } if (oldStorageID == null || regenerateStorageIds) { sd.setStorageUuid(DatanodeStorage.generateUuid()); LOG.info("Generated new storageID {} for directory {} {}", sd .getStorageUuid(), sd.getRoot(), (oldStorageID == null ? "" : (" to replace " + oldStorageID))); return true; } return false; }
private void addVolume(Storage.StorageDirectory sd) throws IOException { final StorageLocation storageLocation = sd.getStorageLocation(); // If IOException raises from FsVolumeImpl() or getVolumeMap(), there is // nothing needed to be rolled back to make various data structures, e.g., // storageMap and asyncDiskService, consistent. FsVolumeImpl fsVolume = new FsVolumeImplBuilder() .setDataset(this) .setStorageID(sd.getStorageUuid()) .setStorageDirectory(sd) .setFileIoProvider(datanode.getFileIoProvider()) .setConf(this.conf) .build(); FsVolumeReference ref = fsVolume.obtainReference(); ReplicaMap tempVolumeMap = new ReplicaMap(datasetLock); fsVolume.getVolumeMap(tempVolumeMap, ramDiskReplicaTracker); activateVolume(tempVolumeMap, sd, storageLocation.getStorageType(), ref); LOG.info("Added volume - " + storageLocation + ", StorageType: " + storageLocation.getStorageType()); }
FsVolumeImpl build() throws IOException { if (sd.getStorageLocation().getStorageType() == StorageType.PROVIDED) { return new ProvidedVolumeImpl(dataset, storageID, sd, fileIoProvider != null ? fileIoProvider : new FileIoProvider(null, null), conf); } if (null == usage) { // set usage unless overridden by unit tests usage = new DF(sd.getCurrentDir().getParentFile(), conf); } return new FsVolumeImpl( dataset, storageID, sd, fileIoProvider != null ? fileIoProvider : new FileIoProvider(null, null), conf, usage); } }
StorageType storageType = location.getStorageType(); final FsVolumeImpl fsVolume = createFsVolume(sd.getStorageUuid(), sd, location);
StartupOption startOpt, List<Callable<StorageDirectory>> callables, Configuration conf) throws IOException { if (sd.getStorageLocation().getStorageType() == StorageType.PROVIDED) { return false; // regular startup for PROVIDED storage directories
StartupOption startOpt, List<Callable<StorageDirectory>> callables, Configuration conf) throws IOException { if (sd.getStorageLocation().getStorageType() == StorageType.PROVIDED) { createStorageID(sd, layoutVersion, conf); return false; // regular start up for PROVIDED storage directories
@Override public int compareTo(StorageLocation obj) { if (obj == this) { return 0; } else if (obj == null) { return -1; } StorageLocation otherStorage = (StorageLocation) obj; if (this.getNormalizedUri() != null && otherStorage.getNormalizedUri() != null) { return this.getNormalizedUri().compareTo( otherStorage.getNormalizedUri()); } else if (this.getNormalizedUri() == null && otherStorage.getNormalizedUri() == null) { return this.storageType.compareTo(otherStorage.getStorageType()); } else if (this.getNormalizedUri() == null) { return -1; } else { return 1; } }
private StorageType getStorageTypeFromLocations( Collection<StorageLocation> dataLocations, File dir) { for (StorageLocation dataLocation : dataLocations) { if (dataLocation.getFile().equals(dir)) { return dataLocation.getStorageType(); } } return StorageType.DEFAULT; }
private StorageType getStorageTypeFromLocations( Collection<StorageLocation> dataLocations, File dir) { for (StorageLocation dataLocation : dataLocations) { if (dataLocation.getFile().equals(dir)) { return dataLocation.getStorageType(); } } return StorageType.DEFAULT; }
locations = DataNode.getStorageLocations(conf); assertThat(locations.size(), is(5)); assertThat(locations.get(0).getStorageType(), is(StorageType.DISK)); assertThat(locations.get(0).getUri(), is(dir0.toURI())); assertThat(locations.get(1).getStorageType(), is(StorageType.DISK)); assertThat(locations.get(1).getUri(), is(dir1.toURI())); assertThat(locations.get(2).getStorageType(), is(StorageType.SSD)); assertThat(locations.get(2).getUri(), is(dir2.toURI())); assertThat(locations.get(3).getStorageType(), is(StorageType.DISK)); assertThat(locations.get(3).getUri(), is(dir3.toURI())); assertThat(locations.get(4).getStorageType(), is(StorageType.RAM_DISK)); assertThat(locations.get(4).getUri(), is(dir4.toURI())); locations = DataNode.getStorageLocations(conf); assertThat(locations.size(), is(2)); assertThat(locations.get(0).getStorageType(), is(StorageType.DISK)); assertThat(locations.get(0).getUri(), is(dir0.toURI())); assertThat(locations.get(1).getStorageType(), is(StorageType.DISK)); assertThat(locations.get(1).getUri(), is(dir1.toURI()));
StorageType storageType = location.getStorageType(); final FsVolumeImpl fsVolume = createFsVolume(sd.getStorageUuid(), sd.getCurrentDir(), storageType);
StorageType storageType = location.getStorageType(); final FsVolumeImpl fsVolume = createFsVolume(sd.getStorageUuid(), sd.getCurrentDir(), storageType);