@Override protected void setPropertiesFromFields(Properties props, StorageDirectory sd ) throws IOException { props.setProperty("storageType", storageType.toString()); props.setProperty("clusterID", clusterID); props.setProperty("cTime", String.valueOf(cTime)); props.setProperty("layoutVersion", String.valueOf(layoutVersion)); props.setProperty("storageID", sd.getStorageUuid()); String datanodeUuid = getDatanodeUuid(); if (datanodeUuid != null) { props.setProperty("datanodeUuid", datanodeUuid); } // Set NamespaceID in version before federation if (!DataNodeLayoutVersion.supports( LayoutVersion.Feature.FEDERATION, layoutVersion)) { props.setProperty("namespaceID", String.valueOf(namespaceID)); } }
final StorageLocation sdLocation = sd.getStorageLocation(); LOG.info("Checking removing StorageLocation " + sdLocation + " with id " + sd.getStorageUuid()); if (storageLocationsToRemove.contains(sdLocation)) { LOG.info("Removing StorageLocation " + sdLocation + " with id " + sd.getStorageUuid() + " from FsDataset."); asyncDiskService.removeVolume(sd.getStorageUuid()); volumes.removeVolume(sdLocation, clearFailure); volumes.waitVolumeRemoved(5000, datasetLockCondition); storageToRemove.add(sd.getStorageUuid()); storageLocationsToRemove.remove(sdLocation);
/** * Activate a volume to serve requests. * @throws IOException if the storage UUID already exists. */ private void activateVolume( ReplicaMap replicaMap, Storage.StorageDirectory sd, StorageType storageType, FsVolumeReference ref) throws IOException { try (AutoCloseableLock lock = datasetLock.acquire()) { DatanodeStorage dnStorage = storageMap.get(sd.getStorageUuid()); if (dnStorage != null) { final String errorMsg = String.format( "Found duplicated storage UUID: %s in %s.", sd.getStorageUuid(), sd.getVersionFile()); LOG.error(errorMsg); throw new IOException(errorMsg); } volumeMap.addAll(replicaMap); storageMap.put(sd.getStorageUuid(), new DatanodeStorage(sd.getStorageUuid(), DatanodeStorage.State.NORMAL, storageType)); asyncDiskService.addVolume((FsVolumeImpl) ref.getVolume()); volumes.addVolume(ref); } }
storageToRemove.add(sd.getStorageUuid());
createFsVolume(sd.getStorageUuid(), sd.getCurrentDir(), storageType); final ReplicaMap tempVolumeMap = new ReplicaMap(fsVolume); ArrayList<IOException> exceptions = Lists.newArrayList(); synchronized (this) { volumeMap.addAll(tempVolumeMap); storageMap.put(sd.getStorageUuid(), new DatanodeStorage(sd.getStorageUuid(), DatanodeStorage.State.NORMAL, storageType));
+ STORAGE_FILE_VERSION + " is invalid."); String sid = sd.getStorageUuid(); if (!(sid == null || sid.equals("") || ssid.equals("") || sid.equals(ssid))) {
+ STORAGE_FILE_VERSION + " is invalid."); String sid = sd.getStorageUuid(); if (!(sid == null || sid.equals("") || ssid.equals("") || sid.equals(ssid))) {
private void addVolume(Collection<StorageLocation> dataLocations, Storage.StorageDirectory sd) throws IOException { final File dir = sd.getCurrentDir(); final StorageType storageType = getStorageTypeFromLocations(dataLocations, sd.getRoot()); // If IOException raises from FsVolumeImpl() or getVolumeMap(), there is // nothing needed to be rolled back to make various data structures, e.g., // storageMap and asyncDiskService, consistent. FsVolumeImpl fsVolume = new FsVolumeImpl( this, sd.getStorageUuid(), dir, this.conf, storageType); FsVolumeReference ref = fsVolume.obtainReference(); ReplicaMap tempVolumeMap = new ReplicaMap(this); fsVolume.getVolumeMap(tempVolumeMap, ramDiskReplicaTracker); synchronized (this) { volumeMap.addAll(tempVolumeMap); storageMap.put(sd.getStorageUuid(), new DatanodeStorage(sd.getStorageUuid(), DatanodeStorage.State.NORMAL, storageType)); asyncDiskService.addVolume(sd.getCurrentDir()); volumes.addVolume(ref); } LOG.info("Added volume - " + dir + ", StorageType: " + storageType); }
private void addVolume(Collection<StorageLocation> dataLocations, Storage.StorageDirectory sd) throws IOException { final File dir = sd.getCurrentDir(); final StorageType storageType = getStorageTypeFromLocations(dataLocations, sd.getRoot()); // If IOException raises from FsVolumeImpl() or getVolumeMap(), there is // nothing needed to be rolled back to make various data structures, e.g., // storageMap and asyncDiskService, consistent. FsVolumeImpl fsVolume = new FsVolumeImpl( this, sd.getStorageUuid(), dir, this.conf, storageType); FsVolumeReference ref = fsVolume.obtainReference(); ReplicaMap tempVolumeMap = new ReplicaMap(this); fsVolume.getVolumeMap(tempVolumeMap, ramDiskReplicaTracker); synchronized (this) { volumeMap.addAll(tempVolumeMap); storageMap.put(sd.getStorageUuid(), new DatanodeStorage(sd.getStorageUuid(), DatanodeStorage.State.NORMAL, storageType)); asyncDiskService.addVolume(sd.getCurrentDir()); volumes.addVolume(ref); } LOG.info("Added volume - " + dir + ", StorageType: " + storageType); }
@Override protected void setPropertiesFromFields(Properties props, StorageDirectory sd ) throws IOException { props.setProperty("storageType", storageType.toString()); props.setProperty("clusterID", clusterID); props.setProperty("cTime", String.valueOf(cTime)); props.setProperty("layoutVersion", String.valueOf(layoutVersion)); props.setProperty("storageID", sd.getStorageUuid()); String datanodeUuid = getDatanodeUuid(); if (datanodeUuid != null) { props.setProperty("datanodeUuid", datanodeUuid); } // Set NamespaceID in version before federation if (!DataNodeLayoutVersion.supports( LayoutVersion.Feature.FEDERATION, layoutVersion)) { props.setProperty("namespaceID", String.valueOf(namespaceID)); } }
@Override protected void setPropertiesFromFields(Properties props, StorageDirectory sd ) throws IOException { props.setProperty("storageType", storageType.toString()); props.setProperty("clusterID", clusterID); props.setProperty("cTime", String.valueOf(cTime)); props.setProperty("layoutVersion", String.valueOf(layoutVersion)); props.setProperty("storageID", sd.getStorageUuid()); String datanodeUuid = getDatanodeUuid(); if (datanodeUuid != null) { props.setProperty("datanodeUuid", datanodeUuid); } // Set NamespaceID in version before federation if (!DataNodeLayoutVersion.supports( LayoutVersion.Feature.FEDERATION, layoutVersion)) { props.setProperty("namespaceID", String.valueOf(namespaceID)); } }
/** Create an ID for this storage. * @return true if a new storage ID was generated. * */ public static boolean createStorageID( StorageDirectory sd, boolean regenerateStorageIds) { final String oldStorageID = sd.getStorageUuid(); if (oldStorageID == null || regenerateStorageIds) { sd.setStorageUuid(DatanodeStorage.generateUuid()); LOG.info("Generated new storageID " + sd.getStorageUuid() + " for directory " + sd.getRoot() + (oldStorageID == null ? "" : (" to replace " + oldStorageID))); return true; } return false; }
/** Create an ID for this storage. * @return true if a new storage ID was generated. * */ public static boolean createStorageID( StorageDirectory sd, boolean regenerateStorageIds) { final String oldStorageID = sd.getStorageUuid(); if (oldStorageID == null || regenerateStorageIds) { sd.setStorageUuid(DatanodeStorage.generateUuid()); LOG.info("Generated new storageID " + sd.getStorageUuid() + " for directory " + sd.getRoot() + (oldStorageID == null ? "" : (" to replace " + oldStorageID))); return true; } return false; }
private static boolean createStorageID(StorageDirectory sd, int lv) { // Clusters previously upgraded from layout versions earlier than // ADD_DATANODE_AND_STORAGE_UUIDS failed to correctly generate a // new storage ID. We check for that and fix it now. final boolean haveValidStorageId = DataNodeLayoutVersion.supports( LayoutVersion.Feature.ADD_DATANODE_AND_STORAGE_UUIDS, lv) && DatanodeStorage.isValidStorageId(sd.getStorageUuid()); return createStorageID(sd, !haveValidStorageId); }
private static boolean createStorageID(StorageDirectory sd, int lv) { // Clusters previously upgraded from layout versions earlier than // ADD_DATANODE_AND_STORAGE_UUIDS failed to correctly generate a // new storage ID. We check for that and fix it now. final boolean haveValidStorageId = DataNodeLayoutVersion.supports( LayoutVersion.Feature.ADD_DATANODE_AND_STORAGE_UUIDS, lv) && DatanodeStorage.isValidStorageId(sd.getStorageUuid()); return createStorageID(sd, !haveValidStorageId); }
+ STORAGE_FILE_VERSION + " is invalid."); String sid = sd.getStorageUuid(); if (!(sid == null || sid.equals("") || ssid.equals("") || sid.equals(ssid))) {
createFsVolume(sd.getStorageUuid(), sd, location); final ReplicaMap tempVolumeMap = new ReplicaMap(new AutoCloseableLock()); ArrayList<IOException> exceptions = Lists.newArrayList();
/** Create an ID for this storage. * @return true if a new storage ID was generated. * */ public static boolean createStorageID( StorageDirectory sd, boolean regenerateStorageIds, Configuration conf) { final String oldStorageID = sd.getStorageUuid(); if (sd.getStorageLocation() != null && sd.getStorageLocation().getStorageType() == StorageType.PROVIDED) { // Only one provided storage id is supported. // TODO support multiple provided storage ids sd.setStorageUuid(conf.get(DFSConfigKeys.DFS_PROVIDER_STORAGEUUID, DFSConfigKeys.DFS_PROVIDER_STORAGEUUID_DEFAULT)); return false; } if (oldStorageID == null || regenerateStorageIds) { sd.setStorageUuid(DatanodeStorage.generateUuid()); LOG.info("Generated new storageID {} for directory {} {}", sd .getStorageUuid(), sd.getRoot(), (oldStorageID == null ? "" : (" to replace " + oldStorageID))); return true; } return false; }
private void addVolume(Storage.StorageDirectory sd) throws IOException { final StorageLocation storageLocation = sd.getStorageLocation(); // If IOException raises from FsVolumeImpl() or getVolumeMap(), there is // nothing needed to be rolled back to make various data structures, e.g., // storageMap and asyncDiskService, consistent. FsVolumeImpl fsVolume = new FsVolumeImplBuilder() .setDataset(this) .setStorageID(sd.getStorageUuid()) .setStorageDirectory(sd) .setFileIoProvider(datanode.getFileIoProvider()) .setConf(this.conf) .build(); FsVolumeReference ref = fsVolume.obtainReference(); ReplicaMap tempVolumeMap = new ReplicaMap(datasetLock); fsVolume.getVolumeMap(tempVolumeMap, ramDiskReplicaTracker); activateVolume(tempVolumeMap, sd, storageLocation.getStorageType(), ref); LOG.info("Added volume - " + storageLocation + ", StorageType: " + storageLocation.getStorageType()); }
private static boolean createStorageID(StorageDirectory sd, int lv, Configuration conf) { // Clusters previously upgraded from layout versions earlier than // ADD_DATANODE_AND_STORAGE_UUIDS failed to correctly generate a // new storage ID. We check for that and fix it now. final boolean haveValidStorageId = DataNodeLayoutVersion.supports( LayoutVersion.Feature.ADD_DATANODE_AND_STORAGE_UUIDS, lv) && DatanodeStorage.isValidStorageId(sd.getStorageUuid()); return createStorageID(sd, !haveValidStorageId, conf); }