List<StorageDirectory> succeedDirs = Lists.newArrayList(); try { if (containsStorageDir(location, nsInfo.getBlockPoolID())) { throw new IOException( "BlockPoolSliceStorage.recoverTransitionRead: " + } catch (IOException e) { LOG.warn("Failed to analyze storage directories for block pool {}", nsInfo.getBlockPoolID(), e); throw e;
String getBlockPoolId(boolean quiet) { // avoid lock contention unless the registration hasn't completed. String id = bpId; if (id != null) { return id; } readLock(); try { if (bpNSInfo != null) { return bpNSInfo.getBlockPoolID(); } else { if (!quiet) { LOG.warn("Block pool ID needed, but service not yet registered with " + "NN, trace:", new Exception()); } return null; } } finally { readUnlock(); } }
/** * Analyze storage directories. Recover from previous transitions if required. * * The block pool storages are either all analyzed or none of them is loaded. * Therefore, a failure on loading any block pool storage results a faulty * data volume. * * @param nsInfo namespace information * @param dataDirs storage directories of block pool * @param startOpt startup option * @throws IOException on error */ List<StorageDirectory> recoverTransitionRead(NamespaceInfo nsInfo, StorageLocation location, StartupOption startOpt, List<Callable<StorageDirectory>> callables, Configuration conf) throws IOException { LOG.info("Analyzing storage directories for bpid {}", nsInfo .getBlockPoolID()); final List<StorageDirectory> loaded = loadBpStorageDirectories( nsInfo, location, startOpt, callables, conf); for (StorageDirectory sd : loaded) { addStorageDir(sd); } return loaded; }
/** * Get the BlockPoolSliceStorage from {@link bpStorageMap}. * If the object is not found, create a new object and put it to the map. */ synchronized BlockPoolSliceStorage getBlockPoolSliceStorage( final NamespaceInfo nsInfo) { final String bpid = nsInfo.getBlockPoolID(); BlockPoolSliceStorage bpStorage = bpStorageMap.get(bpid); if (bpStorage == null) { bpStorage = new BlockPoolSliceStorage(nsInfo.getNamespaceID(), bpid, nsInfo.getCTime(), nsInfo.getClusterID()); bpStorageMap.put(bpid, bpStorage); } return bpStorage; }
/** * Format a block pool slice storage. * @param dnCurDir DataStorage current directory * @param nsInfo the name space info * @throws IOException Signals that an I/O exception has occurred. */ void format(File dnCurDir, NamespaceInfo nsInfo) throws IOException { File curBpDir = getBpRoot(nsInfo.getBlockPoolID(), dnCurDir); StorageDirectory bpSdir = new StorageDirectory(curBpDir); format(bpSdir, nsInfo); }
private List<StorageDirectory> loadBlockPoolSliceStorage(DataNode datanode, NamespaceInfo nsInfo, Collection<StorageLocation> dataDirs, StartupOption startOpt, ExecutorService executor) throws IOException { final String bpid = nsInfo.getBlockPoolID(); final BlockPoolSliceStorage bpStorage = getBlockPoolSliceStorage(nsInfo); final List<StorageDirectory> success = Lists.newArrayList();
void format(FSNamesystem fsn, String clusterId, boolean force) throws IOException { long fileCount = fsn.getFilesTotal(); // Expect 1 file, which is the root inode Preconditions.checkState(fileCount == 1, "FSImage.format should be called with an uninitialized namesystem, has " + fileCount + " files"); NamespaceInfo ns = NNStorage.newNamespaceInfo(); LOG.info("Allocated new BlockPoolId: " + ns.getBlockPoolID()); ns.clusterID = clusterId; storage.format(ns); editLog.formatNonFileJournals(ns, force); saveFSImageInAllDirs(fsn, 0); }
/** * Format all available storage directories. */ public void format(NamespaceInfo nsInfo) throws IOException { Preconditions.checkArgument(nsInfo.getLayoutVersion() == 0 || nsInfo.getLayoutVersion() == HdfsServerConstants.NAMENODE_LAYOUT_VERSION, "Bad layout version: %s", nsInfo.getLayoutVersion()); this.setStorageInfo(nsInfo); this.blockpoolID = nsInfo.getBlockPoolID(); for (Iterator<StorageDirectory> it = dirIterator(); it.hasNext();) { StorageDirectory sd = it.next(); format(sd); } }
@VisibleForTesting NamespaceInfo setNamespaceInfo(NamespaceInfo nsInfo) throws IOException { writeLock(); try { NamespaceInfo old = bpNSInfo; if (bpNSInfo != null && nsInfo != null) { checkNSEquality(bpNSInfo.getBlockPoolID(), nsInfo.getBlockPoolID(), "Blockpool ID"); checkNSEquality(bpNSInfo.getNamespaceID(), nsInfo.getNamespaceID(), "Namespace ID"); checkNSEquality(bpNSInfo.getClusterID(), nsInfo.getClusterID(), "Cluster ID"); } bpNSInfo = nsInfo; // cache the block pool id for lock-free access. bpId = (nsInfo != null) ? nsInfo.getBlockPoolID() : null; return old; } finally { writeUnlock(); } }
new VolumeBuilder(this, sd); for (NamespaceInfo nsInfo : nsInfos) { location.makeBlockPoolDir(nsInfo.getBlockPoolID(), null); builder.addBpStorageDirectories(nsInfo.getBlockPoolID(), dirs);
/** * Format a block pool slice storage. * @param bpSdir the block pool storage * @param nsInfo the name space info * @throws IOException Signals that an I/O exception has occurred. */ private void format(StorageDirectory bpSdir, NamespaceInfo nsInfo) throws IOException { LOG.info("Formatting block pool {} directory {}", blockpoolID, bpSdir .getCurrentDir()); bpSdir.clearDirectory(); // create directory this.layoutVersion = HdfsServerConstants.DATANODE_LAYOUT_VERSION; this.cTime = nsInfo.getCTime(); this.namespaceID = nsInfo.getNamespaceID(); this.blockpoolID = nsInfo.getBlockPoolID(); writeProperties(bpSdir); }
throws IOException { StorageDirectory sd = new StorageDirectory( nsInfo.getBlockPoolID(), null, true, location); try { StorageState curState = sd.analyzeStorage(startOpt, this, true); case NON_EXISTENT: LOG.info("Block pool storage directory for location {} and block pool" + " id {} does not exist", location, nsInfo.getBlockPoolID()); throw new IOException("Storage directory for location " + location + " and block pool id " + nsInfo.getBlockPoolID() + " does not exist"); case NOT_FORMATTED: // format LOG.info("Block pool storage directory for location {} and block pool" + " id {} is not formatted. Formatting ...", location, nsInfo.getBlockPoolID()); format(sd, nsInfo); break;
public NameNodeConnector(String name, URI nameNodeUri, Path idPath, List<Path> targetPaths, Configuration conf, int maxNotChangedIterations) throws IOException { this.nameNodeUri = nameNodeUri; this.idPath = idPath; this.targetPaths = targetPaths == null || targetPaths.isEmpty() ? Arrays .asList(new Path("/")) : targetPaths; this.maxNotChangedIterations = maxNotChangedIterations; this.namenode = NameNodeProxies.createProxy(conf, nameNodeUri, NamenodeProtocol.class).getProxy(); this.client = NameNodeProxies.createProxy(conf, nameNodeUri, ClientProtocol.class, fallbackToSimpleAuth).getProxy(); this.fs = (DistributedFileSystem)FileSystem.get(nameNodeUri, conf); final NamespaceInfo namespaceinfo = namenode.versionRequest(); this.blockpoolID = namespaceinfo.getBlockPoolID(); final FsServerDefaults defaults = fs.getServerDefaults(new Path("/")); this.keyManager = new KeyManager(blockpoolID, namenode, defaults.getEncryptDataTransfer(), conf); // if it is for test, we do not create the id file if (checkOtherInstanceRunning) { out = checkAndMarkRunning(); if (out == null) { // Exit if there is another one running. throw new IOException("Another " + name + " is running."); } } }
@Override // NameNode protected void initialize(Configuration conf) throws IOException { // async edit logs are incompatible with backup node due to race // conditions resulting from laxer synchronization conf.setBoolean(DFSConfigKeys.DFS_NAMENODE_EDITS_ASYNC_LOGGING, false); // Trash is disabled in BackupNameNode, // but should be turned back on if it ever becomes active. conf.setLong(CommonConfigurationKeys.FS_TRASH_INTERVAL_KEY, CommonConfigurationKeys.FS_TRASH_INTERVAL_DEFAULT); NamespaceInfo nsInfo = handshake(conf); super.initialize(conf); namesystem.setBlockPoolId(nsInfo.getBlockPoolID()); if (false == namesystem.isInSafeMode()) { namesystem.setSafeMode(SafeModeAction.SAFEMODE_ENTER); } // Backup node should never do lease recovery, // therefore lease hard limit should never expire. namesystem.leaseManager.setLeasePeriod( HdfsConstants.LEASE_SOFTLIMIT_PERIOD, Long.MAX_VALUE); // register with the active name-node registerWith(nsInfo); // Checkpoint daemon should start after the rpc server started runCheckpointDaemon(conf); InetSocketAddress addr = getHttpAddress(); if (addr != null) { conf.set(BN_HTTP_ADDRESS_NAME_KEY, NetUtils.getHostPortString(getHttpAddress())); } }
setClusterId(nsInfo.clusterID, nsInfo.getBlockPoolID()); data.addBlockPool(nsInfo.getBlockPoolID(), getConf()); blockScanner.enableBlockPoolId(bpos.getBlockPoolId()); initDirectoryScanner(getConf());
storage.setBlockPoolID(nsInfo.getBlockPoolID()); storage.setClusterID(nsInfo.getClusterID()); } else {
/** * Create a DatanodeRegistration for a specific block pool. * @param nsInfo the namespace info from the first part of the NN handshake */ DatanodeRegistration createBPRegistration(NamespaceInfo nsInfo) { StorageInfo storageInfo = storage.getBPStorage(nsInfo.getBlockPoolID()); if (storageInfo == null) { // it's null in the case of SimulatedDataSet storageInfo = new StorageInfo( DataNodeLayoutVersion.CURRENT_LAYOUT_VERSION, nsInfo.getNamespaceID(), nsInfo.clusterID, nsInfo.getCTime(), NodeType.DATA_NODE); } DatanodeID dnId = new DatanodeID( streamingAddr.getAddress().getHostAddress(), hostName, storage.getDatanodeUuid(), getXferPort(), getInfoPort(), infoSecurePort, getIpcPort()); return new DatanodeRegistration(dnId, storageInfo, new ExportedBlockKeys(), VersionInfo.getVersion()); }
String bpid = nsInfo.getBlockPoolID(); try { fsVolume.addBlockPool(bpid, this.conf, this.timer);
throw new IOException("Startup option not set."); final String bpid = nsInfo.getBlockPoolID();
public static NamespaceInfoProto convert(NamespaceInfo info) { NamespaceInfoProto.Builder builder = NamespaceInfoProto.newBuilder(); builder.setBlockPoolID(info.getBlockPoolID()) .setBuildVersion(info.getBuildVersion()) .setUnused(0) .setStorageInfo(PBHelper.convert((StorageInfo)info)) .setSoftwareVersion(info.getSoftwareVersion()) .setCapabilities(info.getCapabilities()); HAServiceState state = info.getState(); if(state != null) { builder.setState(convert(info.getState())); } return builder.build(); }