/** * Format the local storage with the given namespace. */ void format(NamespaceInfo nsInfo, boolean force) throws IOException { Preconditions.checkState(nsInfo.getNamespaceID() != 0, "can't format with uninitialized namespace info: %s", nsInfo); LOG.info("Formatting journal id : " + journalId + " with namespace info: " + nsInfo + " and force: " + force); storage.format(nsInfo, force); refreshCachedData(); }
void checkConsistentNamespace(NamespaceInfo nsInfo) throws IOException { if (nsInfo.getNamespaceID() != getNamespaceID()) { throw new IOException("Incompatible namespaceID for journal " + this.sd + ": NameNode has nsId " + nsInfo.getNamespaceID() + " but storage has nsId " + getNamespaceID()); } if (!nsInfo.getClusterID().equals(getClusterID())) { throw new IOException("Incompatible clusterID for journal " + this.sd + ": NameNode has clusterId '" + nsInfo.getClusterID() + "' but storage has clusterId '" + getClusterID() + "'"); } }
/** * Get the BlockPoolSliceStorage from {@link bpStorageMap}. * If the object is not found, create a new object and put it to the map. */ synchronized BlockPoolSliceStorage getBlockPoolSliceStorage( final NamespaceInfo nsInfo) { final String bpid = nsInfo.getBlockPoolID(); BlockPoolSliceStorage bpStorage = bpStorageMap.get(bpid); if (bpStorage == null) { bpStorage = new BlockPoolSliceStorage(nsInfo.getNamespaceID(), bpid, nsInfo.getCTime(), nsInfo.getClusterID()); bpStorageMap.put(bpid, bpStorage); } return bpStorage; }
private void doUpgrade(String name, final StorageDirectory bpSd, NamespaceInfo nsInfo, final File bpPrevDir, final File bpTmpDir, final File bpCurDir, final int oldLV, Configuration conf) throws IOException { // 3. Create new <SD>/current with block files hardlinks and VERSION linkAllBlocks(bpTmpDir, bpCurDir, oldLV, conf); this.layoutVersion = HdfsServerConstants.DATANODE_LAYOUT_VERSION; assert this.namespaceID == nsInfo.getNamespaceID() : "Data-node and name-node layout versions must be the same."; this.cTime = nsInfo.getCTime(); writeProperties(bpSd); // 4.rename <SD>/current/<bpid>/previous.tmp to // <SD>/current/<bpid>/previous rename(bpTmpDir, bpPrevDir); LOG.info("Upgrade of {} is complete", name); }
/** * Format a block pool slice storage. * @param bpSdir the block pool storage * @param nsInfo the name space info * @throws IOException Signals that an I/O exception has occurred. */ private void format(StorageDirectory bpSdir, NamespaceInfo nsInfo) throws IOException { LOG.info("Formatting block pool {} directory {}", blockpoolID, bpSdir .getCurrentDir()); bpSdir.clearDirectory(); // create directory this.layoutVersion = HdfsServerConstants.DATANODE_LAYOUT_VERSION; this.cTime = nsInfo.getCTime(); this.namespaceID = nsInfo.getNamespaceID(); this.blockpoolID = nsInfo.getBlockPoolID(); writeProperties(bpSdir); }
void format(StorageDirectory sd, NamespaceInfo nsInfo, String newDatanodeUuid, Configuration conf) throws IOException { sd.clearDirectory(); // create directory this.layoutVersion = HdfsServerConstants.DATANODE_LAYOUT_VERSION; this.clusterID = nsInfo.getClusterID(); this.namespaceID = nsInfo.getNamespaceID(); this.cTime = 0; setDatanodeUuid(newDatanodeUuid); createStorageID(sd, false, conf); writeProperties(sd); }
@VisibleForTesting NamespaceInfo setNamespaceInfo(NamespaceInfo nsInfo) throws IOException { writeLock(); try { NamespaceInfo old = bpNSInfo; if (bpNSInfo != null && nsInfo != null) { checkNSEquality(bpNSInfo.getBlockPoolID(), nsInfo.getBlockPoolID(), "Blockpool ID"); checkNSEquality(bpNSInfo.getNamespaceID(), nsInfo.getNamespaceID(), "Namespace ID"); checkNSEquality(bpNSInfo.getClusterID(), nsInfo.getClusterID(), "Cluster ID"); } bpNSInfo = nsInfo; // cache the block pool id for lock-free access. bpId = (nsInfo != null) ? nsInfo.getBlockPoolID() : null; return old; } finally { writeUnlock(); } }
/** * Verifies a journal request */ private void verifyJournalRequest(JournalInfo journalInfo) throws IOException { verifyLayoutVersion(journalInfo.getLayoutVersion()); String errorMsg = null; int expectedNamespaceID = namesystem.getNamespaceInfo().getNamespaceID(); if (journalInfo.getNamespaceId() != expectedNamespaceID) { errorMsg = "Invalid namespaceID in journal request - expected " + expectedNamespaceID + " actual " + journalInfo.getNamespaceId(); LOG.warn(errorMsg); throw new UnregisteredNodeException(journalInfo); } if (!journalInfo.getClusterId().equals(namesystem.getClusterId())) { errorMsg = "Invalid clusterId in journal request - expected " + journalInfo.getClusterId() + " actual " + namesystem.getClusterId(); LOG.warn(errorMsg); throw new UnregisteredNodeException(journalInfo); } }
" Other NN's HTTP address: " + proxyInfo.getHttpAddress() + "\n" + " Other NN's IPC address: " + proxyInfo.getIpcAddress() + "\n" + " Namespace ID: " + nsInfo.getNamespaceID() + "\n" + " Block pool ID: " + nsInfo.getBlockPoolID() + "\n" + " Cluster ID: " + nsInfo.getClusterID() + "\n" +
assert this.layoutVersion >= HdfsServerConstants.DATANODE_LAYOUT_VERSION : "Future version is not allowed"; if (getNamespaceID() != nsInfo.getNamespaceID()) { throw new IOException("Incompatible namespaceIDs in " + sd.getRoot().getCanonicalPath() + ": namenode namespaceID = " + nsInfo.getNamespaceID() + "; datanode namespaceID = " + getNamespaceID());
case NOT_FORMATTED: // format LOG.info("Storage directory with location {} is not formatted for " + "namespace {}. Formatting...", location, nsInfo.getNamespaceID()); format(sd, nsInfo, datanode.getDatanodeUuid(), datanode.getConf()); break;
.getNamespaceID()); out.writeLong(numINodes); final BlockIdManager blockIdManager = sourceNamesystem.getBlockManager()
getNamespaceID() != nsInfo.getNamespaceID()) { throw new IOException("Incompatible namespaceIDs in " + sd.getRoot().getCanonicalPath() + ": namenode namespaceID = " + nsInfo.getNamespaceID() + "; datanode namespaceID = " + getNamespaceID());
/** * Create a DatanodeRegistration for a specific block pool. * @param nsInfo the namespace info from the first part of the NN handshake */ DatanodeRegistration createBPRegistration(NamespaceInfo nsInfo) { StorageInfo storageInfo = storage.getBPStorage(nsInfo.getBlockPoolID()); if (storageInfo == null) { // it's null in the case of SimulatedDataSet storageInfo = new StorageInfo( DataNodeLayoutVersion.CURRENT_LAYOUT_VERSION, nsInfo.getNamespaceID(), nsInfo.clusterID, nsInfo.getCTime(), NodeType.DATA_NODE); } DatanodeID dnId = new DatanodeID( streamingAddr.getAddress().getHostAddress(), hostName, storage.getDatanodeUuid(), getXferPort(), getInfoPort(), infoSecurePort, getIpcPort()); return new DatanodeRegistration(dnId, storageInfo, new ExportedBlockKeys(), VersionInfo.getVersion()); }
void setNamespaceInfo(NamespaceInfo nsinfo) { this.nsInfo = nsinfo; this.namespaceId = nsinfo.getNamespaceID(); namespaceManager.addNamespace(this); }
/** * For unit testing only, get Namespace ID * TODO temporary */ public int getNamespaceID() { return namesystem.getNamespaceInfo().getNamespaceID(); }
public void write(DataOutput out) throws IOException { UTF8.writeString(out, getBuildVersion()); out.writeInt(getLayoutVersion()); out.writeInt(getNamespaceID()); out.writeLong(getCTime()); out.writeInt(getDistributedUpgradeVersion()); }
public void write(DataOutput out) throws IOException { UTF8.writeString(out, getBuildVersion()); out.writeInt(getLayoutVersion()); out.writeInt(getNamespaceID()); out.writeLong(getCTime()); out.writeInt(getDistributedUpgradeVersion()); }
private void saveNameSystemSection(FileSummary.Builder summary) throws IOException { final FSNamesystem fsn = context.getSourceNamesystem(); OutputStream out = sectionOutputStream; BlockIdManager blockIdManager = fsn.getBlockManager().getBlockIdManager(); NameSystemSection.Builder b = NameSystemSection.newBuilder() .setGenstampV1(blockIdManager.getLegacyGenerationStamp()) .setGenstampV1Limit(blockIdManager.getLegacyGenerationStampLimit()) .setGenstampV2(blockIdManager.getGenerationStamp()) .setLastAllocatedBlockId(blockIdManager.getLastAllocatedContiguousBlockId()) .setLastAllocatedStripedBlockId(blockIdManager.getLastAllocatedStripedBlockId()) .setTransactionId(context.getTxId()); // We use the non-locked version of getNamespaceInfo here since // the coordinating thread of saveNamespace already has read-locked // the namespace for us. If we attempt to take another readlock // from the actual saver thread, there's a potential of a // fairness-related deadlock. See the comments on HDFS-2223. b.setNamespaceId(fsn.unprotectedGetNamespaceInfo().getNamespaceID()); if (fsn.isRollingUpgrade()) { b.setRollingUpgradeStartTime(fsn.getRollingUpgradeInfo().getStartTime()); } NameSystemSection s = b.build(); s.writeDelimitedTo(out); commitSection(summary, SectionName.NS_INFO); }
void format(StorageDirectory sd, NamespaceInfo nsInfo, String datanodeUuid) throws IOException { sd.clearDirectory(); // create directory this.layoutVersion = HdfsConstants.DATANODE_LAYOUT_VERSION; this.clusterID = nsInfo.getClusterID(); this.namespaceID = nsInfo.getNamespaceID(); this.cTime = 0; setDatanodeUuid(datanodeUuid); createStorageID(sd, false); writeProperties(sd); }