private boolean checkLayoutVersion(NamespaceInfo nsInfo) throws IOException { return (nsInfo.getLayoutVersion() == HdfsServerConstants.NAMENODE_LAYOUT_VERSION); }
/** * Format all available storage directories. */ public void format(NamespaceInfo nsInfo) throws IOException { Preconditions.checkArgument(nsInfo.getLayoutVersion() == 0 || nsInfo.getLayoutVersion() == HdfsServerConstants.NAMENODE_LAYOUT_VERSION, "Bad layout version: %s", nsInfo.getLayoutVersion()); this.setStorageInfo(nsInfo); this.blockpoolID = nsInfo.getBlockPoolID(); for (Iterator<StorageDirectory> it = dirIterator(); it.hasNext();) { StorageDirectory sd = it.next(); format(sd); } }
private static NamespaceInfo handshake(NamenodeProtocol namenode) throws IOException, SocketTimeoutException { NamespaceInfo nsInfo; nsInfo = namenode.versionRequest(); // throws SocketTimeoutException String errorMsg = null; // verify build version if( ! nsInfo.getBuildVersion().equals( Storage.getBuildVersion())) { errorMsg = "Incompatible build versions: active name-node BV = " + nsInfo.getBuildVersion() + "; backup node BV = " + Storage.getBuildVersion(); LOG.error(errorMsg); throw new IOException(errorMsg); } assert HdfsServerConstants.NAMENODE_LAYOUT_VERSION == nsInfo.getLayoutVersion() : "Active and backup node layout versions must be the same. Expected: " + HdfsServerConstants.NAMENODE_LAYOUT_VERSION + " actual "+ nsInfo.getLayoutVersion(); return nsInfo; }
LOG.error("Layout version on remote node (" + nsInfo.getLayoutVersion() + ") does not match " + "this node's layout version (" + HdfsServerConstants.NAMENODE_LAYOUT_VERSION + ")"); " Block pool ID: " + nsInfo.getBlockPoolID() + "\n" + " Cluster ID: " + nsInfo.getClusterID() + "\n" + " Layout version: " + nsInfo.getLayoutVersion() + "\n" + " isUpgradeFinalized: " + isUpgradeFinalized + "\n" + "=====================================================");
+ " CTime = " + this.getCTime() + " is newer than the namespace state: LV = " + nsInfo.getLayoutVersion() + " CTime = " + nsInfo.getCTime());
+ "CTime = {}", bpSd.getRoot(), nsInfo.getLayoutVersion(), nsInfo.getCTime()); File tmpDir = bpSd.getRemovedTmp();
private boolean checkLayoutVersion(NamespaceInfo nsInfo) throws IOException { return (nsInfo.getLayoutVersion() == HdfsConstants.NAMENODE_LAYOUT_VERSION); }
private boolean checkLayoutVersion(NamespaceInfo nsInfo) throws IOException { return (nsInfo.getLayoutVersion() == HdfsConstants.NAMENODE_LAYOUT_VERSION); }
/** * Format all available storage directories. */ public void format(NamespaceInfo nsInfo) throws IOException { Preconditions.checkArgument(nsInfo.getLayoutVersion() == 0 || nsInfo.getLayoutVersion() == HdfsConstants.NAMENODE_LAYOUT_VERSION, "Bad layout version: %s", nsInfo.getLayoutVersion()); this.setStorageInfo(nsInfo); this.blockpoolID = nsInfo.getBlockPoolID(); for (Iterator<StorageDirectory> it = dirIterator(); it.hasNext();) { StorageDirectory sd = it.next(); format(sd); } }
/** * Format all available storage directories. */ public void format(NamespaceInfo nsInfo) throws IOException { Preconditions.checkArgument(nsInfo.getLayoutVersion() == 0 || nsInfo.getLayoutVersion() == HdfsConstants.NAMENODE_LAYOUT_VERSION, "Bad layout version: %s", nsInfo.getLayoutVersion()); this.setStorageInfo(nsInfo); this.blockpoolID = nsInfo.getBlockPoolID(); for (Iterator<StorageDirectory> it = dirIterator(); it.hasNext();) { StorageDirectory sd = it.next(); format(sd); } }
/** * Analyze storage directories for a specific block pool. * Recover from previous transitions if required. * Perform fs state transition if necessary depending on the namespace info. * Read storage info. * <br> * This method should be synchronized between multiple DN threads. Only the * first DN thread does DN level storage dir recoverTransitionRead. * * @param datanode DataNode * @param nsInfo Namespace info of namenode corresponding to the block pool * @param dataDirs Storage directories * @param startOpt startup option * @throws IOException on error */ void recoverTransitionRead(DataNode datanode, NamespaceInfo nsInfo, Collection<StorageLocation> dataDirs, StartupOption startOpt) throws IOException { if (this.initialized) { LOG.info("DataNode version: " + HdfsConstants.DATANODE_LAYOUT_VERSION + " and NameNode layout version: " + nsInfo.getLayoutVersion()); this.storageDirs = new ArrayList<StorageDirectory>(dataDirs.size()); // mark DN storage is initialized this.initialized = true; } if (addStorageLocations(datanode, nsInfo, dataDirs, startOpt).isEmpty()) { throw new IOException("All specified directories are failed to load."); } }
/** * Analyze storage directories for a specific block pool. * Recover from previous transitions if required. * Perform fs state transition if necessary depending on the namespace info. * Read storage info. * <br> * This method should be synchronized between multiple DN threads. Only the * first DN thread does DN level storage dir recoverTransitionRead. * * @param datanode DataNode * @param nsInfo Namespace info of namenode corresponding to the block pool * @param dataDirs Storage directories * @param startOpt startup option * @throws IOException on error */ void recoverTransitionRead(DataNode datanode, NamespaceInfo nsInfo, Collection<StorageLocation> dataDirs, StartupOption startOpt) throws IOException { if (this.initialized) { LOG.info("DataNode version: " + HdfsConstants.DATANODE_LAYOUT_VERSION + " and NameNode layout version: " + nsInfo.getLayoutVersion()); this.storageDirs = new ArrayList<StorageDirectory>(dataDirs.size()); // mark DN storage is initialized this.initialized = true; } if (addStorageLocations(datanode, nsInfo, dataDirs, startOpt).isEmpty()) { throw new IOException("All specified directories are failed to load."); } }
public void write(DataOutput out) throws IOException { UTF8.writeString(out, getBuildVersion()); out.writeInt(getLayoutVersion()); out.writeInt(getNamespaceID()); out.writeLong(getCTime()); out.writeInt(getDistributedUpgradeVersion()); }
public void write(DataOutput out) throws IOException { UTF8.writeString(out, getBuildVersion()); out.writeInt(getLayoutVersion()); out.writeInt(getNamespaceID()); out.writeLong(getCTime()); out.writeInt(getDistributedUpgradeVersion()); }
private static NamespaceInfo handshake(NamenodeProtocol namenode) throws IOException, SocketTimeoutException { NamespaceInfo nsInfo; nsInfo = namenode.versionRequest(); // throws SocketTimeoutException String errorMsg = null; // verify build version if( ! nsInfo.getBuildVersion().equals( Storage.getBuildVersion())) { errorMsg = "Incompatible build versions: active name-node BV = " + nsInfo.getBuildVersion() + "; backup node BV = " + Storage.getBuildVersion(); LOG.error(errorMsg); throw new IOException(errorMsg); } assert HdfsConstants.NAMENODE_LAYOUT_VERSION == nsInfo.getLayoutVersion() : "Active and backup node layout versions must be the same. Expected: " + HdfsConstants.NAMENODE_LAYOUT_VERSION + " actual "+ nsInfo.getLayoutVersion(); return nsInfo; }
private static NamespaceInfo handshake(NamenodeProtocol namenode) throws IOException, SocketTimeoutException { NamespaceInfo nsInfo; nsInfo = namenode.versionRequest(); // throws SocketTimeoutException String errorMsg = null; // verify build version if( ! nsInfo.getBuildVersion().equals( Storage.getBuildVersion())) { errorMsg = "Incompatible build versions: active name-node BV = " + nsInfo.getBuildVersion() + "; backup node BV = " + Storage.getBuildVersion(); LOG.error(errorMsg); throw new IOException(errorMsg); } assert HdfsConstants.NAMENODE_LAYOUT_VERSION == nsInfo.getLayoutVersion() : "Active and backup node layout versions must be the same. Expected: " + HdfsConstants.NAMENODE_LAYOUT_VERSION + " actual "+ nsInfo.getLayoutVersion(); return nsInfo; }
synchronized Collection<StorageDirectory> recoverTransitionAdditionalRead(NamespaceInfo nsInfo, Collection<File> dataDirs, StartupOption startOpt ) throws IOException{ assert FSConstants.LAYOUT_VERSION == nsInfo.getLayoutVersion() : "Data-node and name-node layout versions must be the same."; // 1. For each data directory calculate its state and // check whether all is consistent before transitioning. // Format and recover. ArrayList<StorageDirectory> newDirs = analyzeStorageDirs(nsInfo, dataDirs, startOpt); // 2. Do transitions // Each storage directory is treated individually. // During startup some of them can upgrade or rollback // while others could be uptodate for the regular startup. doTransition(newDirs, nsInfo, startOpt); assert this.getLayoutVersion() == nsInfo.getLayoutVersion() : "Data-node and name-node layout versions must be the same."; assert this.getCTime() == nsInfo.getCTime() : "Data-node and name-node CTimes must be the same."; // 3. Update all storages. Some of them might have just been formatted. if (this.layoutVersion == 0) { layoutVersion = FSConstants.LAYOUT_VERSION; } for (StorageDirectory sd : newDirs) { sd.write(); } return newDirs; }
+ ".\n old LV = " + si.getLayoutVersion() + "; old CTime = " + si.getCTime() + ".\n new LV = " + nsInfo.getLayoutVersion() + "; new CTime = " + nsInfo.getCTime()); File curDir = sd.getCurrentDir(); upgrade(si.getLayoutVersion(), nsInfo.getLayoutVersion(), tmpDir, curDir); } catch (Throwable t) {
@Test public void testDifferentLayoutVersions() throws Exception { // We expect no exceptions to be thrown when the layout versions match. assertEquals(HdfsConstants.NAMENODE_LAYOUT_VERSION, actor.retrieveNamespaceInfo().getLayoutVersion()); // We expect an exception to be thrown when the NN reports a layout version // different from that of the DN. doReturn(HdfsConstants.NAMENODE_LAYOUT_VERSION * 1000).when(fakeNsInfo) .getLayoutVersion(); try { actor.retrieveNamespaceInfo(); } catch (IOException e) { fail("Should not fail to retrieve NS info from DN with different layout version"); } } }
@Before public void setUp() throws IOException { mockDnConf = mock(DNConf.class); doReturn(VersionInfo.getVersion()).when(mockDnConf).getMinimumNameNodeVersion(); DataNode mockDN = mock(DataNode.class); doReturn(true).when(mockDN).shouldRun(); doReturn(mockDnConf).when(mockDN).getDnConf(); BPOfferService mockBPOS = mock(BPOfferService.class); doReturn(mockDN).when(mockBPOS).getDataNode(); actor = new BPServiceActor(INVALID_ADDR, mockBPOS); fakeNsInfo = mock(NamespaceInfo.class); // Return a a good software version. doReturn(VersionInfo.getVersion()).when(fakeNsInfo).getSoftwareVersion(); // Return a good layout version for now. doReturn(HdfsConstants.NAMENODE_LAYOUT_VERSION).when(fakeNsInfo) .getLayoutVersion(); DatanodeProtocolClientSideTranslatorPB fakeDnProt = mock(DatanodeProtocolClientSideTranslatorPB.class); when(fakeDnProt.versionRequest()).thenReturn(fakeNsInfo); actor.setNameNode(fakeDnProt); }