public static DatanodeRegistration convert(DatanodeRegistrationProto proto) { StorageInfo si = convert(proto.getStorageInfo(), NodeType.DATA_NODE); return new DatanodeRegistration(PBHelperClient.convert(proto.getDatanodeID()), si, convert(proto.getKeys()), proto.getSoftwareVersion()); }
@Override public String toString() { return getClass().getSimpleName() + "(" + super.toString() + ", datanodeUuid=" + getDatanodeUuid() + ", infoPort=" + getInfoPort() + ", infoSecurePort=" + getInfoSecurePort() + ", ipcPort=" + getIpcPort() + ", storageInfo=" + storageInfo + ")"; }
@VisibleForTesting public DatanodeRegistration(String uuid, DatanodeRegistration dnr) { this(new DatanodeID(uuid, dnr), dnr.getStorageInfo(), dnr.getExportedKeys(), dnr.getSoftwareVersion()); }
/** * After one of the BPServiceActors registers successfully with the * NN, it calls this function to verify that the NN it connected to * is consistent with other NNs serving the block-pool. */ void registrationSucceeded(BPServiceActor bpServiceActor, DatanodeRegistration reg) throws IOException { writeLock(); try { if (bpRegistration != null) { checkNSEquality(bpRegistration.getStorageInfo().getNamespaceID(), reg.getStorageInfo().getNamespaceID(), "namespace ID"); checkNSEquality(bpRegistration.getStorageInfo().getClusterID(), reg.getStorageInfo().getClusterID(), "cluster ID"); } bpRegistration = reg; dn.bpRegistrationSucceeded(bpRegistration, getBlockPoolId()); // Add the initial block token secret keys to the DN's secret manager. if (dn.isBlockTokenEnabled) { dn.blockPoolTokenSecretManager.addKeys(getBlockPoolId(), reg.getExportedKeys()); } } finally { writeUnlock(); } }
nodeReg.setIpAddr(ip); nodeReg.setPeerHostName(hostname); nodeReg.setExportedKeys(blockManager.getBlockKeys()); + nodeReg + " storage " + nodeReg.getDatanodeUuid()); DatanodeDescriptor nodeS = getDatanode(nodeReg.getDatanodeUuid()); DatanodeDescriptor nodeN = host2DatanodeMap.getDatanodeByXferAddr( nodeReg.getIpAddr(), nodeReg.getXferPort()); + nodeReg.getDatanodeUuid()); nodeS.setSoftwareVersion(nodeReg.getSoftwareVersion()); nodeS.setDisallowed(false); // Node is in the include list nodeDescr.setSoftwareVersion(nodeReg.getSoftwareVersion()); resolveUpgradeDomain(nodeDescr); incrementVersionCount(nodeReg.getSoftwareVersion()); startAdminOperationIfNecessary(nodeDescr); success = true; invalidNodeNames.add(nodeReg.getIpAddr()); invalidNodeNames.add(nodeReg.getHostName()); invalidNodeNames.add(nodeReg.getPeerHostName()); dnsToSwitchMapping.reloadCachedMappings(invalidNodeNames); throw e;
dnAddress = nodeReg.getHost(); String hostName = nodeReg.getHost(); DatanodeID dnReg = new DatanodeID(dnAddress + ":" + nodeReg.getPort(), nodeReg.getStorageID(), nodeReg.getInfoPort(), nodeReg.getIpcPort()); nodeReg.updateRegInfo(dnReg); + "node registration from " + nodeReg.getName() + " storage " + nodeReg.getStorageID()); DatanodeDescriptor nodeS = datanodeMap.get(nodeReg.getStorageID()); DatanodeDescriptor nodeN = host2DataNodeMap.getDatanodeByName(nodeReg.getName()); + " is replaced by " + nodeReg.getName() + " with the same storageID " + nodeReg.getStorageID()); if (nodeReg.getStorageID().equals("")) { NameNode.stateChangeLog.debug( "BLOCK* NameSystem.registerDatanode: " + "new storageID " + nodeReg.getStorageID() + " assigned.");
storage = new DataStorage(); this.dnRegistration = new DatanodeRegistration(machineName + ":" + tmpPort); conf.set("StorageId", dnRegistration.getStorageID()); try { this.dnRegistration.setStorageInfo(storage); selfAddr = new InetSocketAddress(ss.getInetAddress().getHostAddress(), tmpPort); this.dnRegistration.setName(machineName + ":" + tmpPort); LOG.info("Opened info server at " + tmpPort); this.infoServer.start(); this.dnRegistration.setInfoPort(this.infoServer.getPort()); myMetrics = new DataNodeMetrics(conf, dnRegistration.getStorageID()); conf.getInt("dfs.datanode.handler.count", 3), false, conf); ipcServer.start(); dnRegistration.setIpcPort(ipcServer.getListenerAddress().getPort());
LOG.info("Removing node " + toRemove.getDatanodeUuid() + " ip " + toRemove.getXferAddr() + " version : " + toRemove.getSoftwareVersion()); Mockito.when(dr.getDatanodeUuid()).thenReturn(storageID); dr.setIpAddr(dr.getIpAddr() + "newIP"); Mockito.when(dr.getIpAddr()).thenReturn(ip); Mockito.when(dr.getXferAddr()).thenReturn(ip + ":9000"); Mockito.when(dr.getXferPort()).thenReturn(9000); Mockito.when(dr.getSoftwareVersion()).thenReturn( "version" + rng.nextInt(5)); LOG.info("Registering node storageID: " + dr.getDatanodeUuid() + ", version: " + dr.getSoftwareVersion() + ", IP address: " + dr.getXferAddr()); String ver = it.getValue().getSoftwareVersion(); if(!mapToCheck.containsKey(ver)) { throw new AssertionError("The correct number of datanodes of a "
doReturn(HdfsConstants.DATANODE_LAYOUT_VERSION).when(mockDnReg).getVersion(); doReturn("127.0.0.1").when(mockDnReg).getIpAddr(); doReturn(123).when(mockDnReg).getXferPort(); doReturn("fake-storage-id").when(mockDnReg).getDatanodeUuid(); doReturn(mockStorageInfo).when(mockDnReg).getStorageInfo(); doReturn("3.0.0").when(mockDnReg).getSoftwareVersion(); rpcServer.registerDatanode(mockDnReg); doReturn("4.0.0").when(mockDnReg).getSoftwareVersion(); rpcServer.registerDatanode(mockDnReg); doReturn("2.0.0").when(mockDnReg).getSoftwareVersion(); try { rpcServer.registerDatanode(mockDnReg);
@Test public void testConvertDatanodeRegistration() { DatanodeID dnId = DFSTestUtil.getLocalDatanodeID(); BlockKey[] keys = new BlockKey[] { getBlockKey(2), getBlockKey(3) }; ExportedBlockKeys expKeys = new ExportedBlockKeys(true, 9, 10, getBlockKey(1), keys); DatanodeRegistration reg = new DatanodeRegistration(dnId, new StorageInfo(NodeType.DATA_NODE), expKeys, "3.0.0"); DatanodeRegistrationProto proto = PBHelper.convert(reg); DatanodeRegistration reg2 = PBHelper.convert(proto); compare(reg.getStorageInfo(), reg2.getStorageInfo()); compare(reg.getExportedKeys(), reg2.getExportedKeys()); compare(reg, reg2); assertEquals(reg.getSoftwareVersion(), reg2.getSoftwareVersion()); }
/** * Transfer blocks to another data-node. * Just report on behalf of the other data-node * that the blocks have been received. */ private int transferBlocks( Block blocks[], DatanodeInfo xferTargets[][] ) throws IOException { for(int i = 0; i < blocks.length; i++) { DatanodeInfo blockTargets[] = xferTargets[i]; for(int t = 0; t < blockTargets.length; t++) { DatanodeInfo dnInfo = blockTargets[t]; DatanodeRegistration receivedDNReg; receivedDNReg = new DatanodeRegistration(dnInfo.getName()); receivedDNReg.setStorageInfo( new DataStorage(nsInfo, dnInfo.getStorageID())); receivedDNReg.setInfoPort(dnInfo.getInfoPort()); nameNode.blockReceived( receivedDNReg, new Block[] {blocks[i]}, new String[] {DataNode.EMPTY_DEL_HINT}); } } return blocks.length; } }
if (nsRegistration.getStorageID().equals("")) { nsRegistration.storageID = createNewStorageId(nsRegistration.getPort()); nsRegistration.setName(machineName + ":" + nsRegistration.getPort()); nsRegistration = nsNamenode.register(nsRegistration, DataTransferProtocol.DATA_TRANSFER_VERSION); && !"".equals(nsRegistration.getStorageID())) || storage.getStorageID().equals(nsRegistration.getStorageID()) : "New storageID can be assigned only if data-node is not formatted"; storage.setStorageID(nsRegistration.getStorageID()); storage.writeAll(); LOG.info("New storage id " + nsRegistration.getStorageID() + " is assigned to data-node " + nsRegistration.getName()); if(! storage.getStorageID().equals(nsRegistration.getStorageID())) { throw new IOException("Inconsistent storage IDs. Name-node returned " + nsRegistration.getStorageID() + ". Expecting " + storage.getStorageID());
@Override public DatanodeRegistration answer(InvocationOnMock invocation) throws Throwable { i++; if ( i > 1 && i < 5) { LOG.info("mockito exception " + i); throw new EOFException("TestDatanodeProtocolRetryPolicy"); } else { DatanodeRegistration dr = (DatanodeRegistration) invocation.getArguments()[0]; datanodeRegistration = new DatanodeRegistration(dr.getDatanodeUuid(), dr); LOG.info("mockito succeeded " + datanodeRegistration); return datanodeRegistration; } } }).when(namenode).registerDatanode(
String getName() { return dnRegistration.getName(); }
if (dnRegistration.getStorageID().equals("")) { setNewStorageID(dnRegistration); try { dnRegistration.name = machineName + ":" + dnRegistration.getPort(); dnRegistration = namenode.register(dnRegistration); break; && !"".equals(dnRegistration.getStorageID())) || storage.getStorageID().equals(dnRegistration.getStorageID()) : "New storageID can be assigned only if data-node is not formatted"; if (storage.getStorageID().equals("")) { storage.setStorageID(dnRegistration.getStorageID()); storage.writeAll(); LOG.info("New storage id " + dnRegistration.getStorageID() + " is assigned to data-node " + dnRegistration.getName()); if(! storage.getStorageID().equals(dnRegistration.getStorageID())) { throw new IOException("Inconsistent storage IDs. Name-node returned " + dnRegistration.getStorageID() + ". Expecting " + storage.getStorageID());
public void testDataNodeRedirect() throws Exception { if (hdfs.exists(TEST_FILE)) { hdfs.delete(TEST_FILE, true); } FSDataOutputStream out = hdfs.create(TEST_FILE, (short) 1); out.writeBytes("0123456789"); out.close(); BlockLocation[] locations = hdfs.getFileBlockLocations(TEST_FILE, 0, 10); String locationName = locations[0].getNames()[0]; URL u = hftpFs.getNamenodeFileURL(TEST_FILE); HttpURLConnection conn = (HttpURLConnection)u.openConnection(); conn.setFollowRedirects(true); conn.connect(); conn.getInputStream(); boolean checked = false; // Find the datanode that has the block according to locations // and check that the URL was redirected to this DN's info port for (DataNode node : cluster.getDataNodes()) { if (node.getDatanodeRegistration().getName().equals(locationName)) { checked = true; assertEquals(node.getDatanodeRegistration().getInfoPort(), conn.getURL().getPort()); } } assertTrue("The test never checked that location of " + "the block and hftp desitnation are the same", checked); } /**
/** * Check that the registration returned from a NameNode is consistent * with the information in the storage. If the storage is fresh/unformatted, * sets the storage ID based on this registration. * Also updates the block pool's state in the secret manager. */ synchronized void bpRegistrationSucceeded(DatanodeRegistration bpRegistration, String blockPoolId) throws IOException { id = bpRegistration; if(!storage.getDatanodeUuid().equals(bpRegistration.getDatanodeUuid())) { throw new IOException("Inconsistent Datanode IDs. Name-node returned " + bpRegistration.getDatanodeUuid() + ". Expecting " + storage.getDatanodeUuid()); } registerBlockPoolWithSecretManager(bpRegistration, blockPoolId); }
private void verifySoftwareVersion(DatanodeRegistration dnReg) throws IncorrectVersionException { String dnVersion = dnReg.getSoftwareVersion(); if (VersionUtil.compareVersions(dnVersion, minimumDataNodeVersion) < 0) { IncorrectVersionException ive = new IncorrectVersionException( minimumDataNodeVersion, dnVersion, "DataNode", "NameNode"); LOG.warn(ive.getMessage() + " DN: " + dnReg); throw ive; } String nnVersion = VersionInfo.getVersion(); if (!dnVersion.equals(nnVersion)) { String messagePrefix = "Reported DataNode version '" + dnVersion + "' of DN " + dnReg + " does not match NameNode version '" + nnVersion + "'"; long nnCTime = nn.getFSImage().getStorage().getCTime(); long dnCTime = dnReg.getStorageInfo().getCTime(); if (nnCTime != dnCTime) { IncorrectVersionException ive = new IncorrectVersionException( messagePrefix + " and CTime of DN ('" + dnCTime + "') does not match CTime of NN ('" + nnCTime + "')"); LOG.warn(ive.toString(), ive); throw ive; } else { LOG.info(messagePrefix + ". Note: This is normal during a rolling upgrade."); } } }
DatanodeRegistration dnR = DataNodeTestUtils.getDNRegistrationForBP(node, blockPoolId); if (dnR.getXferAddr().equals(xferAddr)) { checked = true; assertEquals(dnR.getInfoPort(), conn.getURL().getPort());
void register() throws IOException { // get versions from the namenode nsInfo = nameNodeProto.versionRequest(); dnRegistration = new DatanodeRegistration( new DatanodeID(DNS.getDefaultIP("default"), DNS.getDefaultHost("default", "default"), DataNode.generateUuid(), getNodePort(dnIdx), DFSConfigKeys.DFS_DATANODE_HTTP_DEFAULT_PORT, DFSConfigKeys.DFS_DATANODE_HTTPS_DEFAULT_PORT, DFSConfigKeys.DFS_DATANODE_IPC_DEFAULT_PORT), new DataStorage(nsInfo), new ExportedBlockKeys(), VersionInfo.getVersion()); // register datanode dnRegistration = dataNodeProto.registerDatanode(dnRegistration); dnRegistration.setNamespaceInfo(nsInfo); //first block reports storage = new DatanodeStorage(DatanodeStorage.generateUuid()); final StorageBlockReport[] reports = { new StorageBlockReport(storage, BlockListAsLongs.EMPTY) }; dataNodeProto.blockReport(dnRegistration, bpid, reports, new BlockReportContext(1, 0, System.nanoTime())); }