public static DatanodeRegistration convert(DatanodeRegistrationProto proto) { StorageInfo si = convert(proto.getStorageInfo(), NodeType.DATA_NODE); return new DatanodeRegistration(PBHelperClient.convert(proto.getDatanodeID()), si, convert(proto.getKeys()), proto.getSoftwareVersion()); }
/** * Create a DatanodeRegistration for a specific block pool. * @param nsInfo the namespace info from the first part of the NN handshake */ DatanodeRegistration createBPRegistration(NamespaceInfo nsInfo) { StorageInfo storageInfo = storage.getBPStorage(nsInfo.getBlockPoolID()); if (storageInfo == null) { // it's null in the case of SimulatedDataSet storageInfo = new StorageInfo( DataNodeLayoutVersion.CURRENT_LAYOUT_VERSION, nsInfo.getNamespaceID(), nsInfo.clusterID, nsInfo.getCTime(), NodeType.DATA_NODE); } DatanodeID dnId = new DatanodeID( streamingAddr.getAddress().getHostAddress(), hostName, storage.getDatanodeUuid(), getXferPort(), getInfoPort(), infoSecurePort, getIpcPort()); return new DatanodeRegistration(dnId, storageInfo, new ExportedBlockKeys(), VersionInfo.getVersion()); }
public Writable newInstance() { return new DatanodeRegistration(); } });
public Writable newInstance() { return new DatanodeRegistration(); } });
TinyDatanode(int dnIdx, int blockCapacity) throws IOException { dnRegistration = new DatanodeRegistration(getNodeName(dnIdx)); this.blocks = new ArrayList<Block>(blockCapacity); this.nrBlocks = 0; }
NSOfferService(InetSocketAddress isa, String nameserviceId) { this.nsRegistration = new DatanodeRegistration(getMachineName()); this.nnAddr = isa; this.nameserviceId = nameserviceId; }
@Override public DatanodeRegistration answer(InvocationOnMock invocation) throws Throwable { i++; if ( i > 1 && i < 5) { LOG.info("mockito exception " + i); throw new EOFException("TestDatanodeProtocolRetryPolicy"); } else { DatanodeRegistration dr = (DatanodeRegistration) invocation.getArguments()[0]; datanodeRegistration = new DatanodeRegistration(dr.getDatanodeUuid(), dr); LOG.info("mockito succeeded " + datanodeRegistration); return datanodeRegistration; } } }).when(namenode).registerDatanode(
public static DatanodeRegistration convert(DatanodeRegistrationProto proto) { StorageInfo si = convert(proto.getStorageInfo(), NodeType.DATA_NODE); return new DatanodeRegistration(PBHelper.convert(proto.getDatanodeID()), si, PBHelper.convert(proto.getKeys()), proto.getSoftwareVersion()); }
public static DatanodeRegistration convert(DatanodeRegistrationProto proto) { StorageInfo si = convert(proto.getStorageInfo(), NodeType.DATA_NODE); return new DatanodeRegistration(PBHelper.convert(proto.getDatanodeID()), si, PBHelper.convert(proto.getKeys()), proto.getSoftwareVersion()); }
/** * Create a DatanodeRegistration for a specific block pool. * @param nsInfo the namespace info from the first part of the NN handshake */ DatanodeRegistration createBPRegistration(NamespaceInfo nsInfo) { StorageInfo storageInfo = storage.getBPStorage(nsInfo.getBlockPoolID()); if (storageInfo == null) { // it's null in the case of SimulatedDataSet storageInfo = new StorageInfo( DataNodeLayoutVersion.CURRENT_LAYOUT_VERSION, nsInfo.getNamespaceID(), nsInfo.clusterID, nsInfo.getCTime(), NodeType.DATA_NODE); } DatanodeID dnId = new DatanodeID( streamingAddr.getAddress().getHostAddress(), hostName, storage.getDatanodeUuid(), getXferPort(), getInfoPort(), infoSecurePort, getIpcPort()); return new DatanodeRegistration(dnId, storageInfo, new ExportedBlockKeys(), VersionInfo.getVersion()); }
dm.registerDatanode(new DatanodeRegistration( new DatanodeID(ip, "", storageID, 9000, 0, 0, 0), null, null, "version" + i));
/** * Create a DatanodeRegistration for a specific block pool. * @param nsInfo the namespace info from the first part of the NN handshake */ DatanodeRegistration createBPRegistration(NamespaceInfo nsInfo) { StorageInfo storageInfo = storage.getBPStorage(nsInfo.getBlockPoolID()); if (storageInfo == null) { // it's null in the case of SimulatedDataSet storageInfo = new StorageInfo( DataNodeLayoutVersion.CURRENT_LAYOUT_VERSION, nsInfo.getNamespaceID(), nsInfo.clusterID, nsInfo.getCTime(), NodeType.DATA_NODE); } DatanodeID dnId = new DatanodeID( streamingAddr.getAddress().getHostAddress(), hostName, storage.getDatanodeUuid(), getXferPort(), getInfoPort(), infoSecurePort, getIpcPort()); return new DatanodeRegistration(dnId, storageInfo, new ExportedBlockKeys(), VersionInfo.getVersion()); }
/** * Transfer blocks to another data-node. * Just report on behalf of the other data-node * that the blocks have been received. */ private int transferBlocks( Block blocks[], DatanodeInfo xferTargets[][] ) throws IOException { for(int i = 0; i < blocks.length; i++) { DatanodeInfo blockTargets[] = xferTargets[i]; for(int t = 0; t < blockTargets.length; t++) { DatanodeInfo dnInfo = blockTargets[t]; DatanodeRegistration receivedDNReg; receivedDNReg = new DatanodeRegistration(dnInfo.getName()); receivedDNReg.setStorageInfo( new DataStorage(nsInfo, dnInfo.getStorageID())); receivedDNReg.setInfoPort(dnInfo.getInfoPort()); nameNode.blockReceived( receivedDNReg, new Block[] {blocks[i]}, new String[] {DataNode.EMPTY_DEL_HINT}); } } return blocks.length; } }
public static DatanodeRegistration getLocalDatanodeRegistration() { return new DatanodeRegistration(getLocalDatanodeID(), new StorageInfo( NodeType.DATA_NODE), new ExportedBlockKeys(), VersionInfo.getVersion()); }
@Test public void testSafeModeIBRAfterIncremental() throws Exception { DatanodeDescriptor node = spy(nodes.get(0)); DatanodeStorageInfo ds = node.getStorageInfos()[0]; node.isAlive = true; DatanodeRegistration nodeReg = new DatanodeRegistration(node, null, null, ""); // pretend to be in safemode doReturn(true).when(fsn).isInStartupSafeMode(); // register new node bm.getDatanodeManager().registerDatanode(nodeReg); bm.getDatanodeManager().addDatanode(node); // swap in spy assertEquals(node, bm.getDatanodeManager().getDatanode(node)); assertEquals(0, ds.getBlockReportCount()); // send block report while pretending to already have blocks reset(node); doReturn(1).when(node).numBlocks(); bm.processReport(node, new DatanodeStorage(ds.getStorageID()), BlockListAsLongs.EMPTY, null); assertEquals(1, ds.getBlockReportCount()); }
/** * Transfer blocks to another data-node. * Just report on behalf of the other data-node * that the blocks have been received. */ private int transferBlocks( Block blocks[], DatanodeInfo xferTargets[][], String targetStorageIDs[][] ) throws IOException { for(int i = 0; i < blocks.length; i++) { DatanodeInfo blockTargets[] = xferTargets[i]; for(int t = 0; t < blockTargets.length; t++) { DatanodeInfo dnInfo = blockTargets[t]; String targetStorageID = targetStorageIDs[i][t]; DatanodeRegistration receivedDNReg; receivedDNReg = new DatanodeRegistration(dnInfo, new DataStorage(nsInfo), new ExportedBlockKeys(), VersionInfo.getVersion()); ReceivedDeletedBlockInfo[] rdBlocks = { new ReceivedDeletedBlockInfo( blocks[i], ReceivedDeletedBlockInfo.BlockStatus.RECEIVED_BLOCK, null) }; StorageReceivedDeletedBlocks[] report = { new StorageReceivedDeletedBlocks( targetStorageID, rdBlocks) }; dataNodeProto.blockReceivedAndDeleted(receivedDNReg, bpid, report); } } return blocks.length; } }
/** * Reformatted DataNodes will replace the original UUID in the * {@link DatanodeManager#datanodeMap}. This tests if block * invalidation work on the original DataNode can be skipped. */ @Test(timeout=120000) public void testDatanodeReformat() throws Exception { namesystem.writeLock(); try { // Change the datanode UUID to emulate a reformat String poolId = cluster.getNamesystem().getBlockPoolId(); DatanodeRegistration dnr = cluster.getDataNode(nodes[0].getIpcPort()) .getDNRegistrationForBP(poolId); dnr = new DatanodeRegistration(UUID.randomUUID().toString(), dnr); cluster.stopDataNode(nodes[0].getXferAddr()); Block block = new Block(0, 0, GenerationStamp.LAST_RESERVED_STAMP); bm.addToInvalidates(block, nodes[0]); bm.getDatanodeManager().registerDatanode(dnr); // Since UUID has changed, the invalidation work should be skipped assertEquals(0, bm.computeInvalidateWork(1)); assertEquals(0, bm.getPendingDeletionBlocksCount()); } finally { namesystem.writeUnlock(); } }
void register() throws IOException { // get versions from the namenode nsInfo = nameNodeProto.versionRequest(); dnRegistration = new DatanodeRegistration( new DatanodeID(DNS.getDefaultIP("default"), DNS.getDefaultHost("default", "default"), DataNode.generateUuid(), getNodePort(dnIdx), DFSConfigKeys.DFS_DATANODE_HTTP_DEFAULT_PORT, DFSConfigKeys.DFS_DATANODE_HTTPS_DEFAULT_PORT, DFSConfigKeys.DFS_DATANODE_IPC_DEFAULT_PORT), new DataStorage(nsInfo), new ExportedBlockKeys(), VersionInfo.getVersion()); // register datanode dnRegistration = dataNodeProto.registerDatanode(dnRegistration); dnRegistration.setNamespaceInfo(nsInfo); //first block reports storage = new DatanodeStorage(DatanodeStorage.generateUuid()); final StorageBlockReport[] reports = { new StorageBlockReport(storage, BlockListAsLongs.EMPTY) }; dataNodeProto.blockReport(dnRegistration, bpid, reports, new BlockReportContext(1, 0, System.nanoTime())); }
DatanodeRegistration reg = new DatanodeRegistration( new DatanodeID(UUID.randomUUID().toString(), did), new StorageInfo(HdfsServerConstants.NodeType.DATA_NODE),
@Test public void testConvertDatanodeRegistration() { DatanodeID dnId = DFSTestUtil.getLocalDatanodeID(); BlockKey[] keys = new BlockKey[] { getBlockKey(2), getBlockKey(3) }; ExportedBlockKeys expKeys = new ExportedBlockKeys(true, 9, 10, getBlockKey(1), keys); DatanodeRegistration reg = new DatanodeRegistration(dnId, new StorageInfo(NodeType.DATA_NODE), expKeys, "3.0.0"); DatanodeRegistrationProto proto = PBHelper.convert(reg); DatanodeRegistration reg2 = PBHelper.convert(proto); compare(reg.getStorageInfo(), reg2.getStorageInfo()); compare(reg.getExportedKeys(), reg2.getExportedKeys()); compare(reg, reg2); assertEquals(reg.getSoftwareVersion(), reg2.getSoftwareVersion()); }