public ServerName[] getDataNodes() throws IOException { DistributedFileSystem fs = (DistributedFileSystem) FSUtils.getRootDir(getConf()) .getFileSystem(getConf()); DFSClient dfsClient = fs.getClient(); List<ServerName> hosts = new LinkedList<>(); for (DatanodeInfo dataNode: dfsClient.datanodeReport(HdfsConstants.DatanodeReportType.LIVE)) { hosts.add(ServerName.valueOf(dataNode.getHostName(), -1, -1)); } return hosts.toArray(new ServerName[hosts.size()]); } }
/** @return datanode statistics for the given type. */ public DatanodeInfo[] getDataNodeStats(final DatanodeReportType type ) throws IOException { return dfs.datanodeReport(type); }
/** @return datanode statistics for the given type. */ public DatanodeInfo[] getDataNodeStats(final DatanodeReportType type) throws IOException { return dfs.datanodeReport(type); }
/** Return statistics for each datanode. */ public DatanodeInfo[] getDataNodeStats() throws IOException { return dfs.datanodeReport(DatanodeReportType.ALL); }
/** @return datanode statistics for the given type. */ public DatanodeInfo[] getDataNodeStats(final DatanodeReportType type ) throws IOException { return dfs.datanodeReport(type); }
/** Return statistics for each live datanode. */ public DatanodeInfo[] getLiveDataNodeStats() throws IOException { return dfs.datanodeReport(DatanodeReportType.LIVE); }
/** Return statistics for each datanode. */ public DatanodeInfo[] getDataNodeStats() throws IOException { return dfs.datanodeReport(DatanodeReportType.ALL); }
private String decommissionNode(FSNamesystem namesystem, Configuration conf, DFSClient client, FileSystem localFileSys, int nodeIndex) throws IOException { DatanodeInfo[] info = client.datanodeReport(DatanodeReportType.LIVE); String nodename = info[nodeIndex].getName(); System.out.println("Decommissioning node: " + nodename); // write nodename into the exclude file. ArrayList<String> nodes = new ArrayList<String>(decommissionedNodes); nodes.add(nodename); writeConfigFile(localFileSys, excludeFile, nodes); namesystem.refreshNodes(conf); return nodename; }
private static void validateCluster(DFSClient client, int numDNs) throws IOException { DatanodeInfo[] info = client.datanodeReport(DatanodeReportType.LIVE); assertEquals("Number of Datanodes ", numDNs, info.length); }
private String decommissionNode(FSNamesystem namesystem, Configuration conf, DFSClient client, FileSystem localFileSys) throws IOException { DatanodeInfo[] info = client.datanodeReport(DatanodeReportType.LIVE); // // pick one datanode randomly. // int index = 0; boolean found = false; while (!found) { index = myrand.nextInt(info.length); if (!info[index].isDecommissioned()) { found = true; } } String nodename = info[index].getName(); System.out.println("Decommissioning node: " + nodename); // write nodename into the exclude file. ArrayList<String> nodes = new ArrayList<String>(decommissionedNodes); nodes.add(nodename); writeConfigFile(localFileSys, excludeFile, nodes); namesystem.refreshNodes(conf); return nodename; }
/** * Wait until the cluster is active and running. */ public void waitActive() throws IOException { if (nameNode == null) { return; } InetSocketAddress addr = new InetSocketAddress("localhost", getNameNodePort()); DFSClient client = new DFSClient(addr, conf); // make sure all datanodes have registered and sent heartbeat while (shouldWait(client.datanodeReport(DatanodeReportType.LIVE))) { try { Thread.sleep(100); } catch (InterruptedException e) { } } client.close(); }
private String decommissionNode(FSNamesystem namesystem, DFSClient client, FileSystem localFileSys, int nodeIndex) throws IOException { DatanodeInfo[] info = client.datanodeReport(DatanodeReportType.LIVE); String nodename = info[nodeIndex].getXferAddr(); decommissionNode(namesystem, localFileSys, nodename); return nodename; }
public ServerName[] getDataNodes() throws IOException { DistributedFileSystem fs = (DistributedFileSystem) FSUtils.getRootDir(getConf()) .getFileSystem(getConf()); DFSClient dfsClient = fs.getClient(); List<ServerName> hosts = new LinkedList<>(); for (DatanodeInfo dataNode: dfsClient.datanodeReport(HdfsConstants.DatanodeReportType.LIVE)) { hosts.add(ServerName.valueOf(dataNode.getHostName(), -1, -1)); } return hosts.toArray(new ServerName[hosts.size()]); } }
@Override public Boolean get() { BlockManagerTestUtil .checkHeartbeat(cluster.getNamesystem().getBlockManager()); try { DatanodeInfo info[] = client.datanodeReport(DatanodeReportType.DEAD); return info.length == 1; } catch (IOException e) { LOG.warn("Failed to check dead DNs", e); return false; } } }, 500, 5000);
/** Wait until the given namenode gets registration from all the datanodes */ public void waitActive(int nnIndex) throws IOException { if (nameNodes.length == 0 || nameNodes[nnIndex] == null || nameNodes[nnIndex].nameNode == null) { return; } InetSocketAddress addr = nameNodes[nnIndex].nameNode.getServiceRpcAddress(); assert addr.getPort() != 0; DFSClient client = new DFSClient(addr, conf); // ensure all datanodes have registered and sent heartbeat to the namenode while (shouldWait(client.datanodeReport(DatanodeReportType.LIVE), addr)) { try { LOG.info("Waiting for cluster to become active"); Thread.sleep(100); } catch (InterruptedException e) { } } client.close(); }
@Override public void run() { String msg = ""; try { // find a random datanode from the destination cluster DatanodeInfo[] targets = destFs.getClient().datanodeReport(DatanodeReportType.LIVE); DatanodeInfo target = targets[rand.nextInt(targets.length)]; // find a source datanode from among the datanodes that host this block DatanodeInfo srcdn = goodBlock.getLocations()[rand.nextInt(goodBlock.getLocations().length)]; // The RPC is asynchronous, i.e. the RPC will return immediately even before the // physical block copy occurs from the datanode. msg = "File " + badfile + ": Copying block " + goodBlock.getBlock().getBlockName() + " from " + srcdn.getName() + " to block " + badBlock.getBlock().getBlockName() + " on " + target.getName(); LOG.info(msg); ClientDatanodeProtocol datanode = createClientDatanodeProtocolProxy(srcdn, conf); datanode.copyBlock(goodBlock.getBlock(), badBlock.getBlock(), target); RPC.stopProxy(datanode); HighTideNode.getMetrics().fixSuccessfullyStarted.inc(); } catch (Throwable e) { HighTideNode.getMetrics().fixFailedDatanodeError.inc(); LOG.error(StringUtils.stringifyException(e) + msg + ". Failed to contact datanode."); } } }
cluster.getNameNodePort()), conf); fs = cluster.getFileSystem(); DatanodeInfo dn = dfsClient.datanodeReport(DatanodeReportType.LIVE)[0];
DatanodeInfo[] info = client.datanodeReport(DatanodeReportType.LIVE); for (int i = 0 ; i < 5 && info.length != 0; i++) { LOG.info("Waiting for datanode to be marked dead"); Thread.sleep(HEARTBEAT_INTERVAL * 1000); info = client.datanodeReport(DatanodeReportType.LIVE); info = client.datanodeReport(DatanodeReportType.DEAD); assertEquals("There should be 1 dead node", 1, info.length); assertEquals(bogusIp, info[0].getHostName());
@Override public Boolean get() { BlockManagerTestUtil .checkHeartbeat(cluster.getNamesystem().getBlockManager()); try { DatanodeInfo info[] = client.datanodeReport(DatanodeReportType.LIVE); if (info.length == 1) { Assert.assertFalse(info[0].isDecommissioned()); Assert.assertFalse(info[0].isDecommissionInProgress()); assertEquals(registrationName, info[0].getHostName()); return true; } } catch (IOException e) { LOG.warn("Failed to check dead DNs", e); } return false; } }, 500, 5000);
static void assertReports(int numDatanodes, DatanodeReportType type, DFSClient client, List<DataNode> datanodes, String bpid) throws IOException { final DatanodeInfo[] infos = client.datanodeReport(type); assertEquals(numDatanodes, infos.length); final DatanodeStorageReport[] reports = client.getDatanodeStorageReport(type); assertEquals(numDatanodes, reports.length); for(int i = 0; i < infos.length; i++) { assertEquals(infos[i], reports[i].getDatanodeInfo()); final DataNode d = findDatanode(infos[i].getDatanodeUuid(), datanodes); if (bpid != null) { //check storage final StorageReport[] computed = reports[i].getStorageReports(); Arrays.sort(computed, CMP); final StorageReport[] expected = d.getFSDataset().getStorageReports(bpid); Arrays.sort(expected, CMP); assertEquals(expected.length, computed.length); for(int j = 0; j < expected.length; j++) { assertEquals(expected[j].getStorage().getStorageID(), computed[j].getStorage().getStorageID()); } } } }