@Before public void setup() throws Exception { Configuration hdfsConf = new HdfsConfiguration(); hdfsConf.setLong(DFSConfigKeys.DFS_NAMENODE_MIN_BLOCK_SIZE_KEY, 0); String namenodeDir = new File(MiniDFSCluster.getBaseDirectory(), "name"). getAbsolutePath(); hdfsConf.set(DFSConfigKeys.DFS_NAMENODE_NAME_DIR_KEY, namenodeDir); hdfsConf.set(DFSConfigKeys.DFS_NAMENODE_EDITS_DIR_KEY, namenodeDir); hdfsConf.setLong(DFSConfigKeys.DFS_BLOCK_SIZE_KEY, BLOCKSIZE); cluster = new MiniDFSCluster.Builder(hdfsConf).numDataNodes(15).build(); fs = cluster.getFileSystem(); fs.enableErasureCodingPolicy(ecPolicy.getName()); fs.setErasureCodingPolicy(new Path("/"), ecPolicy.getName()); cluster.waitActive(); conf = new Configuration(); submitDir = new Path("/"); testFile = new Path("/testfile"); DFSTestUtil.writeFile(fs, testFile, StripedFileTestUtil.generateBytes(BLOCKSIZE)); conf.set(FileInputFormat.INPUT_DIR, fs.getUri().toString() + testFile.toString()); }
private static void testViewingFile(MiniDFSCluster cluster, String filePath, boolean doTail) throws IOException { FileSystem fs = cluster.getFileSystem(); Path testPath = new Path(filePath); if (!fs.exists(testPath)) { DFSTestUtil.writeFile(fs, testPath, FILE_DATA); } InetSocketAddress nnHttpAddress = cluster.getNameNode().getHttpAddress(); int dnInfoPort = cluster.getDataNodes().get(0).getInfoPort(); String jspName = doTail ? "tail.jsp" : "browseDirectory.jsp"; String fileParamName = doTail ? "filename" : "dir"; URL url = new URL("http://localhost:" + dnInfoPort + "/" + jspName + JspHelper.getUrlParam(fileParamName, URLEncoder.encode(testPath.toString(), "UTF-8"), true) + JspHelper.getUrlParam("namenodeInfoPort", Integer.toString(nnHttpAddress.getPort()))); String viewFilePage = DFSTestUtil.urlGet(url); assertTrue("page should show preview of file contents", viewFilePage.contains(FILE_DATA)); if (!doTail) { assertTrue("page should show link to download file", viewFilePage .contains("/streamFile" + URIUtil.encodePath(testPath.toString()))); } }
private void writeUsingBothNameNodes() throws ServiceFailedException, IOException, URISyntaxException { cluster.transitionToActive(0); FileSystem fs = HATestUtil.configureFailoverFs(cluster, conf); DFSTestUtil.writeFile(fs, TEST_PATH, TEST_DATA); cluster.transitionToStandby(0); cluster.transitionToActive(1); fs.delete(TEST_PATH, false); DFSTestUtil.writeFile(fs, TEST_PATH, TEST_DATA); }
Path excludeFile = new Path(dir, "exclude"); assertTrue(localFileSys.mkdirs(dir)); DFSTestUtil.writeFile(localFileSys, excludeFile, ""); conf.set("dfs.hosts.exclude", excludeFile.toUri().getPath()); fs.getFileStatus(filePath), 0, Long.MAX_VALUE); String name = locs[0].getNames()[0]; DFSTestUtil.writeFile(localFileSys, excludeFile, name); ns.refreshNodes(conf); DFSTestUtil.waitForDecommission(fs, name);
Path excludeFile = new Path(dir, "exclude"); assertTrue(localFileSys.mkdirs(dir)); DFSTestUtil.writeFile(localFileSys, excludeFile, ""); conf.set("dfs.hosts.exclude", excludeFile.toUri().getPath()); if (!top.startsWith("/rack2")) { String name = top.substring("/rack1".length()+1); DFSTestUtil.writeFile(localFileSys, excludeFile, name); ns.refreshNodes(conf); DFSTestUtil.waitForDecommission(fs, name);
includeHosts.append("localhost:52").append("\n").append("127.0.0.1:7777") .append("\n"); DFSTestUtil.writeFile(localFileSys, excludeFile, ""); DFSTestUtil.writeFile(localFileSys, includeFile, includeHosts.toString()); conf.set(DFSConfigKeys.DFS_HOSTS_EXCLUDE, excludeFile.toUri().getPath()); conf.set(DFSConfigKeys.DFS_HOSTS, includeFile.toUri().getPath());
DFSTestUtil.writeFile(fileSys, myFile, "hello world");
cluster.transitionToActive(nn1); assertTrue(fs.exists(TEST_DIR)); DFSTestUtil.writeFile(fs, TEST_FILE_PATH, TEST_FILE_DATA);
Path includeFile = new Path(dir, "include"); assertTrue(localFileSys.mkdirs(dir)); DFSTestUtil.writeFile(localFileSys, excludeFile, ""); DFSTestUtil.writeFile(localFileSys, includeFile, ""); conf.set(DFSConfigKeys.DFS_HOSTS_EXCLUDE, excludeFile.toUri().getPath()); conf.set(DFSConfigKeys.DFS_HOSTS, includeFile.toUri().getPath()); String names = name + "\n" + "localhost:42\n"; LOG.info("adding '" + names + "' to exclude file " + excludeFile.toUri().getPath()); DFSTestUtil.writeFile(localFileSys, excludeFile, name); ns.getBlockManager().getDatanodeManager().refreshNodes(conf); DFSTestUtil.waitForDecommission(fs, name);
includeHosts.append(dn.getDisplayName()).append("\n"); DFSTestUtil.writeFile(localFileSys, includeFile, includeHosts.toString()); conf.set(DFSConfigKeys.DFS_HOSTS, includeFile.toUri().getPath()); fsn.getBlockManager().getDatanodeManager().refreshNodes(conf);
Path includeFile = new Path(dir, "include"); assertTrue(localFileSys.mkdirs(dir)); DFSTestUtil.writeFile(localFileSys, excludeFile, ""); DFSTestUtil.writeFile(localFileSys, includeFile, ""); conf.set(DFSConfigKeys.DFS_HOSTS, includeFile.toUri().getPath()); conf.set(DFSConfigKeys.DFS_HOSTS_EXCLUDE, excludeFile.toUri().getPath()); if (!top.startsWith("/rack2")) { String name = top.substring("/rack1".length()+1); DFSTestUtil.writeFile(localFileSys, excludeFile, name); ns.getBlockManager().getDatanodeManager().refreshNodes(conf); DFSTestUtil.waitForDecommission(fs, name);
Path includeFile = new Path(dir, "include"); assertTrue(localFileSys.mkdirs(dir)); DFSTestUtil.writeFile(localFileSys, excludeFile, ""); DFSTestUtil.writeFile(localFileSys, includeFile, ""); conf.set(DFSConfigKeys.DFS_HOSTS_EXCLUDE, excludeFile.toUri().getPath()); conf.set(DFSConfigKeys.DFS_HOSTS, includeFile.toUri().getPath()); fs.getFileStatus(filePath), 0, Long.MAX_VALUE); String name = locs[0].getNames()[0]; DFSTestUtil.writeFile(localFileSys, excludeFile, name); ns.getBlockManager().getDatanodeManager().refreshNodes(conf); DFSTestUtil.waitForDecommission(fs, name);
final Path p = new Path("/test"); String data = "testingmissingblock"; DFSTestUtil.writeFile(dfs, p, data);
DFSTestUtil.writeFile(localFs, includeFile, Joiner.on("\n").join(dnAddresses)); DFSTestUtil.writeFile(localFs, excludeFile, ""); bm.getDatanodeManager().refreshNodes(CONF); assertGauge("NumDecomLiveDataNodes", 0, getMetrics(NS_METRICS)); DFSTestUtil.writeFile(localFs, excludeFile, dnAddresses[0]); bm.getDatanodeManager().refreshNodes(CONF); assertGauge("NumDecommissioningDataNodes", 1, getMetrics(NS_METRICS)); includeHosts[i] = dnAddresses[i + 1]; DFSTestUtil.writeFile(localFs, includeFile, Joiner.on("\n").join(includeHosts)); DFSTestUtil.writeFile(localFs, excludeFile, ""); bm.getDatanodeManager().refreshNodes(CONF); assertGauge("NumDecomLiveDataNodes", 0, getMetrics(NS_METRICS));
FileSystem fs = HATestUtil.configureFailoverFs(cluster, conf); DFSTestUtil.writeFile(fs, TEST_PATH, TEST_DATA); assertEquals(TEST_DATA, DFSTestUtil.readFile(fs, TEST_PATH));
DFSTestUtil.writeFile(fs, TEST_FILE_PATH, TEST_FILE_DATA);
DFSTestUtil.writeFile(dfs, new Path(filePath), "hello world"); try { dfs.mkdir(new Path(filePath + "/mkdir"), FsPermission.getDefault());
DFSTestUtil.writeFile(fs, TEST_FILE_PATH, TEST_FILE_DATA);