protected void setUp() throws Exception { Configuration conf = new Configuration(); conf.setInt("dfs.replication", 1); dfs = new MiniDFSCluster(conf, numTT, true, null); fs = DFS_UGI.doAs(new PrivilegedExceptionAction<FileSystem>() { public FileSystem run() throws IOException { return dfs.getFileSystem(); } }); TestMiniMRWithDFSWithDistinctUsers.mkdir(fs, "/user", "mapred", "mapred", (short)01777); TestMiniMRWithDFSWithDistinctUsers.mkdir(fs, "/mapred", "mapred", "mapred", (short)01777); TestMiniMRWithDFSWithDistinctUsers.mkdir(fs, conf.get(JTConfig.JT_STAGING_AREA_ROOT), "mapred", "mapred", (short)01777); UserGroupInformation MR_UGI = UserGroupInformation.getLoginUser(); // Create a TestJobInProgressListener.MyListener and associate // it with the MiniMRCluster myListener = new MyListener(); conf.set(JTConfig.JT_IPC_HANDLER_COUNT, "1"); mrCluster = new MiniMRCluster(0, 0, numTT, fs.getUri().toString(), 1, null, null, MR_UGI, new JobConf()); // make cleanup inline sothat validation of existence of these directories // can be done mrCluster.setInlineCleanupThreads(); mrCluster.getJobTrackerRunner().getJobTracker() .addJobInProgressListener(myListener); }
public void testJobDirCleanup() throws Exception { String namenode = null; MiniDFSCluster dfs = null; MiniMRCluster mr = null; FileSystem fileSys = null; try { final int taskTrackers = 10; Configuration conf = new Configuration(); JobConf mrConf = new JobConf(); mrConf.set(TTConfig.TT_REDUCE_SLOTS, "1"); dfs = new MiniDFSCluster(conf, 1, true, null); fileSys = dfs.getFileSystem(); namenode = fileSys.getUri().toString(); mr = new MiniMRCluster(10, namenode, 3, null, null, mrConf); // make cleanup inline sothat validation of existence of these directories // can be done mr.setInlineCleanupThreads(); // run the sleep job JobConf jobConf = mr.createJobConf(); JobID jobid = runSleepJob(jobConf); // verify the job directories are cleaned up. verifyJobDirCleanup(mr, taskTrackers, jobid); } finally { if (fileSys != null) { fileSys.close(); } if (dfs != null) { dfs.shutdown(); } if (mr != null) { mr.shutdown(); } } }
public void testWithDFS() throws IOException, InterruptedException, ClassNotFoundException { MiniDFSCluster dfs = null; MiniMRCluster mr = null; FileSystem fileSys = null; try { final int taskTrackers = 4; Configuration conf = new Configuration(); dfs = new MiniDFSCluster(conf, 4, true, null); fileSys = dfs.getFileSystem(); mr = new MiniMRCluster(taskTrackers, fileSys.getUri().toString(), 1); // make cleanup inline sothat validation of existence of these directories // can be done mr.setInlineCleanupThreads(); runPI(mr, mr.createJobConf()); runWordCount(mr, mr.createJobConf()); } finally { if (dfs != null) { dfs.shutdown(); } if (mr != null) { mr.shutdown(); } } }