@Override public JobConf obtainJobConf(MiniMRCluster cluster) { if (cluster == null) return null; try { Object runner = cluster.getJobTrackerRunner(); Method meth = runner.getClass().getDeclaredMethod("getJobTracker", emptyParam); Object tracker = meth.invoke(runner, new Object []{}); Method m = tracker.getClass().getDeclaredMethod("getConf", emptyParam); return (JobConf) m.invoke(tracker, new Object []{}); } catch (NoSuchMethodException nsme) { return null; } catch (InvocationTargetException ite) { return null; } catch (IllegalAccessException iae) { return null; } }
@Override public JobConf obtainJobConf(MiniMRCluster cluster) { if (cluster == null) return null; try { Object runner = cluster.getJobTrackerRunner(); Method meth = runner.getClass().getDeclaredMethod("getJobTracker", emptyParam); Object tracker = meth.invoke(runner, new Object []{}); Method m = tracker.getClass().getDeclaredMethod("getConf", emptyParam); return (JobConf) m.invoke(tracker, new Object []{}); } catch (NoSuchMethodException nsme) { return null; } catch (InvocationTargetException ite) { return null; } catch (IllegalAccessException iae) { return null; } }
/** * Read the jobtracker.jspx status page and validate that the XML is well formed. */ public void testXmlWellFormed() throws IOException, ParserConfigurationException, SAXException { MiniMRCluster cluster = getMRCluster(); int infoPort = cluster.getJobTrackerRunner().getJobTrackerInfoPort(); String xmlJspUrl = "http://localhost:" + infoPort + "/jobtracker.jspx"; LOG.info("Retrieving XML from URL: " + xmlJspUrl); DocumentBuilder parser = DocumentBuilderFactory.newInstance().newDocumentBuilder(); Document doc = parser.parse(xmlJspUrl); // If we get here, then the document was successfully parsed by SAX and is well-formed. LOG.info("Document received and parsed."); // Make sure it has a <cluster> element as top-level. NodeList clusterNodes = doc.getElementsByTagName("cluster"); assertEquals("There should be exactly 1 <cluster> element", 1, clusterNodes.getLength()); } }
private void cleanupLocalFiles(MiniMRCluster mr) throws IOException { Configuration conf = mr.createJobConf(); JobTracker jt = mr.getJobTrackerRunner().getJobTracker(); Path sysDir = new Path(jt.getSystemDir()); FileSystem fs = sysDir.getFileSystem(conf); fs.delete(sysDir, true); Path jobHistoryDir = mr.getJobTrackerRunner().getJobTracker().getJobHistory(). getJobHistoryLocation(); fs = jobHistoryDir.getFileSystem(conf); fs.delete(jobHistoryDir, true); }
private void startCluster() throws Exception { super.setUp(); Configuration conf = new Configuration(); dfsCluster = new MiniDFSCluster(conf, numSlaves, true, null); JobConf jConf = new JobConf(conf); jConf.setLong("mapred.job.submission.expiry.interval", 6 * 1000); mrCluster = new MiniMRCluster(0, 0, numSlaves, dfsCluster.getFileSystem().getUri().toString(), 1, null, null, null, jConf); jt = mrCluster.getJobTrackerRunner().getJobTracker(); fs = FileSystem.get(mrCluster.createJobConf()); }
@Override public JobConf obtainJobConf(MiniMRCluster cluster) { if (cluster == null) return null; try { Object runner = cluster.getJobTrackerRunner(); Method meth = runner.getClass().getDeclaredMethod("getJobTracker", emptyParam); Object tracker = meth.invoke(runner, new Object []{}); Method m = tracker.getClass().getDeclaredMethod("getConf", emptyParam); return (JobConf) m.invoke(tracker, new Object []{}); } catch (NoSuchMethodException nsme) { return null; } catch (InvocationTargetException ite) { return null; } catch (IllegalAccessException iae) { return null; } }
protected void setUp() throws Exception { Configuration conf = new Configuration(); conf.setClass(JTConfig.JT_TASK_SCHEDULER, FakeTaskScheduler.class, TaskScheduler.class); mr = new MiniMRCluster(0, "file:///", 1, null, null, new JobConf(conf)); jobTracker = mr.getJobTrackerRunner().getJobTracker(); for (String tracker : trackers) { FakeObjectUtilities.establishFirstContact(jobTracker, tracker); } cluster = new Cluster(mr.createJobConf()); }
@Override public JobConf obtainJobConf(MiniMRCluster cluster) { if (cluster == null) return null; try { Object runner = cluster.getJobTrackerRunner(); Method meth = runner.getClass().getDeclaredMethod("getJobTracker", emptyParam); Object tracker = meth.invoke(runner, new Object []{}); Method m = tracker.getClass().getDeclaredMethod("getConf", emptyParam); return (JobConf) m.invoke(tracker, new Object []{}); } catch (NoSuchMethodException nsme) { return null; } catch (InvocationTargetException ite) { return null; } catch (IllegalAccessException iae) { return null; } }
public Void run() throws IOException { verifyOutput(outDir.getFileSystem(conf), outDir); //TestJobHistory TestJobHistory.validateJobHistoryFileFormat( mrCluster.getJobTrackerRunner().getJobTracker().getJobHistory(), jobId, conf, "SUCCEEDED", false); TestJobHistory.validateJobHistoryFileContent(mrCluster, job, conf); // Since we keep setKeepTaskFilesPattern, these files should still be // present and will not be cleaned up. for(int i=0; i < numTT; ++i) { Path jobDirPath = new Path(mrCluster.getTaskTrackerLocalDir(i), TaskTracker .getJobCacheSubdir(TEST1_UGI.getUserName())); boolean b = FileSystem.getLocal(conf).delete(jobDirPath, true); assertTrue(b); } return null; } });
@Override protected void setUp() throws Exception { conf = new JobConf(); conf.setClass(JTConfig.JT_TASK_SCHEDULER, MyScheduler.class, TaskScheduler.class); mr = new MiniMRCluster(1, "file:///", 1, null, null, conf); jobtracker = mr.getJobTrackerRunner().getJobTracker(); myScheduler = (MyScheduler)jobtracker.getScheduler(); conf = mr.createJobConf(); }
@Override public void shutdown() throws IOException { MiniMRCluster.JobTrackerRunner runner = mr.getJobTrackerRunner(); JobTracker tracker = runner.getJobTracker(); if (tracker != null) { for (JobInProgress running : tracker.getRunningJobs()) { try { running.kill(); } catch (Exception e) { // ignore } } } runner.shutdown(); }
@Override public void shutdown() throws IOException { MiniMRCluster.JobTrackerRunner runner = mr.getJobTrackerRunner(); JobTracker tracker = runner.getJobTracker(); if (tracker != null) { for (JobInProgress running : tracker.getRunningJobs()) { try { running.kill(); } catch (Exception e) { // ignore } } } runner.shutdown(); }
public void testJobKillFailAndSucceed() throws IOException { if (Shell.WINDOWS) { System.out.println( "setsid doesn't work on WINDOWS as expected. Not testing"); return; } JobConf conf=null; try { mr = new MiniMRCluster(1, "file:///", 1); // run the TCs conf = mr.createJobConf(); JobTracker jt = mr.getJobTrackerRunner().getJobTracker(); runTests(conf, jt); } finally { if (mr != null) { mr.shutdown(); } } }
public void testJobRetire() throws Exception { MiniMRCluster mr = null; try { JobConf conf = new JobConf(); mr = startCluster(conf, 1); JobConf jobConf = mr.createJobConf(); JobTracker jobtracker = mr.getJobTrackerRunner().getJobTracker(); Path inDir = new Path(testDir, "input1"); Path outDir = new Path(testDir, "output1"); JobID id1 = validateJobRetire(jobConf, inDir, outDir, jobtracker); outDir = new Path(testDir, "output2"); JobID id2 = validateJobRetire(jobConf, inDir, outDir, jobtracker); assertNull("Job not removed from cache", jobtracker.getJobStatus(id1)); assertEquals("Total job in cache not correct", 1, jobtracker.getAllJobs().length); } finally { if (mr != null) { mr.shutdown();} FileUtil.fullyDelete(new File(testDir.toString())); } }
private RunningJob launchJobWithWaitingSetupAndCleanup(MiniMRCluster mr) throws IOException { // launch job with waiting setup/cleanup JobConf jobConf = mr.createJobConf(); jobConf.setOutputCommitter(CommitterWithLongSetupAndCommit.class); RunningJob job = UtilsForTests.runJob(jobConf, inDir, outDir); JobTracker jt = mr.getJobTrackerRunner().getJobTracker(); JobInProgress jip = jt.getJob(job.getID()); while (!jip.inited()) { try { Thread.sleep(10); } catch (InterruptedException ie) {} } return job; }
private void killTaskWithLostTracker(MiniMRCluster mr, TaskAttemptID taskid) { JobTracker jt = mr.getJobTrackerRunner().getJobTracker(); String trackerName = jt.getTaskStatus(taskid).getTaskTracker(); int trackerID = mr.getTaskTrackerID(trackerName); assertTrue(trackerID != -1); mr.stopTaskTracker(trackerID); }
private void validateCommonServlets(MiniMRCluster cluster) throws IOException { int infoPort = cluster.getJobTrackerRunner().getJobTrackerInfoPort(); String jtURL = "http://localhost:" + infoPort; for (String servlet : new String[] { "logs", "stacks", "logLevel" }) { String url = jtURL + "/" + servlet; checkAccessToCommonServlet(url); } // validate access to common servlets for TaskTracker. String ttURL = "http://localhost:" + cluster.getTaskTrackerRunner(0).getTaskTracker().getHttpPort(); for (String servlet : new String[] { "logs", "stacks", "logLevel" }) { String url = ttURL + "/" + servlet; checkAccessToCommonServlet(url); } }
static void runTest(int maxTasks, int numMaps, int numReds, boolean shouldFail) throws Exception { JobConf conf = new JobConf(); conf.setInt(JTConfig.JT_TASKS_PER_JOB, maxTasks); conf.set(JTConfig.JT_IPC_HANDLER_COUNT, "1"); MiniMRCluster mr = new MiniMRCluster(0, "file:///", 1, null, null, conf); JobTracker jt = mr.getJobTrackerRunner().getJobTracker(); JobConf jc = mr.createJobConf(); jc.setNumMapTasks(numMaps); jc.setNumReduceTasks(numReds); JobInProgress jip = new JobInProgress(new JobID(), jc, jt); boolean failed = false; try { jip.checkTaskLimits(); } catch (IOException e) { failed = true; } assertEquals(shouldFail, failed); mr.shutdown(); }
private void verifyTestResults(boolean excludeDynamic) { FakeTaskScheduler scheduler = (FakeTaskScheduler)miniMRCluster.getJobTrackerRunner(). getJobTracker().getTaskScheduler(); assertTrue(scheduler.getFailureMessage(), scheduler.hasTestPassed()); if (!excludeDynamic) { assertTrue(scheduler.getFailureMessage(), scheduler.hasDynamicTestPassed()); } }
private void validateJob(RunningJob job, MiniMRCluster mr) throws IOException { assertEquals(JobStatus.SUCCEEDED, job.getJobState()); long uses = job.getCounters().findCounter("jvm", "use").getValue(); assertTrue("maps = " + numMappers + ", jvms = " + uses, numMappers < uses); JobID jobId = job.getID(); for (int i = 0; i < numMappers; i++) { TaskAttemptID attemptId = new TaskAttemptID(new TaskID(jobId, TaskType.MAP, i), 0); TaskInProgress tip = mr.getJobTrackerRunner().getJobTracker().getTip( attemptId.getTaskID()); TaskStatus ts = mr.getJobTrackerRunner().getJobTracker().getTaskStatus( attemptId); validateAttempt(tip, attemptId, ts, i == taskWithCleanup); if (i == taskWithCleanup) { // validate second attempt of the task attemptId = new TaskAttemptID(new TaskID(jobId, TaskType.MAP, i), 1); ts = mr.getJobTrackerRunner().getJobTracker().getTaskStatus(attemptId); validateAttempt(tip, attemptId, ts, false); } } }