public MiniMrShim(Configuration conf, int numberOfTaskTrackers, String nameNode, int numDir) throws IOException { this.conf = conf; JobConf jConf = new JobConf(conf); jConf.set("yarn.scheduler.capacity.root.queues", "default"); jConf.set("yarn.scheduler.capacity.root.default.capacity", "100"); jConf.setInt(MRJobConfig.MAP_MEMORY_MB, 512); jConf.setInt(MRJobConfig.REDUCE_MEMORY_MB, 512); jConf.setInt(MRJobConfig.MR_AM_VMEM_MB, 128); jConf.setInt(YarnConfiguration.YARN_MINICLUSTER_NM_PMEM_MB, 512); jConf.setInt(YarnConfiguration.RM_SCHEDULER_MINIMUM_ALLOCATION_MB, 128); jConf.setInt(YarnConfiguration.RM_SCHEDULER_MAXIMUM_ALLOCATION_MB, 512); mr = new MiniMRCluster(numberOfTaskTrackers, nameNode, numDir, null, null, jConf); }
private void setupMiniDfsAndMrClusters() { try { final int dataNodes = 1; // There will be 4 data nodes final int taskTrackers = 1; // There will be 4 task tracker nodes Configuration config = new Configuration(); // Builds and starts the mini dfs and mapreduce clusters if(System.getProperty("hadoop.log.dir") == null) { System.setProperty("hadoop.log.dir", "target/tmp/logs/"); } m_dfs = new MiniDFSCluster(config, dataNodes, true, null); m_fileSys = m_dfs.getFileSystem(); m_mr = new MiniMRCluster(taskTrackers, m_fileSys.getUri().toString(), 1); // Create the configuration hadoop-site.xml file File conf_dir = new File(System.getProperty("user.home"), "pigtest/conf/"); conf_dir.mkdirs(); File conf_file = new File(conf_dir, "hadoop-site.xml"); // Write the necessary config info to hadoop-site.xml m_conf = m_mr.createJobConf(); m_conf.setInt("mapred.submit.replication", 1); m_conf.set("dfs.datanode.address", "0.0.0.0:0"); m_conf.set("dfs.datanode.http.address", "0.0.0.0:0"); m_conf.writeXml(new FileOutputStream(conf_file)); // Set the system properties needed by Pig System.setProperty("cluster", m_conf.get("mapred.job.tracker")); System.setProperty("namenode", m_conf.get("fs.default.name")); System.setProperty("junit.hadoop.conf", conf_dir.getPath()); } catch (IOException e) { throw new RuntimeException(e); } }
@BeforeClass public static void setup() throws Exception { System.clearProperty("mapred.job.tracker"); String testDir = System.getProperty("test.tmp.dir", "./"); testDir = testDir + "/test_multitable_" + Math.abs(new Random().nextLong()) + "/"; workDir = new File(new File(testDir).getCanonicalPath()); FileUtil.fullyDelete(workDir); workDir.mkdirs(); warehousedir = new Path(System.getProperty("test.warehouse.dir")); HiveConf metastoreConf = new HiveConf(); metastoreConf.setVar(HiveConf.ConfVars.METASTOREWAREHOUSE, warehousedir.toString()); // Run hive metastore server MetaStoreTestUtils.startMetaStoreWithRetry(metastoreConf); // Read the warehouse dir, which can be changed so multiple MetaStore tests could be run on // the same server warehousedir = new Path(MetastoreConf.getVar(metastoreConf, MetastoreConf.ConfVars.WAREHOUSE)); // LocalJobRunner does not work with mapreduce OutputCommitter. So need // to use MiniMRCluster. MAPREDUCE-2350 Configuration conf = new Configuration(true); conf.set("yarn.scheduler.capacity.root.queues", "default"); conf.set("yarn.scheduler.capacity.root.default.capacity", "100"); FileSystem fs = FileSystem.get(conf); System.setProperty("hadoop.log.dir", new File(workDir, "/logs").getAbsolutePath()); mrCluster = new MiniMRCluster(1, fs.getUri().toString(), 1, null, null, new JobConf(conf)); mrConf = mrCluster.createJobConf(); initializeSetup(metastoreConf); warehousedir.getFileSystem(conf).mkdirs(warehousedir); }
public void testWithDFS() throws IOException { MiniDFSCluster dfs = null; MiniMRCluster mr = null; FileSystem fileSys = null; try { final int taskTrackers = 4; JobConf conf = new JobConf(); conf.set(JTConfig.JT_SYSTEM_DIR, "/tmp/custom/mapred/system"); dfs = new MiniDFSCluster(conf, 4, true, null); fileSys = dfs.getFileSystem(); mr = new MiniMRCluster(taskTrackers, fileSys.getUri().toString(), 1, null, null, conf); runWordCount(mr, mr.createJobConf(), conf.get("mapred.system.dir")); } finally { if (dfs != null) { dfs.shutdown(); } if (mr != null) { mr.shutdown(); } } }
try { final int taskTrackers = 2; JobConf conf = new JobConf(); fileSys = FileSystem.get(conf); new MiniMRCluster(taskTrackers, fileSys.getUri().toString(), 1, null, null, conf); mr.shutdown();
/** Tests using a full MiniMRCluster. */ public void testMiniMRJobRunner() throws Exception { MiniMRCluster m = new MiniMRCluster(1, "file:///", 1); try { testWithConf(m.createJobConf()); } finally { m.shutdown(); } }
public void testBringUp() throws IOException { MiniMRCluster mr = null; try { mr = new MiniMRCluster(1, "local", 1); } finally { if (mr != null) { mr.shutdown(); } } }
public void testOutOfBandHeartbeats() throws Exception { MiniDFSCluster dfs = null; MiniMRCluster mr = null; try { Configuration conf = new Configuration(); dfs = new MiniDFSCluster(conf, 4, true, null); int taskTrackers = 1; JobConf jobConf = new JobConf(); jobConf.setFloat(JTConfig.JT_HEARTBEATS_SCALING_FACTOR, 30.0f); jobConf.setBoolean(TTConfig.TT_OUTOFBAND_HEARBEAT, true); mr = new MiniMRCluster(taskTrackers, dfs.getFileSystem().getUri().toString(), 3, null, null, jobConf); long start = System.currentTimeMillis(); TestMiniMRDFSSort.runRandomWriter(mr.createJobConf(), new Path("rw")); long end = System.currentTimeMillis(); final int expectedRuntimeSecs = 120; final int runTimeSecs = (int)((end-start) / 1000); System.err.println("Runtime is " + runTimeSecs); assertEquals("Actual runtime " + runTimeSecs + "s not less than expected " + "runtime of " + expectedRuntimeSecs + "s!", true, (runTimeSecs <= 120)); } finally { if (mr != null) { mr.shutdown(); } if (dfs != null) { dfs.shutdown(); } } }
@BeforeClass public static void setup() throws IOException { createWorkDir(); Configuration conf = new Configuration(true); conf.set("yarn.scheduler.capacity.root.queues", "default"); conf.set("yarn.scheduler.capacity.root.default.capacity", "100"); fs = FileSystem.get(conf); System.setProperty("hadoop.log.dir", new File(workDir, "/logs").getAbsolutePath()); // LocalJobRunner does not work with mapreduce OutputCommitter. So need // to use MiniMRCluster. MAPREDUCE-2350 mrConf = new JobConf(conf); mrCluster = new MiniMRCluster(1, fs.getUri().toString(), 1, null, null, mrConf); }
mrCluster = new MiniMRCluster(servers, FS_URI != null ? FS_URI : FileSystem.get(conf).getUri().toString(), 1, null, null, new JobConf(this.conf)); JobConf jobConf = MapreduceTestingShim.getJobConf(mrCluster); if (jobConf == null) { jobConf = mrCluster.createJobConf();
public void testJobWithDFS() throws IOException { String namenode = null; MiniDFSCluster dfs = null; MiniMRCluster mr = null; FileSystem fileSys = null; try { final int taskTrackers = 4; final int jobTrackerPort = 60050; Configuration conf = new Configuration(); dfs = new MiniDFSCluster(conf, 1, true, null); fileSys = dfs.getFileSystem(); namenode = fileSys.getUri().toString(); mr = new MiniMRCluster(taskTrackers, namenode, 2); final String jobTrackerName = "localhost:" + mr.getJobTrackerPort(); JobConf jobConf = new JobConf(); boolean result; result = launchJob(fileSys.getUri(), jobTrackerName, jobConf, 3, 1); assertTrue(result); } finally { if (dfs != null) { dfs.shutdown(); } if (mr != null) { mr.shutdown(); } } }
public void testWithLocal() throws Exception { MiniMRCluster mr = null; try { mr = new MiniMRCluster(2, "file:///", 3); Configuration conf = mr.createJobConf(); runWordCount(conf); runMultiFileWordCount(conf); } finally { if (mr != null) { mr.shutdown(); } } }
public static void main(String[] args) throws IOException { LOG.info("Bringing up Jobtracker and tasktrackers."); MiniMRCluster mr = new MiniMRCluster(4, "file:///", 1); LOG.info("JobTracker and TaskTrackers are up."); mr.shutdown(); LOG.info("JobTracker and TaskTrackers brought down."); } }
final MiniDFSCluster cluster = new MiniDFSCluster(conf, 2, true, null); final FileSystem fs = cluster.getFileSystem(); final MiniMRCluster mr = new MiniMRCluster(2, fs.getUri().toString(), 1); final FsShell shell = new FsShell(conf); new DistCh(mr.createJobConf()).run(args); runLsr(shell, tree.root, 0);
public void testJobDirCleanup() throws Exception { String namenode = null; MiniDFSCluster dfs = null; MiniMRCluster mr = null; FileSystem fileSys = null; try { final int taskTrackers = 10; Configuration conf = new Configuration(); JobConf mrConf = new JobConf(); mrConf.set(TTConfig.TT_REDUCE_SLOTS, "1"); dfs = new MiniDFSCluster(conf, 1, true, null); fileSys = dfs.getFileSystem(); namenode = fileSys.getUri().toString(); mr = new MiniMRCluster(10, namenode, 3, null, null, mrConf); // make cleanup inline sothat validation of existence of these directories // can be done mr.setInlineCleanupThreads(); // run the sleep job JobConf jobConf = mr.createJobConf(); JobID jobid = runSleepJob(jobConf); // verify the job directories are cleaned up. verifyJobDirCleanup(mr, taskTrackers, jobid); } finally { if (fileSys != null) { fileSys.close(); } if (dfs != null) { dfs.shutdown(); } if (mr != null) { mr.shutdown(); } } }
public MiniMrShim(Configuration conf, int numberOfTaskTrackers, String nameNode, int numDir) throws IOException { this.conf = conf; JobConf jConf = new JobConf(conf); jConf.set("yarn.scheduler.capacity.root.queues", "default"); jConf.set("yarn.scheduler.capacity.root.default.capacity", "100"); mr = new MiniMRCluster(numberOfTaskTrackers, nameNode, numDir, null, null, jConf); }
mrCluster = new MiniMRCluster(1, fs.getUri().toString(), 1, null, null, new JobConf(conf)); mrConf = mrCluster.createJobConf();
fileSys = dfs.getFileSystem(); namenode = fileSys.getUri().toString(); mr = new MiniMRCluster(taskTrackers, namenode, 3); JobConf jobConf = new JobConf(); String result; final String jobTrackerName = "localhost:" + mr.getJobTrackerPort(); if (mr != null) { mr.shutdown();
public void testJobKillFailAndSucceed() throws IOException { if (Shell.WINDOWS) { System.out.println( "setsid doesn't work on WINDOWS as expected. Not testing"); return; } JobConf conf=null; try { mr = new MiniMRCluster(1, "file:///", 1); // run the TCs conf = mr.createJobConf(); JobTracker jt = mr.getJobTrackerRunner().getJobTracker(); runTests(conf, jt); } finally { if (mr != null) { mr.shutdown(); } } }
public void testBringUp() throws IOException { MiniMRCluster mr = null; try { mr = new MiniMRCluster(1, "local", 1); } finally { if (mr != null) { mr.shutdown(); } } }