@Test public void testZeroShuffleMemoryLimitPercent() throws Exception { final JobConf jobConf = new JobConf(); jobConf.setFloat(MRJobConfig.SHUFFLE_MEMORY_LIMIT_PERCENT, 0.0f); final MergeManagerImpl<Text, Text> mgr = new MergeManagerImpl<>(null, jobConf, mock(LocalFileSystem.class), null, null, null, null, null, null, null, null, null, null, new MROutputFiles()); verifyReservedMapOutputType(mgr, 10L, "DISK"); } }
@Test public void testLargeMemoryLimits() throws Exception { final JobConf conf = new JobConf(); // Xmx in production conf.setLong(MRJobConfig.REDUCE_MEMORY_TOTAL_BYTES, 8L * 1024 * 1024 * 1024); // M1 = Xmx fraction for map outputs conf.setFloat(MRJobConfig.SHUFFLE_INPUT_BUFFER_PERCENT, 1.0f); // M2 = max M1 fraction for a single maple output conf.setFloat(MRJobConfig.SHUFFLE_MEMORY_LIMIT_PERCENT, 0.95f); // M3 = M1 fraction at which in memory merge is triggered conf.setFloat(MRJobConfig.SHUFFLE_MERGE_PERCENT, 1.0f); // M4 = M1 fraction of map outputs remaining in memory for a reduce conf.setFloat(MRJobConfig.REDUCE_INPUT_BUFFER_PERCENT, 1.0f); final MergeManagerImpl<Text, Text> mgr = new MergeManagerImpl<Text, Text>( null, conf, mock(LocalFileSystem.class), null, null, null, null, null, null, null, null, null, null, new MROutputFiles()); assertTrue("Large shuffle area unusable: " + mgr.memoryLimit, mgr.memoryLimit > Integer.MAX_VALUE); final long maxInMemReduce = mgr.getMaxInMemReduceLimit(); assertTrue("Large in-memory reduce area unusable: " + maxInMemReduce, maxInMemReduce > Integer.MAX_VALUE); assertEquals("maxSingleShuffleLimit to be capped at Integer.MAX_VALUE", Integer.MAX_VALUE, mgr.maxSingleShuffleLimit); verifyReservedMapOutputType(mgr, 10L, "MEMORY"); verifyReservedMapOutputType(mgr, 1L + Integer.MAX_VALUE, "DISK"); }
final int OUTPUT_SIZE = 7950; JobConf conf = new JobConf(); conf.setFloat(MRJobConfig.SHUFFLE_INPUT_BUFFER_PERCENT, 1.0f); conf.setLong(MRJobConfig.REDUCE_MEMORY_TOTAL_BYTES, TOTAL_MEM_BYTES); conf.setFloat(MRJobConfig.SHUFFLE_MEMORY_LIMIT_PERCENT, 0.8f); conf.setFloat(MRJobConfig.SHUFFLE_MERGE_PERCENT, 0.9f); TestExceptionReporter reporter = new TestExceptionReporter(); CyclicBarrier mergeStart = new CyclicBarrier(2);
public void testOutOfBandHeartbeats() throws Exception { MiniDFSCluster dfs = null; MiniMRCluster mr = null; try { Configuration conf = new Configuration(); dfs = new MiniDFSCluster(conf, 4, true, null); int taskTrackers = 1; JobConf jobConf = new JobConf(); jobConf.setFloat(JTConfig.JT_HEARTBEATS_SCALING_FACTOR, 30.0f); jobConf.setBoolean(TTConfig.TT_OUTOFBAND_HEARBEAT, true); mr = new MiniMRCluster(taskTrackers, dfs.getFileSystem().getUri().toString(), 3, null, null, jobConf); long start = System.currentTimeMillis(); TestMiniMRDFSSort.runRandomWriter(mr.createJobConf(), new Path("rw")); long end = System.currentTimeMillis(); final int expectedRuntimeSecs = 120; final int runTimeSecs = (int)((end-start) / 1000); System.err.println("Runtime is " + runTimeSecs); assertEquals("Actual runtime " + runTimeSecs + "s not less than expected " + "runtime of " + expectedRuntimeSecs + "s!", true, (runTimeSecs <= 120)); } finally { if (mr != null) { mr.shutdown(); } if (dfs != null) { dfs.shutdown(); } } }
/** * Regression test for MAPREDUCE-2327. Verifies that, even if a map * task makes lots of spills (more than fit in the spill index cache) * that it will succeed. */ public void testMultipleSpills() throws Exception { JobConf job1 = mr.createJobConf(); // Make sure it spills twice job1.setFloat(MRJobConfig.MAP_SORT_SPILL_PERCENT, 0.0001f); job1.setInt(MRJobConfig.IO_SORT_MB, 1); // Make sure the spill records don't fit in index cache job1.setInt(MRJobConfig.INDEX_CACHE_MEMORY_LIMIT, 0); String input = "The quick brown fox\nhas many silly\n" + "red fox sox\n"; Path inDir = new Path("/testing/distinct/input"); Path outDir = new Path("/user/alice/output"); TestMiniMRClasspath.configureWordCount(fs, jobTrackerName, job1, input, 2, 1, inDir, outDir); runJobAsUser(job1, ALICE_UGI); } }
conf.setFloat(MRJobConfig.COMPLETED_MAPS_FOR_REDUCE_SLOWSTART, 1.0f);
conf.setFloat(MRJobConfig.COMPLETED_MAPS_FOR_REDUCE_SLOWSTART, 1.0f);
AvroUtils.addAvroCacheFiles(conf, new Path(modelPath)); conf.set(MODEL_PATH, modelPath); conf.setFloat(LAMBDA, Float.parseFloat(lambda)); conf.setBoolean(BINARY_FEATURE, props.getBoolean(BINARY_FEATURE, false)); AvroJob.setInputSchema(conf, AvroUtils.getAvroInputSchema(conf)); AvroUtils.addAvroCacheFiles(conf, new Path(modelPath)); conf.set(MODEL_PATH, modelPath); conf.setFloat(LAMBDA, -1); conf.setBoolean(BINARY_FEATURE, props.getBoolean(BINARY_FEATURE, false)); AvroJob.setInputSchema(conf, AvroUtils.getAvroInputSchema(conf));
conf.setFloat(INTERCEPT_DEFAULT_PRIOR_MEAN, (float)props.getDouble(INTERCEPT_DEFAULT_PRIOR_MEAN,0)); conf.set(INTERCEPT_LAMBDAS,props.get(INTERCEPT_LAMBDAS)); conf.set(DEFAULT_LAMBDAS,props.get(DEFAULT_LAMBDAS)); conf.setLong(REPORT_FREQUENCY, props.getLong(REPORT_FREQUENCY, 1000000)); conf.setFloat(LIBLINEAR_EPSILON, (float) props.getDouble(LIBLINEAR_EPSILON, 0.001f)); conf.setBoolean(COMPUTE_VAR, props.getBoolean(COMPUTE_VAR,false)); conf.setBoolean(BINARY_FEATURE, props.getBoolean(BINARY_FEATURE, false));
@Override public void run() throws Exception { JobConfig props = super.getJobConfig(); List<String> lambdastr = props.getStringList(LAMBDA, ","); String outBasePath = props.getString(OUTPUT_BASE_PATH); for (String lambda : lambdastr) { String outPath = outBasePath + "/lambda-" + lambda; props.put("output.path", outPath); JobConf conf = createJobConf(PerItemTestMapper.class, PerItemTestReducer.class); AvroUtils.addAvroCacheFilesAndSetTheProperty(conf, new Path(props.get(MODEL_PATH)), MODEL_PATH); conf.set(ITEM_KEY, props.getString(ITEM_KEY)); conf.setFloat(LAMBDA, Float.parseFloat(lambda)); conf.setBoolean(BINARY_FEATURE, props.getBoolean(BINARY_FEATURE, false)); conf.setPartitionerClass(PerItemTestPartitioner.class); conf.setInt(NUM_REDUCERS, conf.getNumReduceTasks()); AvroUtils.runAvroJob(conf); } }
/** * Regression test for MAPREDUCE-2327. Verifies that, even if a map * task makes lots of spills (more than fit in the spill index cache) * that it will succeed. */ @Test public void testMultipleSpills() throws Exception { JobConf job1 = mr.createJobConf(); // Make sure it spills twice job1.setFloat(MRJobConfig.MAP_SORT_SPILL_PERCENT, 0.0001f); job1.setInt(MRJobConfig.IO_SORT_MB, 1); // Make sure the spill records don't fit in index cache job1.setInt(MRJobConfig.INDEX_CACHE_MEMORY_LIMIT, 0); String input = "The quick brown fox\nhas many silly\n" + "red fox sox\n"; Path inDir = new Path("/testing/distinct/input"); Path outDir = new Path("/user/alice/output"); TestMiniMRClasspath .configureWordCount(fs, job1, input, 2, 1, inDir, outDir); runJobAsUser(job1, ALICE_UGI); } }
conf.setFloat(PRIOR_MEAN, props.getFloat(PRIOR_MEAN,0.0)); conf.setBoolean(PENALIZE_INTERCEPT, props.getBoolean(PENALIZE_INTERCEPT, false)); conf.setBoolean(HAS_INTERCEPT, props.getBoolean(HAS_INTERCEPT, true)); conf.setLong(REPORT_FREQUENCY, props.getLong(REPORT_FREQUENCY, 1000000)); boolean removeTmpDir = props.getBoolean(REMOVE_TMP_DIR, true); conf.setFloat(LIBLINEAR_EPSILON, props.getFloat(LIBLINEAR_EPSILON, 0.001f)); String lambdaMap = props.getString(LAMBDA_MAP, ""); conf.set(LAMBDA_MAP, lambdaMap);
conf.setNumMapTasks(5); conf.setNumReduceTasks(0); conf.setFloat(JobContext.SPECULATIVE_SLOWTASK_THRESHOLD, 0.5f); FakeJobInProgress job = new FakeJobInProgress(conf, jobTracker); job.initTasks();
conf.setNumMapTasks(3); conf.setNumReduceTasks(3); conf.setFloat(JobContext.SPECULATIVE_SLOWTASK_THRESHOLD, 0.5f); FakeJobInProgress job = new FakeJobInProgress(conf, jobTracker); job.initTasks();
conf.setNumMapTasks(5); conf.setNumReduceTasks(5); conf.setFloat(JobContext.SPECULATIVE_SLOWTASK_THRESHOLD, 0.5f); FakeJobInProgress job = new FakeJobInProgress(conf, jobTracker); job.initTasks();
conf.setLong("cpuFrequency", 2000000L); conf.setInt("numProcessors", 8); conf.setFloat("cpuUsage", 15.5F); conf.setLong("procCumulativeCpuTime", 1000L); conf.setLong("procVirtualMemorySize", 2 * 1024 * 1024 * 1024L); conf.setLong(DummyResourceCalculatorPlugin.CPU_FREQUENCY, 2000000L); conf.setInt(DummyResourceCalculatorPlugin.NUM_PROCESSORS, 8); conf.setFloat(DummyResourceCalculatorPlugin.CPU_USAGE, 15.5F); try { setUpCluster(conf);