static void cleanup(FileSystem fs, Path p) throws IOException { fs.delete(p, true); assertFalse("output not cleaned up", fs.exists(p)); }
public void testPipes() throws IOException { if (System.getProperty("compile.c++") == null) { LOG.info("compile.c++ is not defined, so skipping TestPipes"); return; } MiniDFSCluster dfs = null; MiniMRCluster mr = null; Path inputPath = new Path("testing/in"); Path outputPath = new Path("testing/out"); try { final int numSlaves = 2; Configuration conf = new Configuration(); dfs = new MiniDFSCluster(conf, numSlaves, true, null); mr = new MiniMRCluster(numSlaves, dfs.getFileSystem().getUri().toString(), 1); writeInputFile(dfs.getFileSystem(), inputPath); runProgram(mr, dfs, wordCountSimple, inputPath, outputPath, 3, 2, twoSplitOutput, null); cleanup(dfs.getFileSystem(), outputPath); runProgram(mr, dfs, wordCountSimple, inputPath, outputPath, 3, 0, noSortOutput, null); cleanup(dfs.getFileSystem(), outputPath); runProgram(mr, dfs, wordCountPart, inputPath, outputPath, 3, 2, fixedPartitionOutput, null); runNonPipedProgram(mr, dfs, wordCountNoPipes, null); mr.waitUntilIdle(); } finally { mr.shutdown(); dfs.shutdown(); } }
dfs.getFileSystem().makeQualified(wordExec).toString(), "-reduces", "2"}); assertEquals(0, ret); } catch (Exception e) { assertTrue("got exception: " + StringUtils.stringifyException(e), false);
rJob = Submitter.runJob(job); assertTrue("pipes job failed", rJob.isSuccessful()); ++numCounters; assertTrue("No counters found!", (numCounters > 0)); results.add(MapReduceTestUtil.readOutput(p, job)); assertEquals("number of reduces is wrong", expectedResults.length, results.size()); for(int i=0; i < results.size(); i++) { assertEquals("pipes program " + program + " output " + i + " wrong", expectedResults[i], results.get(i));
public void testPipes() throws IOException { if (System.getProperty("compile.c++") == null) { LOG.info("compile.c++ is not defined, so skipping TestPipes"); return; } MiniDFSCluster dfs = null; MiniMRCluster mr = null; Path inputPath = new Path("testing/in"); Path outputPath = new Path("testing/out"); try { final int numSlaves = 2; Configuration conf = new Configuration(); dfs = new MiniDFSCluster.Builder(conf).numDataNodes(numSlaves).build(); mr = new MiniMRCluster(numSlaves, dfs.getFileSystem().getUri().toString(), 1); writeInputFile(dfs.getFileSystem(), inputPath); runProgram(mr, dfs, wordCountSimple, inputPath, outputPath, 3, 2, twoSplitOutput, null); cleanup(dfs.getFileSystem(), outputPath); runProgram(mr, dfs, wordCountSimple, inputPath, outputPath, 3, 0, noSortOutput, null); cleanup(dfs.getFileSystem(), outputPath); runProgram(mr, dfs, wordCountPart, inputPath, outputPath, 3, 2, fixedPartitionOutput, null); runNonPipedProgram(mr, dfs, wordCountNoPipes, null); mr.waitUntilIdle(); } finally { mr.shutdown(); dfs.shutdown(); } }
dfs.getFileSystem().makeQualified(wordExec).toString(), "-reduces", "2"}); assertEquals(0, ret); } catch (Exception e) { assertTrue("got exception: " + StringUtils.stringifyException(e), false);
static void cleanup(FileSystem fs, Path p) throws IOException { fs.delete(p, true); assertFalse("output not cleaned up", fs.exists(p)); }
rJob = Submitter.runJob(job); assertTrue("pipes job failed", rJob.isSuccessful()); ++numCounters; assertTrue("No counters found!", (numCounters > 0)); results.add(MapReduceTestUtil.readOutput(p, job)); assertEquals("number of reduces is wrong", expectedResults.length, results.size()); for(int i=0; i < results.size(); i++) { assertEquals("pipes program " + program + " output " + i + " wrong", expectedResults[i], results.get(i));