NLineInputFormat.setInputPaths(job, inputDir); NLineInputFormat.setNumLinesPerSplit(job, 1);
public void testFormat() throws Exception { Job job = Job.getInstance(conf); Path file = new Path(workDir, "test.txt"); int seed = new Random().nextInt(); Random random = new Random(seed); localFs.delete(workDir, true); FileInputFormat.setInputPaths(job, workDir); int numLinesPerMap = 5; NLineInputFormat.setNumLinesPerSplit(job, numLinesPerMap); // for a variety of lengths for (int length = 0; length < MAX_LENGTH; length += random.nextInt(MAX_LENGTH / 10) + 1) { // create a file with length entries Writer writer = new OutputStreamWriter(localFs.create(file)); try { for (int i = 0; i < length; i++) { writer.write(Integer.toString(i)); writer.write("\n"); } } finally { writer.close(); } checkFormat(job, numLinesPerMap); } }
NLineInputFormat.setNumLinesPerSplit(job, 1);
job2.setInputFormatClass(NLineInputFormat.class); NLineInputFormat.addInputPath(job2, fullInputList); NLineInputFormat.setNumLinesPerSplit(job2, numLinesPerSplit); job2.setMapperClass(LineRandomizerMapper.class); job2.setReducerClass(LineRandomizerReducer.class);
job2.setInputFormatClass(NLineInputFormat.class); NLineInputFormat.addInputPath(job2, fullInputList); NLineInputFormat.setNumLinesPerSplit(job2, numLinesPerSplit); job2.setMapperClass(LineRandomizerMapper.class); job2.setReducerClass(LineRandomizerReducer.class);
NLineInputFormat.setNumLinesPerSplit(job, 10000); // TODO Would be
NLineInputFormat.setNumLinesPerSplit(job, 10000); // TODO Would be
NLineInputFormat.setNumLinesPerSplit(job, 10000); // TODO Would be
NLineInputFormat.setNumLinesPerSplit(job, 1);
NLineInputFormat.setNumLinesPerSplit(job, 1);
public void testFormat() throws Exception { Job job = Job.getInstance(conf); Path file = new Path(workDir, "test.txt"); localFs.delete(workDir, true); FileInputFormat.setInputPaths(job, workDir); int numLinesPerMap = 5; NLineInputFormat.setNumLinesPerSplit(job, numLinesPerMap); for (int length = 0; length < MAX_LENGTH; length += 1) { // create a file with length entries Writer writer = new OutputStreamWriter(localFs.create(file)); try { for (int i = 0; i < length; i++) { writer.write(Integer.toString(i)+" some more text"); writer.write("\n"); } } finally { writer.close(); } int lastN = 0; if (length != 0) { lastN = length % 5; if (lastN == 0) { lastN = 5; } } checkFormat(job, numLinesPerMap, lastN); } }
NLineInputFormat.setNumLinesPerSplit(job, options.fanout); FileOutputFormat.setOutputPath(job, outputTreeMergeStep);
NLineInputFormat.setNumLinesPerSplit(job, numLinesPerSplit); FileOutputFormat.setOutputPath(job, outputReduceDir); NLineInputFormat.setNumLinesPerSplit(job, options.fanout); FileOutputFormat.setOutputPath(job, outputTreeMergeStep);
NLineInputFormat.setNumLinesPerSplit(job, options.fanout); FileOutputFormat.setOutputPath(job, outputTreeMergeStep);
NLineInputFormat.setNumLinesPerSplit(job, numLinesPerSplit); FileOutputFormat.setOutputPath(job, outputReduceDir); NLineInputFormat.setNumLinesPerSplit(job, options.fanout); FileOutputFormat.setOutputPath(job, outputTreeMergeStep);
NLineInputFormat.setInputPaths(job, inputDir); NLineInputFormat.setNumLinesPerSplit(job, 1);
NLineInputFormat.setInputPaths(job, inputDir); NLineInputFormat.setNumLinesPerSplit(job, 1);