private void _testInputFiles(boolean withFilter, boolean withGlob) throws Exception { Set<Path> createdFiles = createFiles(); JobConf conf = new JobConf(); Path inputDir = (withGlob) ? new Path(workDir, "a*") : workDir; FileInputFormat.setInputPaths(conf, inputDir); conf.setInputFormat(DummyFileInputFormat.class); if (withFilter) { FileInputFormat.setInputPathFilter(conf, TestPathFilter.class); } DummyFileInputFormat inputFormat = (DummyFileInputFormat) conf.getInputFormat(); Set<Path> computedFiles = new HashSet<Path>(); for (FileStatus file : inputFormat.listStatus(conf)) { computedFiles.add(file.getPath()); } createdFiles.remove(localFs.makeQualified(new Path(workDir, "_hello"))); createdFiles.remove(localFs.makeQualified(new Path(workDir, ".hello"))); if (withFilter) { createdFiles.remove(localFs.makeQualified(new Path(workDir, "aa"))); createdFiles.remove(localFs.makeQualified(new Path(workDir, "bb"))); } if (withGlob) { createdFiles.remove(localFs.makeQualified(new Path(workDir, "b"))); createdFiles.remove(localFs.makeQualified(new Path(workDir, "bb"))); } assertEquals(createdFiles, computedFiles); }
private void _testInputFiles(boolean withFilter, boolean withGlob) throws Exception { Set<Path> createdFiles = createFiles(); JobConf conf = new JobConf(); Path inputDir = (withGlob) ? new Path(workDir, "a*") : workDir; FileInputFormat.setInputPaths(conf, inputDir); conf.setInputFormat(DummyFileInputFormat.class); if (withFilter) { FileInputFormat.setInputPathFilter(conf, TestPathFilter.class); } DummyFileInputFormat inputFormat = (DummyFileInputFormat) conf.getInputFormat(); Set<Path> computedFiles = new HashSet<Path>(); for (FileStatus file : inputFormat.listStatus(conf)) { computedFiles.add(file.getPath()); } createdFiles.remove(localFs.makeQualified(new Path(workDir, "_hello"))); createdFiles.remove(localFs.makeQualified(new Path(workDir, ".hello"))); if (withFilter) { createdFiles.remove(localFs.makeQualified(new Path(workDir, "aa"))); createdFiles.remove(localFs.makeQualified(new Path(workDir, "bb"))); } if (withGlob) { createdFiles.remove(localFs.makeQualified(new Path(workDir, "b"))); createdFiles.remove(localFs.makeQualified(new Path(workDir, "bb"))); } assertEquals(createdFiles, computedFiles); }
LOG.info("Setting input path to " + inputPath); FileInputFormat.addInputPath(job, new Path(inputPath)); FileInputFormat.setInputPathFilter(job, FileCountFilter.class); } else { System.err.println("No input path found.");