private Path[] getMapFiles(FileSystem fs) throws IOException { List<Path> fileList = new ArrayList<Path>(); for(int i = 0; i < numMaps; ++i) { fileList.add(mapOutputFile.getInputFile(i)); } return fileList.toArray(new Path[0]); }
private Path[] getMapFiles(FileSystem fs) throws IOException { List<Path> fileList = new ArrayList<Path>(); for(int i = 0; i < numMaps; ++i) { fileList.add(mapOutputFile.getInputFile(i)); } return fileList.toArray(new Path[0]); }
private Path[] getMapFiles(FileSystem fs) throws IOException { List<Path> fileList = new ArrayList<Path>(); for(int i = 0; i < numMaps; ++i) { fileList.add(mapOutputFile.getInputFile(i)); } return fileList.toArray(new Path[0]); }
private Path[] getMapFiles(FileSystem fs) throws IOException { List<Path> fileList = new ArrayList<Path>(); for(int i = 0; i < numMaps; ++i) { fileList.add(mapOutputFile.getInputFile(i)); } return fileList.toArray(new Path[0]); }
private Path[] getMapFiles(FileSystem fs, boolean isLocal) throws IOException { List<Path> fileList = new ArrayList<Path>(); if (isLocal) { // for local jobs for(int i = 0; i < numMaps; ++i) { fileList.add(mapOutputFile.getInputFile(i)); } } else { // for non local jobs for (FileStatus filestatus : mapOutputFilesOnDisk) { fileList.add(filestatus.getPath()); } } return fileList.toArray(new Path[0]); }
private Path[] getMapFiles(FileSystem fs, boolean isLocal) throws IOException { List<Path> fileList = new ArrayList<Path>(); if (isLocal) { // for local jobs for(int i = 0; i < numMaps; ++i) { fileList.add(mapOutputFile.getInputFile(i, getTaskID())); } } else { // for non local jobs for (FileStatus filestatus : mapOutputFilesOnDisk) { fileList.add(filestatus.getPath()); } } return fileList.toArray(new Path[0]); }
private Path[] getMapFiles(FileSystem fs, boolean isLocal) throws IOException { List<Path> fileList = new ArrayList<Path>(); if (isLocal) { // for local jobs for(int i = 0; i < numMaps; ++i) { fileList.add(mapOutputFile.getInputFile(i, getTaskID())); } } else { // for non local jobs for (FileStatus filestatus : mapOutputFilesOnDisk) { fileList.add(filestatus.getPath()); } } return fileList.toArray(new Path[0]); }
/** * Create empty sequence files for any of the map outputs that we don't have. * @param fs the filesystem to create the files in * @param dir the directory name to create the files in * @param conf the jobconf * @throws IOException if something goes wrong writing */ private static void fillInMissingMapOutputs(FileSystem fs, TaskAttemptID taskId, int numMaps, JobConf conf) throws IOException { Class<? extends WritableComparable> keyClass = conf.getMapOutputKeyClass().asSubclass(WritableComparable.class); Class<? extends Writable> valueClass = conf.getMapOutputValueClass().asSubclass(Writable.class); MapOutputFile namer = new MapOutputFile(taskId.getJobID()); namer.setConf(conf); for(int i=0; i<numMaps; i++) { Path f = namer.getInputFile(i, taskId); if (!fs.exists(f)) { LOG.info("Create missing input: " + f); SequenceFile.Writer out = SequenceFile.createWriter(fs, conf, f, keyClass, valueClass); out.close(); } } }
/** * Create empty sequence files for any of the map outputs that we don't have. * @param fs the filesystem to create the files in * @param dir the directory name to create the files in * @param conf the jobconf * @throws IOException if something goes wrong writing */ private static void fillInMissingMapOutputs(FileSystem fs, TaskAttemptID taskId, int numMaps, JobConf conf) throws IOException { Class<? extends WritableComparable> keyClass = conf.getMapOutputKeyClass().asSubclass(WritableComparable.class); Class<? extends Writable> valueClass = conf.getMapOutputValueClass().asSubclass(Writable.class); MapOutputFile namer = new MapOutputFile(taskId.getJobID()); namer.setConf(conf); for(int i=0; i<numMaps; i++) { Path f = namer.getInputFile(i, taskId); if (!fs.exists(f)) { LOG.info("Create missing input: " + f); SequenceFile.Writer out = SequenceFile.createWriter(fs, conf, f, keyClass, valueClass); out.close(); } } }
public void reduce(WritableComparable key, Iterator values, OutputCollector output, Reporter reporter ) throws IOException { if (first) { first = false; MapOutputFile mapOutputFile = new MapOutputFile(); mapOutputFile.setConf(conf); Path input = mapOutputFile.getInputFile(0); FileSystem fs = FileSystem.get(conf); assertTrue("reduce input exists " + input, fs.exists(input)); SequenceFile.Reader rdr = new SequenceFile.Reader(fs, input, conf); assertEquals("is reduce input compressed " + input, compressInput, rdr.isCompressed()); rdr.close(); } }
public void reduce(WritableComparable key, Iterator values, OutputCollector output, Reporter reporter ) throws IOException { if (first) { first = false; MapOutputFile mapOutputFile = new MROutputFiles(); mapOutputFile.setConf(conf); Path input = mapOutputFile.getInputFile(0); FileSystem fs = FileSystem.get(conf); assertTrue("reduce input exists " + input, fs.exists(input)); SequenceFile.Reader rdr = new SequenceFile.Reader(fs, input, conf); assertEquals("is reduce input compressed " + input, compressInput, rdr.isCompressed()); rdr.close(); } }