@Override public void setConf(Configuration conf) { if (!(conf instanceof JobConf)) { conf = new JobConf(conf); } super.setConf(conf); }
return -1; MapOutputFile mapOutputFile = new MapOutputFile(); mapOutputFile.setJobId(taskId.getJobID()); mapOutputFile.setConf(conf); Path tmp_output = mapOutputFile.getOutputFile(taskId); if(tmp_output == null) return 0;
FileSystem localFs = FileSystem.getLocal(conf); Path mapOut = subMapOutputFile.getOutputFile(); FileStatus mStatus = localFs.getFileStatus(mapOut); Path reduceIn = subMapOutputFile.getInputFileForWrite( TypeConverter.fromYarn(mapId).getTaskID(), mStatus.getLen()); Path mapOutIndex = subMapOutputFile.getOutputIndexFile(); Path reduceInIndex = new Path(reduceIn.toString() + ".index"); if (LOG.isDebugEnabled()) {
conf.set(MRConfig.LOCAL_DIR, TEST_ROOT_DIR.getAbsolutePath()); MapOutputFile mof = new MROutputFiles(); mof.setConf(conf); TaskAttemptID attemptId = new TaskAttemptID("12345", 1, TaskType.MAP, 1, 1); MapTask mockTask = mock(MapTask.class); mob.flush(); mob.close(); Path outputFile = mof.getOutputFile(); FileSystem lfs = FileSystem.getLocal(conf); FsPermission perms = lfs.getFileStatus(outputFile).getPermission(); Assert.assertEquals("Incorrect output file perms", (short)0640, perms.toShort()); Path indexFile = mof.getOutputIndexFile(); perms = lfs.getFileStatus(indexFile).getPermission(); Assert.assertEquals("Incorrect index file perms",
public TaskRunner(TaskTracker.TaskInProgress tip, TaskTracker tracker, JobConf conf) { this.tip = tip; this.t = tip.getTask(); this.tracker = tracker; this.conf = conf; this.mapOutputFile = new MapOutputFile(t.getJobID()); this.mapOutputFile.setConf(conf); this.jvmManager = tracker.getJvmManagerInstance(); }
filename[i] = mapOutputFile.getSpillFile(i); finalOutFileSize += rfs.getFileStatus(filename[i]).getLen(); mapOutputFile.getOutputFileForWriteInVolume(filename[0])); if (indexCacheList.size() == 0) { sameVolRename(mapOutputFile.getSpillIndexFile(0), mapOutputFile.getOutputIndexFileForWriteInVolume(filename[0])); } else { indexCacheList.get(0).writeToFile( mapOutputFile.getOutputIndexFileForWriteInVolume(filename[0]), job); Path indexFileName = mapOutputFile.getSpillIndexFile(i); indexCacheList.add(new SpillRecord(indexFileName, job)); finalIndexFileSize = partitions * MAP_OUTPUT_INDEX_RECORD_LENGTH; Path finalOutputFile = mapOutputFile.getOutputFileForWrite(finalOutFileSize); Path finalIndexFile = mapOutputFile.getOutputIndexFileForWrite(finalIndexFileSize);
filename[i] = task.mapOutputFile.getSpillFile(mapId, i); finalOutFileSize += rfs.getFileStatus(filename[i]).getLen(); rfs.rename(filename[0], outFile); if (indexCacheList.size() == 0) { rfs.rename(task.mapOutputFile.getSpillIndexFile(mapId, 0), new Path( filename[0].getParent(), "file.out.index")); } else { Path indexFileName = task.mapOutputFile.getSpillIndexFile(mapId, i); indexCacheList.add(new SpillRecord(indexFileName, job)); Path finalOutputFile = task.mapOutputFile.getOutputFileForWrite(mapId, finalOutFileSize); Path finalIndexFile = task.mapOutputFile.getOutputIndexFileForWrite(mapId, finalIndexFileSize);
final Path filename = task.mapOutputFile .getSpillFileForWrite(getTaskID(), numSpills, this.memoryBlockAllocator.getEstimatedSize()); out = rfs.create(filename); task.mapOutputFile.getSpillIndexFileForWrite(getTaskID(), numSpills, partitions
if (!this.isInterrupted()) { TaskAttemptID mapId = mapIds.get(i); Path mapOut = mapOutputFiles.get(mapId).getOutputFile(); MapOutputFile localOutputFile = new MapOutputFile(); localOutputFile.setConf(localConf); Path reduceIn = localOutputFile.getInputFileForWrite(mapId.getTaskID(), localFs.getFileStatus(mapOut).getLen()); if (!localFs.mkdirs(reduceIn.getParent())) { output.removeAll();
public void reduce(WritableComparable key, Iterator values, OutputCollector output, Reporter reporter ) throws IOException { if (first) { first = false; MapOutputFile mapOutputFile = new MapOutputFile(); mapOutputFile.setConf(conf); Path input = mapOutputFile.getInputFile(0); FileSystem fs = FileSystem.get(conf); assertTrue("reduce input exists " + input, fs.exists(input)); SequenceFile.Reader rdr = new SequenceFile.Reader(fs, input, conf); assertEquals("is reduce input compressed " + input, compressInput, rdr.isCompressed()); rdr.close(); } }
/** * Calculates the size of output for this task. * * @return -1 if it can't be found. */ private long calculateOutputSize() throws IOException { if (!isMapOrReduce()) { return -1; } if (isMapTask() && conf.getNumReduceTasks() > 0) { try { Path mapOutput = mapOutputFile.getOutputFile(); FileSystem localFS = FileSystem.getLocal(conf); return localFS.getFileStatus(mapOutput).getLen(); } catch (IOException e) { LOG.warn ("Could not find output size " , e); } } return -1; }
Path mapOut = this.mapoutputFile.getOutputFile(mapId); Path reduceIn = this.mapoutputFile.getInputFileForWrite(mapId.getTaskID(),reduceId, localFs.getLength(mapOut)); if (!localFs.mkdirs(reduceIn.getParent())) { this.mapoutputFile.removeAll(mapId); this.mapoutputFile.removeAll(reduceId);
@Deprecated public OnDiskMapOutput(TaskAttemptID mapId, TaskAttemptID reduceId, MergeManagerImpl<K,V> merger, long size, JobConf conf, MapOutputFile mapOutputFile, int fetcher, boolean primaryMapOutput) throws IOException { this(mapId, merger, size, conf, fetcher, primaryMapOutput, FileSystem.getLocal(conf).getRaw(), mapOutputFile.getInputFileForWrite(mapId.getTaskID(), size)); }
public void reduce(WritableComparable key, Iterator values, OutputCollector output, Reporter reporter ) throws IOException { if (first) { first = false; MapOutputFile mapOutputFile = new MROutputFiles(); mapOutputFile.setConf(conf); Path input = mapOutputFile.getInputFile(0); FileSystem fs = FileSystem.get(conf); assertTrue("reduce input exists " + input, fs.exists(input)); SequenceFile.Reader rdr = new SequenceFile.Reader(fs, input, conf); assertEquals("is reduce input compressed " + input, compressInput, rdr.isCompressed()); rdr.close(); } }
private Path[] getMapFiles(FileSystem fs) throws IOException { List<Path> fileList = new ArrayList<Path>(); for(int i = 0; i < numMaps; ++i) { fileList.add(mapOutputFile.getInputFile(i)); } return fileList.toArray(new Path[0]); }
public void close() throws IOException, InterruptedException { long totalSize = 0; for (int i = 0; i < numberOfPartitions; i++) { recordWriters[i].close(); outStreams[i].close(); totalSize += outStreams[i].size(); } MapOutputFile mapOutputFile = mapTask.getMapOutputFile(); Path finalOutput = mapOutputFile.getOutputFileForWrite(totalSize); Path indexPath = mapOutputFile.getOutputIndexFileForWrite( numberOfPartitions*mapTask.MAP_OUTPUT_INDEX_RECORD_LENGTH); // Copy partitions to final map output. copyPartitions(finalOutput, indexPath); }
/** * This function returns the index information for the given mapId, Spill * number and reducer combination. Index Information is obtained * transparently from either the indexMap or the underlying indexFile * @param mapId * @param spillNum * @param reducer * @return * @throws IOException */ private IndexRecord getIndexInformation( TaskAttemptID mapId, int spillNum, int reducer) throws IOException { IndexRecord[] irArray = null; if (indexCacheList.size() > spillNum) { irArray = indexCacheList.get(spillNum); } else { Path indexFileName = mapOutputFile.getSpillIndexFile(mapId, spillNum); irArray = IndexRecord.readIndexFile(indexFileName, job); indexCacheList.add(spillNum,irArray); rfs.delete(indexFileName,false); } return irArray[reducer]; }
filename[i] = mapOutputFile.getSpillFile(i); finalOutFileSize += rfs.getFileStatus(filename[i]).getLen(); mapOutputFile.getOutputFileForWriteInVolume(filename[0])); if (indexCacheList.size() == 0) { sameVolRename(mapOutputFile.getSpillIndexFile(0), mapOutputFile.getOutputIndexFileForWriteInVolume(filename[0])); } else { indexCacheList.get(0).writeToFile( mapOutputFile.getOutputIndexFileForWriteInVolume(filename[0]), job); Path indexFileName = mapOutputFile.getSpillIndexFile(i); indexCacheList.add(new SpillRecord(indexFileName, job)); finalIndexFileSize = partitions * MAP_OUTPUT_INDEX_RECORD_LENGTH; Path finalOutputFile = mapOutputFile.getOutputFileForWrite(finalOutFileSize); Path finalIndexFile = mapOutputFile.getOutputIndexFileForWrite(finalIndexFileSize);
filename[i] = mapOutputFile.getSpillFile(mapId, i); finalOutFileSize += rfs.getFileStatus(filename[i]).getLen(); new Path(filename[0].getParent(), "file.out")); if (indexCacheList.size() == 0) { rfs.rename(mapOutputFile.getSpillIndexFile(mapId, 0), new Path(filename[0].getParent(),"file.out.index")); } else { Path indexFileName = mapOutputFile.getSpillIndexFile(mapId, i); indexCacheList.add(new SpillRecord(indexFileName, job)); Path finalOutputFile = mapOutputFile.getOutputFileForWrite(mapId, finalOutFileSize); Path finalIndexFile = mapOutputFile.getOutputIndexFileForWrite( mapId, finalIndexFileSize);
try { Path filename = mapOutputFile.getSpillFileForWrite(getTaskID(), numSpills, size); out = rfs.create(filename); Path indexFilename = mapOutputFile.getSpillIndexFileForWrite( getTaskID(), numSpills, partitions * MAP_OUTPUT_INDEX_RECORD_LENGTH);