for(int i = 0; i < 2*SORT_FACTOR; ++i) { Path path = new Path("somePath"); CompressAwarePath cap = new CompressAwarePath(path, 1l, rand.nextInt()); manager.closeOnDiskFile(cap); inputs.size() > 0 && inputs.size() <= SORT_FACTOR); assertTrue("Inputs to be merged were not sorted according to size: ", inputs.get(j).getCompressedSize() >= inputs.get(j-1).getCompressedSize());
Merger.writeFile(rIter, writer, reporter, job); writer.close(); onDiskMapOutputs.add(new CompressAwarePath(outputPath, writer.getRawLength(), writer.getCompressedLength())); writer = null; long fileLength = fs.getFileStatus(file).getLen(); onDiskBytes += fileLength; rawBytes += (file.getRawDataLength() > 0) ? file.getRawDataLength() : fileLength; (file.toString().endsWith( Task.MERGED_OUTPUT_PREFIX) ? null : mergedMapOutputsCounter), file.getRawDataLength() ));
Merger.writeFile(rIter, writer, reporter, job); writer.close(); onDiskMapOutputs.add(new CompressAwarePath(outputPath, writer.getRawLength(), writer.getCompressedLength())); writer = null; long fileLength = fs.getFileStatus(file).getLen(); onDiskBytes += fileLength; rawBytes += (file.getRawDataLength() > 0) ? file.getRawDataLength() : fileLength; (file.toString().endsWith( Task.MERGED_OUTPUT_PREFIX) ? null : mergedMapOutputsCounter), file.getRawDataLength() ));
Merger.writeFile(rIter, writer, reporter, job); writer.close(); onDiskMapOutputs.add(new CompressAwarePath(outputPath, writer.getRawLength(), writer.getCompressedLength())); writer = null; long fileLength = fs.getFileStatus(file).getLen(); onDiskBytes += fileLength; rawBytes += (file.getRawDataLength() > 0) ? file.getRawDataLength() : fileLength; (file.toString().endsWith( Task.MERGED_OUTPUT_PREFIX) ? null : mergedMapOutputsCounter), file.getRawDataLength() ));
Merger.writeFile(rIter, writer, reporter, job); writer.close(); onDiskMapOutputs.add(new CompressAwarePath(outputPath, writer.getRawLength(), writer.getCompressedLength())); writer = null; long fileLength = fs.getFileStatus(file).getLen(); onDiskBytes += fileLength; rawBytes += (file.getRawDataLength() > 0) ? file.getRawDataLength() : fileLength; (file.toString().endsWith( Task.MERGED_OUTPUT_PREFIX) ? null : mergedMapOutputsCounter), file.getRawDataLength() ));
@Override public int compareTo(Object obj) { if(obj instanceof CompressAwarePath) { CompressAwarePath compPath = (CompressAwarePath) obj; if(this.compressedSize < compPath.getCompressedSize()) { return -1; } else if (this.getCompressedSize() > compPath.getCompressedSize()) { return 1; } // Not returning 0 here so that objects with the same size (but // different paths) are still added to the TreeSet. } return super.compareTo(obj); } }
@Override public int compareTo(Object obj) { if(obj instanceof CompressAwarePath) { CompressAwarePath compPath = (CompressAwarePath) obj; if(this.compressedSize < compPath.getCompressedSize()) { return -1; } else if (this.getCompressedSize() > compPath.getCompressedSize()) { return 1; } // Not returning 0 here so that objects with the same size (but // different paths) are still added to the TreeSet. } return super.compareTo(obj); } }
@Override public int compareTo(Object obj) { if(obj instanceof CompressAwarePath) { CompressAwarePath compPath = (CompressAwarePath) obj; if(this.compressedSize < compPath.getCompressedSize()) { return -1; } else if (this.getCompressedSize() > compPath.getCompressedSize()) { return 1; } // Not returning 0 here so that objects with the same size (but // different paths) are still added to the TreeSet. } return super.compareTo(obj); } }
@Override public int compareTo(Object obj) { if(obj instanceof CompressAwarePath) { CompressAwarePath compPath = (CompressAwarePath) obj; if(this.compressedSize < compPath.getCompressedSize()) { return -1; } else if (this.getCompressedSize() > compPath.getCompressedSize()) { return 1; } // Not returning 0 here so that objects with the same size (but // different paths) are still added to the TreeSet. } return super.compareTo(obj); } }
@Override public void commit() throws IOException { fs.rename(tmpOutputPath, outputPath); CompressAwarePath compressAwarePath = new CompressAwarePath(outputPath, getSize(), this.compressedSize); getMerger().closeOnDiskFile(compressAwarePath); }
@Override public void commit() throws IOException { fs.rename(tmpOutputPath, outputPath); CompressAwarePath compressAwarePath = new CompressAwarePath(outputPath, getSize(), this.compressedSize); merger.closeOnDiskFile(compressAwarePath); }
@Override public void commit() throws IOException { fs.rename(tmpOutputPath, outputPath); CompressAwarePath compressAwarePath = new CompressAwarePath(outputPath, getSize(), this.compressedSize); merger.closeOnDiskFile(compressAwarePath); }
@Override public void commit() throws IOException { fs.rename(tmpOutputPath, outputPath); CompressAwarePath compressAwarePath = new CompressAwarePath(outputPath, getSize(), this.compressedSize); merger.closeOnDiskFile(compressAwarePath); }