public MergeQueue(Configuration conf, FileSystem fs, Path[] inputs, boolean deleteInputs, CompressionCodec codec, RawComparator<K> comparator, Progressable reporter, Counters.Counter mergedMapOutputsCounter) throws IOException { this.conf = conf; this.fs = fs; this.codec = codec; this.comparator = comparator; this.reporter = reporter; for (Path file : inputs) { LOG.debug("MergeQ: adding: " + file); segments.add(new Segment<K, V>(conf, fs, file, codec, !deleteInputs, (file.toString().endsWith( Task.MERGED_OUTPUT_PREFIX) ? null : mergedMapOutputsCounter))); } // Sort segments on file-lengths Collections.sort(segments, segmentComparator); }
public MergeQueue(Configuration conf, FileSystem fs, Path[] inputs, boolean deleteInputs, CompressionCodec codec, RawComparator<K> comparator, Progressable reporter) throws IOException { this.conf = conf; this.fs = fs; this.codec = codec; this.comparator = comparator; this.reporter = reporter; for (Path file : inputs) { segments.add(new Segment<K, V>(conf, fs, file, codec, !deleteInputs)); } // Sort segments on file-lengths Collections.sort(segments, segmentComparator); }
public MergeQueue(Configuration conf, FileSystem fs, Path[] inputs, boolean deleteInputs, CompressionCodec codec, RawComparator<K> comparator, Progressable reporter) throws IOException { this.conf = conf; this.fs = fs; this.codec = codec; this.comparator = comparator; this.reporter = reporter; for (Path file : inputs) { segments.add(new Segment<K, V>(conf, fs, file, codec, !deleteInputs)); } // Sort segments on file-lengths Collections.sort(segments, segmentComparator); }
new Segment<K, V>(job, rfs, filename[i], segmentOffset, segmentLength, codec, true); segmentList.add(i, s);
new Segment<K,V>(job, rfs, filename[i], indexRecord.startOffset, indexRecord.partLength, codec, true); segmentList.add(i, s);
new Segment<K,V>(job, rfs, filename[i], indexRecord.startOffset, indexRecord.partLength, codec, true); segmentList.add(i, s);
private long createInMemorySegments(List<InMemoryMapOutput<K,V>> inMemoryMapOutputs, List<Segment<K, V>> inMemorySegments, long leaveBytes ) throws IOException { long totalSize = 0L; // We could use fullSize could come from the RamManager, but files can be // closed but not yet present in inMemoryMapOutputs long fullSize = 0L; for (InMemoryMapOutput<K,V> mo : inMemoryMapOutputs) { fullSize += mo.getMemory().length; } while(fullSize > leaveBytes) { InMemoryMapOutput<K,V> mo = inMemoryMapOutputs.remove(0); byte[] data = mo.getMemory(); long size = data.length; totalSize += size; fullSize -= size; Reader<K,V> reader = new InMemoryReader<K,V>(MergeManagerImpl.this, mo.getMapId(), data, 0, (int)size, jobConf); inMemorySegments.add(new Segment<K,V>(reader, true, (mo.isPrimaryMapOutput() ? mergedMapOutputsCounter : null))); } return totalSize; }
private long createInMemorySegments( List<Segment<K, V>> inMemorySegments, long leaveBytes) throws IOException { long totalSize = 0L; synchronized (mapOutputsFilesInMemory) { // fullSize could come from the RamManager, but files can be // closed but not yet present in mapOutputsFilesInMemory long fullSize = 0L; for (MapOutput mo : mapOutputsFilesInMemory) { fullSize += mo.data.length; } while(fullSize > leaveBytes) { MapOutput mo = mapOutputsFilesInMemory.remove(0); totalSize += mo.data.length; fullSize -= mo.data.length; Reader<K, V> reader = new InMemoryReader<K, V>(ramManager, mo.mapAttemptId, mo.data, 0, mo.data.length); Segment<K, V> segment = new Segment<K, V>(reader, true); inMemorySegments.add(segment); } } return totalSize; }
public MergeQueue(Configuration conf, FileSystem fs, Path[] inputs, boolean deleteInputs, CompressionCodec codec, RawComparator<K> comparator, Progressable reporter, Counters.Counter mergedMapOutputsCounter, TaskType taskType) throws IOException { this.conf = conf; this.fs = fs; this.codec = codec; this.comparator = comparator; this.reporter = reporter; if (taskType == TaskType.MAP) { considerFinalMergeForProgress(); } for (Path file : inputs) { LOG.debug("MergeQ: adding: " + file); segments.add(new Segment<K, V>(conf, fs, file, codec, !deleteInputs, (file.toString().endsWith( Task.MERGED_OUTPUT_PREFIX) ? null : mergedMapOutputsCounter))); } // Sort segments on file-lengths Collections.sort(segments, segmentComparator); }
private long createInMemorySegments(List<InMemoryMapOutput<K,V>> inMemoryMapOutputs, List<Segment<K, V>> inMemorySegments, long leaveBytes ) throws IOException { long totalSize = 0L; // We could use fullSize could come from the RamManager, but files can be // closed but not yet present in inMemoryMapOutputs long fullSize = 0L; for (InMemoryMapOutput<K,V> mo : inMemoryMapOutputs) { fullSize += mo.getMemory().length; } while(fullSize > leaveBytes) { InMemoryMapOutput<K,V> mo = inMemoryMapOutputs.remove(0); byte[] data = mo.getMemory(); long size = data.length; totalSize += size; fullSize -= size; Reader<K,V> reader = new InMemoryReader<K,V>(MergeManagerImpl.this, mo.getMapId(), data, 0, (int)size, jobConf); inMemorySegments.add(new Segment<K,V>(reader, true, (mo.isPrimaryMapOutput() ? mergedMapOutputsCounter : null))); } return totalSize; }
private long createInMemorySegments(List<MapOutput<K,V>> inMemoryMapOutputs, List<Segment<K, V>> inMemorySegments, long leaveBytes ) throws IOException { long totalSize = 0L; // We could use fullSize could come from the RamManager, but files can be // closed but not yet present in inMemoryMapOutputs long fullSize = 0L; for (MapOutput<K,V> mo : inMemoryMapOutputs) { fullSize += mo.getMemory().length; } while(fullSize > leaveBytes) { MapOutput<K,V> mo = inMemoryMapOutputs.remove(0); byte[] data = mo.getMemory(); long size = data.length; totalSize += size; fullSize -= size; Reader<K,V> reader = new InMemoryReader<K,V>(MergeManager.this, mo.getMapId(), data, 0, (int)size); inMemorySegments.add(new Segment<K,V>(reader, true, (mo.isPrimaryMapOutput() ? mergedMapOutputsCounter : null))); } return totalSize; }
private long createInMemorySegments(List<InMemoryMapOutput<K,V>> inMemoryMapOutputs, List<Segment<K, V>> inMemorySegments, long leaveBytes ) throws IOException { long totalSize = 0L; // We could use fullSize could come from the RamManager, but files can be // closed but not yet present in inMemoryMapOutputs long fullSize = 0L; for (InMemoryMapOutput<K,V> mo : inMemoryMapOutputs) { fullSize += mo.getMemory().length; } while(fullSize > leaveBytes) { InMemoryMapOutput<K,V> mo = inMemoryMapOutputs.remove(0); byte[] data = mo.getMemory(); long size = data.length; totalSize += size; fullSize -= size; Reader<K,V> reader = new InMemoryReader<K,V>(MergeManagerImpl.this, mo.getMapId(), data, 0, (int)size, jobConf); inMemorySegments.add(new Segment<K,V>(reader, true, (mo.isPrimaryMapOutput() ? mergedMapOutputsCounter : null))); } return totalSize; }
private long createInMemorySegments( List<Segment<K, V>> inMemorySegments, long leaveBytes) throws IOException { long totalSize = 0L; synchronized (mapOutputsFilesInMemory) { // fullSize could come from the RamManager, but files can be // closed but not yet present in mapOutputsFilesInMemory long fullSize = 0L; for (MapOutput mo : mapOutputsFilesInMemory) { fullSize += mo.data.length; } while(fullSize > leaveBytes) { MapOutput mo = mapOutputsFilesInMemory.remove(0); totalSize += mo.data.length; fullSize -= mo.data.length; Reader<K, V> reader = new InMemoryReader<K, V>(ramManager, mo.mapAttemptId, mo.data, 0, mo.data.length); Segment<K, V> segment = new Segment<K, V>(reader, true); inMemorySegments.add(segment); } } return totalSize; }
private long createInMemorySegments(List<InMemoryMapOutput<K,V>> inMemoryMapOutputs, List<Segment<K, V>> inMemorySegments, long leaveBytes ) throws IOException { long totalSize = 0L; // We could use fullSize could come from the RamManager, but files can be // closed but not yet present in inMemoryMapOutputs long fullSize = 0L; for (InMemoryMapOutput<K,V> mo : inMemoryMapOutputs) { fullSize += mo.getMemory().length; } while(fullSize > leaveBytes) { InMemoryMapOutput<K,V> mo = inMemoryMapOutputs.remove(0); byte[] data = mo.getMemory(); long size = data.length; totalSize += size; fullSize -= size; Reader<K,V> reader = new InMemoryReader<K,V>(MergeManagerImpl.this, mo.getMapId(), data, 0, (int)size, jobConf); inMemorySegments.add(new Segment<K,V>(reader, true, (mo.isPrimaryMapOutput() ? mergedMapOutputsCounter : null))); } return totalSize; }
public MergeQueue(Configuration conf, FileSystem fs, Path[] inputs, boolean deleteInputs, CompressionCodec codec, RawComparator<K> comparator, Progressable reporter, Counters.Counter mergedMapOutputsCounter, TaskType taskType) throws IOException { this.conf = conf; this.fs = fs; this.codec = codec; this.comparator = comparator; this.reporter = reporter; if (taskType == TaskType.MAP) { considerFinalMergeForProgress(); } for (Path file : inputs) { LOG.debug("MergeQ: adding: " + file); segments.add(new Segment<K, V>(conf, fs, file, codec, !deleteInputs, (file.toString().endsWith( Task.MERGED_OUTPUT_PREFIX) ? null : mergedMapOutputsCounter))); } // Sort segments on file-lengths Collections.sort(segments, segmentComparator); }
public MergeQueue(Configuration conf, FileSystem fs, Path[] inputs, boolean deleteInputs, CompressionCodec codec, RawComparator<K> comparator, Progressable reporter, Counters.Counter mergedMapOutputsCounter, TaskType taskType) throws IOException { this.conf = conf; this.fs = fs; this.codec = codec; this.comparator = comparator; this.reporter = reporter; if (taskType == TaskType.MAP) { considerFinalMergeForProgress(); } for (Path file : inputs) { LOG.debug("MergeQ: adding: " + file); segments.add(new Segment<K, V>(conf, fs, file, codec, !deleteInputs, (file.toString().endsWith( Task.MERGED_OUTPUT_PREFIX) ? null : mergedMapOutputsCounter))); } // Sort segments on file-lengths Collections.sort(segments, segmentComparator); }
public MergeQueue(Configuration conf, FileSystem fs, Path[] inputs, boolean deleteInputs, CompressionCodec codec, RawComparator<K> comparator, Progressable reporter, Counters.Counter mergedMapOutputsCounter, TaskType taskType) throws IOException { this.conf = conf; this.fs = fs; this.codec = codec; this.comparator = comparator; this.reporter = reporter; if (taskType == TaskType.MAP) { considerFinalMergeForProgress(); } for (Path file : inputs) { LOG.debug("MergeQ: adding: " + file); segments.add(new Segment<K, V>(conf, fs, file, codec, !deleteInputs, (file.toString().endsWith( Task.MERGED_OUTPUT_PREFIX) ? null : mergedMapOutputsCounter))); } // Sort segments on file-lengths Collections.sort(segments, segmentComparator); }
public synchronized void flush() throws IOException, ClassNotFoundException, InterruptedException { if (numSpills > 0 && lastSpillInMem) { // if there is already one spills, we can try to hold this last spill in // memory. sortReduceParts(); for (int i = 0; i < partitions; i++) { this.inMemorySegments[i] = new Segment<K, V>(this.reducePartitions[i].getIReader(), true); } hasInMemorySpill=true; } else { sortAndSpill(); } long mergeStartMilli = System.currentTimeMillis(); ProcResourceValues mergeStartProcVals = task.getCurrentProcResourceValues(); mergeParts(); long mergeEndMilli = System.currentTimeMillis(); ProcResourceValues mergeEndProcVals = task.getCurrentProcResourceValues(); mapSpillSortCounter.incMergeCounters(mergeStartProcVals, mergeEndProcVals, mergeEndMilli - mergeStartMilli); }
private Segment<Text, Text> getCompressedSegment(int i) throws IOException { return new Segment<Text, Text>(getReader(i, true), false, 3000l); }
private Segment<Text, Text> getUncompressedSegment(int i) throws IOException { return new Segment<Text, Text>(getReader(i, false), false); }