/** * Get spill offsets for given partition. */ public IndexRecord getIndex(int partition) { final int pos = partition * MapTask.MAP_OUTPUT_INDEX_RECORD_LENGTH / 8; return new IndexRecord(entries.get(pos), entries.get(pos + 1), entries.get(pos + 2)); }
/** * Get spill offsets for given partition. */ public IndexRecord getIndex(int partition) { final int pos = partition * MapTask.MAP_OUTPUT_INDEX_RECORD_LENGTH / 8; return new IndexRecord(entries.get(pos), entries.get(pos + 1), entries.get(pos + 2)); }
/** * Get spill offsets for given partition. */ public IndexRecord getIndex(int partition) { final int pos = partition * MapTask.MAP_OUTPUT_INDEX_RECORD_LENGTH / 8; return new IndexRecord(entries.get(pos), entries.get(pos + 1), entries.get(pos + 2)); }
/** * Get spill offsets for given partition. */ public IndexRecord getIndex(int partition) { final int pos = partition * MapTask.MAP_OUTPUT_INDEX_RECORD_LENGTH / 8; return new IndexRecord(entries.get(pos), entries.get(pos + 1), entries.get(pos + 2)); }
/** * Get spill offsets for given partition. */ public IndexRecord getIndex(int partition) { final int pos = partition * MapTask.MAP_OUTPUT_INDEX_RECORD_LENGTH / 8; return new IndexRecord(entries.get(pos), entries.get(pos + 1), entries.get(pos + 2)); }
/** * Get spill offsets for given partition. */ public IndexRecord getIndex(int partition) { final int pos = partition * MapTask.MAP_OUTPUT_INDEX_RECORD_LENGTH / 8; return new IndexRecord(entries.get(pos), entries.get(pos + 1), entries.get(pos + 2)); }
long partLength = wrapper.readLong(); indexRecordArray[i] = new IndexRecord(startOffset, rawLength, partLength);
irArray[i] = new IndexRecord(segmentStart, writer.getRawLength(), writer.getCompressedLength());
private void copyPartitions(Path mapOutputPath, Path indexPath) throws IOException { FileSystem localFs = FileSystem.getLocal(jobConf); FileSystem rfs = ((LocalFileSystem)localFs).getRaw(); FSDataOutputStream rawOutput = rfs.create(mapOutputPath, true, BUF_SIZE); SpillRecord spillRecord = new SpillRecord(numberOfPartitions); IndexRecord indexRecord = new IndexRecord(); for (int i = 0; i < numberOfPartitions; i++) { indexRecord.startOffset = rawOutput.getPos(); byte buffer[] = outStreams[i].toByteArray(); IFileOutputStream checksumOutput = new IFileOutputStream(rawOutput); checksumOutput.write(buffer); // Write checksum. checksumOutput.finish(); // Write index record indexRecord.rawLength = (long)buffer.length; indexRecord.partLength = rawOutput.getPos() - indexRecord.startOffset; spillRecord.putIndex(indexRecord, i); reporter.progress(); } rawOutput.close(); spillRecord.writeToFile(indexPath, jobConf); } }
sorter.sort(MapOutputBuffer.this, mstart, mend, reporter); int spindex = mstart; final IndexRecord rec = new IndexRecord(); final InMemValBytes value = new InMemValBytes(); for (int i = 0; i < partitions; ++i) {
numSpills, key.getLength() + value.getLength()); out = rfs.create(filename); IndexRecord rec = new IndexRecord(); for (int i = 0; i < partitions; ++i) { IFile.Writer<K, V> writer = null;
IndexRecord rec = new IndexRecord(); for (int i = 0; i < partitions; ++i) { IFile.Writer<K, V> writer = null;
IndexRecord rec = new IndexRecord(); for (int i = 0; i < partitions; ++i) { IFile.Writer<K, V> writer = null;
sorter.sort(MapOutputBuffer.this, mstart, mend, reporter); int spindex = mstart; final IndexRecord rec = new IndexRecord(); final InMemValBytes value = new InMemValBytes(); for (int i = 0; i < partitions; ++i) {
sorter.sort(MapOutputBuffer.this, mstart, mend, reporter); int spindex = mstart; final IndexRecord rec = new IndexRecord(); final InMemValBytes value = new InMemValBytes(); for (int i = 0; i < partitions; ++i) {
IndexRecord rec = new IndexRecord(); for (int i = 0; i < partitions; ++i) { IFile.Writer<K, V> writer = null;
IndexRecord rec = new IndexRecord(); for (int i = 0; i < partitions; ++i) { IFile.Writer<K, V> writer = null;
IndexRecord rec = new IndexRecord(); for (int i = 0; i < partitions; ++i) { IFile.Writer<K, V> writer = null;
public IndexRecord spill(JobConf job, FSDataOutputStream out, Class<K> keyClass, Class<V> valClass, CompressionCodec codec, Counter spillCounter) throws IOException { IFile.Writer<K, V> writer = null; IndexRecord rec = new IndexRecord(); long segmentStart = out.getPos(); try { writer = new Writer<K, V>(job, out, keyClass, valClass, codec, spillCounter); // spill directly KeyValueSpillIterator kvSortedArray = this.getKeyValueSpillIterator(); MemoryBlockIndex memBlkIdx = kvSortedArray.next(); while (memBlkIdx != null) { int pos = memBlkIdx.getIndex(); MemoryBlock memBlk = memBlkIdx.getMemoryBlock(); writer.append(kvbuffer, memBlk.offsets[pos], memBlk.keyLenArray[pos], memBlk.valueLenArray[pos]); memBlkIdx = kvSortedArray.next(); } } finally { // close the writer if (null != writer) writer.close(); } rec.startOffset = segmentStart; rec.rawLength = writer.getRawLength(); rec.partLength = writer.getCompressedLength(); writer = null; return rec; }
IndexRecord rec = new IndexRecord(); for (int i = 0; i < partitions; ++i) { IFile.Writer<K, V> writer = null;