offset += uaoSize; final UnsafeSorterSpillWriter writer = new UnsafeSorterSpillWriter(blockManager, 32 * 1024, writeMetrics, numRecords); while (numRecords > 0) { int length = UnsafeAlignedOffset.getSize(base, offset); writer.write(base, offset + uaoSize, length, 0); offset += uaoSize + length + 8; numRecords--; writer.close(); spillWriters.add(writer);
/** * Returns a iterator, which will return the rows in the order as inserted. * * It is the caller's responsibility to call `cleanupResources()` * after consuming this iterator. * * TODO: support forced spilling */ public UnsafeSorterIterator getIterator() throws IOException { if (spillWriters.isEmpty()) { assert(inMemSorter != null); return inMemSorter.getSortedIterator(); } else { LinkedList<UnsafeSorterIterator> queue = new LinkedList<>(); for (UnsafeSorterSpillWriter spillWriter : spillWriters) { queue.add(spillWriter.getReader(serializerManager)); } if (inMemSorter != null) { queue.add(inMemSorter.getSortedIterator()); } return new ChainedIterator(queue); } }
int i = 0; for (UnsafeSorterSpillWriter spillWriter : spillWriters) { if (i + spillWriter.recordsSpilled() > startIndex) { UnsafeSorterIterator iter = spillWriter.getReader(serializerManager); moveOver(iter, startIndex - i); queue.add(iter); i += spillWriter.recordsSpilled();
private static void spillIterator(UnsafeSorterIterator inMemIterator, UnsafeSorterSpillWriter spillWriter) throws IOException { while (inMemIterator.hasNext()) { inMemIterator.loadNext(); final Object baseObject = inMemIterator.getBaseObject(); final long baseOffset = inMemIterator.getBaseOffset(); final int recordLength = inMemIterator.getRecordLength(); spillWriter.write(baseObject, baseOffset, recordLength, inMemIterator.getKeyPrefix()); } spillWriter.close(); }
new UnsafeSorterSpillWriter(blockManager, fileBufferSizeBytes, writeMetrics, numRecords); while (inMemIterator.hasNext()) { inMemIterator.loadNext(); final long baseOffset = inMemIterator.getBaseOffset(); final int recordLength = inMemIterator.getRecordLength(); spillWriter.write(baseObject, baseOffset, recordLength, inMemIterator.getKeyPrefix()); spillWriter.close(); spillWriters.add(spillWriter); nextUpstream = spillWriter.getReader(serializerManager);
new UnsafeSorterSpillWriter(blockManager, fileBufferSizeBytes, writeMetrics, numRecords); spillIterator(inMemIterator, spillWriter); spillWriters.add(spillWriter); nextUpstream = spillWriter.getReader(serializerManager);
numRecordsSpilled++; writeIntToBuffer(recordLength, 0); writeLongToBuffer(keyPrefix, 4); int dataRemaining = recordLength; int freeSpaceInWriteBuffer = diskWriteBufferSize - 4 - 8; // space used by prefix + len
private void handleFailedDelete() { // remove the spill file from disk File file = spillWriters.removeFirst().getFile(); if (file != null && file.exists() && !file.delete()) { logger.error("Was unable to delete spill file {}", file.getAbsolutePath()); } } }
new UnsafeSorterSpillWriter(blockManager, fileBufferSizeBytes, writeMetrics, inMemSorter.numRecords()); spillWriters.add(spillWriter);
public UnsafeSorterSpillWriter( BlockManager blockManager, int fileBufferSize, ShuffleWriteMetrics writeMetrics, int numRecordsToWrite) throws IOException { final Tuple2<TempLocalBlockId, File> spilledFileInfo = blockManager.diskBlockManager().createTempLocalBlock(); this.file = spilledFileInfo._2(); this.blockId = spilledFileInfo._1(); this.numRecordsToWrite = numRecordsToWrite; // Unfortunately, we need a serializer instance in order to construct a DiskBlockObjectWriter. // Our write path doesn't actually use this serializer (since we end up calling the `write()` // OutputStream methods), but DiskBlockObjectWriter still calls some methods on it. To work // around this, we pass a dummy no-op serializer. writer = blockManager.getDiskWriter( blockId, file, DummySerializerInstance.INSTANCE, fileBufferSize, writeMetrics); // Write the number of records writeIntToBuffer(numRecordsToWrite, 0); writer.write(writeBuffer, 0, 4); }
spillWriters.removeFirst().removeFile(); } catch (IOException e) { logger.error("", e); reader = spillWriters.getFirst().getReader(); recordsInPage = -1; } catch (IOException e) {
new UnsafeSorterSpillWriter(numRecords); while (inMemIterator.hasNext()) { inMemIterator.loadNext(); final long baseOffset = inMemIterator.getBaseOffset(); final int recordLength = inMemIterator.getRecordLength(); spillWriter.write(baseObject, baseOffset, recordLength, inMemIterator.getKeyPrefix()); spillWriter.close(); spillWriters.add(spillWriter); nextUpstream = spillWriter.getReader();
private static void spillIterator(UnsafeSorterIterator inMemIterator, UnsafeSorterSpillWriter spillWriter) throws IOException { while (inMemIterator.hasNext()) { inMemIterator.loadNext(); final Object baseObject = inMemIterator.getBaseObject(); final long baseOffset = inMemIterator.getBaseOffset(); final int recordLength = inMemIterator.getRecordLength(); spillWriter.write(baseObject, baseOffset, recordLength, inMemIterator.getKeyPrefix()); } spillWriter.close(); }
new UnsafeSorterSpillWriter(blockManager, fileBufferSizeBytes, writeMetrics, numRecords); spillIterator(inMemIterator, spillWriter); spillWriters.add(spillWriter); nextUpstream = spillWriter.getReader(serializerManager);
numRecordsSpilled++; writeIntToBuffer(recordLength, 0); writeLongToBuffer(keyPrefix, 4); int dataRemaining = recordLength; int freeSpaceInWriteBuffer = diskWriteBufferSize - 4 - 8; // space used by prefix + len
private void handleFailedDelete() { // remove the spill file from disk File file = spillWriters.removeFirst().getFile(); if (file != null && file.exists() && !file.delete()) { logger.error("Was unable to delete spill file {}", file.getAbsolutePath()); } } }
int i = 0; for (UnsafeSorterSpillWriter spillWriter : spillWriters) { if (i + spillWriter.recordsSpilled() > startIndex) { UnsafeSorterIterator iter = spillWriter.getReader(serializerManager); moveOver(iter, startIndex - i); queue.add(iter); i += spillWriter.recordsSpilled();
new UnsafeSorterSpillWriter(blockManager, fileBufferSizeBytes, writeMetrics, inMemSorter.numRecords()); spillWriters.add(spillWriter);
public UnsafeSorterSpillWriter( BlockManager blockManager, int fileBufferSize, ShuffleWriteMetrics writeMetrics, int numRecordsToWrite) throws IOException { final Tuple2<TempLocalBlockId, File> spilledFileInfo = blockManager.diskBlockManager().createTempLocalBlock(); this.file = spilledFileInfo._2(); this.blockId = spilledFileInfo._1(); this.numRecordsToWrite = numRecordsToWrite; // Unfortunately, we need a serializer instance in order to construct a DiskBlockObjectWriter. // Our write path doesn't actually use this serializer (since we end up calling the `write()` // OutputStream methods), but DiskBlockObjectWriter still calls some methods on it. To work // around this, we pass a dummy no-op serializer. writer = blockManager.getDiskWriter( blockId, file, DummySerializerInstance.INSTANCE, fileBufferSize, writeMetrics); // Write the number of records writeIntToBuffer(numRecordsToWrite, 0); writer.write(writeBuffer, 0, 4); }