/** * Merges another UnsafeExternalSorters into this one, the other one will be emptied. * * @throws IOException */ public void merge(UnsafeExternalSorter other) throws IOException { other.spill(); spillWriters.addAll(other.spillWriters); // remove them from `spillWriters`, or the files will be deleted in `cleanupResources`. other.spillWriters.clear(); other.cleanupResources(); }
/** * Merges another UnsafeExternalSorters into this one, the other one will be emptied. * * @throws IOException */ public void merge(UnsafeExternalSorter other) throws IOException { other.spill(); spillWriters.addAll(other.spillWriters); // remove them from `spillWriters`, or the files will be deleted in `cleanupResources`. other.spillWriters.clear(); other.cleanupResources(); }
/** * Merges another UnsafeExternalSorters into this one, the other one will be emptied. * * @throws IOException */ public void merge(UnsafeExternalSorter other) throws IOException { other.spill(); spillWriters.addAll(other.spillWriters); // remove them from `spillWriters`, or the files will be deleted in `cleanupResources`. other.spillWriters.clear(); other.cleanupResources(); }
public static UnsafeExternalSorter createWithExistingInMemorySorter( TaskMemoryManager taskMemoryManager, BlockManager blockManager, SerializerManager serializerManager, TaskContext taskContext, Supplier<RecordComparator> recordComparatorSupplier, PrefixComparator prefixComparator, int initialSize, long pageSizeBytes, int numElementsForSpillThreshold, UnsafeInMemorySorter inMemorySorter) throws IOException { UnsafeExternalSorter sorter = new UnsafeExternalSorter(taskMemoryManager, blockManager, serializerManager, taskContext, recordComparatorSupplier, prefixComparator, initialSize, pageSizeBytes, numElementsForSpillThreshold, inMemorySorter, false /* ignored */); sorter.spill(Long.MAX_VALUE, sorter); // The external sorter will be used to insert records, in-memory sorter is not needed. sorter.inMemSorter = null; return sorter; }
public static UnsafeExternalSorter createWithExistingInMemorySorter( TaskMemoryManager taskMemoryManager, BlockManager blockManager, SerializerManager serializerManager, TaskContext taskContext, Supplier<RecordComparator> recordComparatorSupplier, PrefixComparator prefixComparator, int initialSize, long pageSizeBytes, int numElementsForSpillThreshold, UnsafeInMemorySorter inMemorySorter) throws IOException { UnsafeExternalSorter sorter = new UnsafeExternalSorter(taskMemoryManager, blockManager, serializerManager, taskContext, recordComparatorSupplier, prefixComparator, initialSize, pageSizeBytes, numElementsForSpillThreshold, inMemorySorter, false /* ignored */); sorter.spill(Long.MAX_VALUE, sorter); // The external sorter will be used to insert records, in-memory sorter is not needed. sorter.inMemSorter = null; return sorter; }
public static UnsafeExternalSorter createWithExistingInMemorySorter( TaskMemoryManager taskMemoryManager, BlockManager blockManager, SerializerManager serializerManager, TaskContext taskContext, RecordComparator recordComparator, PrefixComparator prefixComparator, int initialSize, long pageSizeBytes, long numElementsForSpillThreshold, UnsafeInMemorySorter inMemorySorter) throws IOException { UnsafeExternalSorter sorter = new UnsafeExternalSorter(taskMemoryManager, blockManager, serializerManager, taskContext, recordComparator, prefixComparator, initialSize, numElementsForSpillThreshold, pageSizeBytes, inMemorySorter, false /* ignored */); sorter.spill(Long.MAX_VALUE, sorter); // The external sorter will be used to insert records, in-memory sorter is not needed. sorter.inMemSorter = null; return sorter; }
/** * Write a record to the sorter. */ public void insertRecord( Object recordBase, long recordOffset, int length, long prefix, boolean prefixIsNull) throws IOException { assert(inMemSorter != null); if (inMemSorter.numRecords() >= numElementsForSpillThreshold) { logger.info("Spilling data because number of spilledRecords crossed the threshold " + numElementsForSpillThreshold); spill(); } growPointerArrayIfNecessary(); int uaoSize = UnsafeAlignedOffset.getUaoSize(); // Need 4 or 8 bytes to store the record length. final int required = length + uaoSize; acquireNewPageIfNecessary(required); final Object base = currentPage.getBaseObject(); final long recordAddress = taskMemoryManager.encodePageNumberAndOffset(currentPage, pageCursor); UnsafeAlignedOffset.putSize(base, pageCursor, length); pageCursor += uaoSize; Platform.copyMemory(recordBase, recordOffset, base, pageCursor, length); pageCursor += length; inMemSorter.insertRecord(recordAddress, prefix, prefixIsNull); }
/** * Write a record to the sorter. */ public void insertRecord( Object recordBase, long recordOffset, int length, long prefix, boolean prefixIsNull) throws IOException { assert(inMemSorter != null); if (inMemSorter.numRecords() >= numElementsForSpillThreshold) { logger.info("Spilling data because number of spilledRecords crossed the threshold " + numElementsForSpillThreshold); spill(); } growPointerArrayIfNecessary(); int uaoSize = UnsafeAlignedOffset.getUaoSize(); // Need 4 or 8 bytes to store the record length. final int required = length + uaoSize; acquireNewPageIfNecessary(required); final Object base = currentPage.getBaseObject(); final long recordAddress = taskMemoryManager.encodePageNumberAndOffset(currentPage, pageCursor); UnsafeAlignedOffset.putSize(base, pageCursor, length); pageCursor += uaoSize; Platform.copyMemory(recordBase, recordOffset, base, pageCursor, length); pageCursor += length; inMemSorter.insertRecord(recordAddress, prefix, prefixIsNull); }
/** * Write a record to the sorter. */ public void insertRecord( Object recordBase, long recordOffset, int length, long prefix, boolean prefixIsNull) throws IOException { assert(inMemSorter != null); if (inMemSorter.numRecords() >= numElementsForSpillThreshold) { logger.info("Spilling data because number of spilledRecords crossed the threshold " + numElementsForSpillThreshold); spill(); } growPointerArrayIfNecessary(); int uaoSize = UnsafeAlignedOffset.getUaoSize(); // Need 4 bytes to store the record length. final int required = length + uaoSize; acquireNewPageIfNecessary(required); final Object base = currentPage.getBaseObject(); final long recordAddress = taskMemoryManager.encodePageNumberAndOffset(currentPage, pageCursor); UnsafeAlignedOffset.putSize(base, pageCursor, length); pageCursor += uaoSize; Platform.copyMemory(recordBase, recordOffset, base, pageCursor, length); pageCursor += length; inMemSorter.insertRecord(recordAddress, prefix, prefixIsNull); }
@Test public void testSortTimeMetric() throws Exception { final UnsafeExternalSorter sorter = newSorter(); long prevSortTime = sorter.getSortTimeNanos(); assertEquals(prevSortTime, 0); sorter.insertRecord(null, 0, 0, 0, false); sorter.spill(); assertThat(sorter.getSortTimeNanos(), greaterThan(prevSortTime)); prevSortTime = sorter.getSortTimeNanos(); sorter.spill(); // no sort needed assertEquals(sorter.getSortTimeNanos(), prevSortTime); sorter.insertRecord(null, 0, 0, 0, false); UnsafeSorterIterator iter = sorter.getSortedIterator(); assertThat(sorter.getSortTimeNanos(), greaterThan(prevSortTime)); }
@Test public void testSortTimeMetric() throws Exception { final UnsafeExternalSorter sorter = newSorter(); long prevSortTime = sorter.getSortTimeNanos(); assertEquals(prevSortTime, 0); sorter.insertRecord(null, 0, 0, 0, false); sorter.spill(); assertThat(sorter.getSortTimeNanos(), greaterThan(prevSortTime)); prevSortTime = sorter.getSortTimeNanos(); sorter.spill(); // no sort needed assertEquals(sorter.getSortTimeNanos(), prevSortTime); sorter.insertRecord(null, 0, 0, 0, false); UnsafeSorterIterator iter = sorter.getSortedIterator(); assertThat(sorter.getSortTimeNanos(), greaterThan(prevSortTime)); }
@Test public void testSortTimeMetric() throws Exception { final UnsafeExternalSorter sorter = newSorter(); long prevSortTime = sorter.getSortTimeNanos(); assertEquals(prevSortTime, 0); sorter.insertRecord(null, 0, 0, 0, false); sorter.spill(); assertThat(sorter.getSortTimeNanos(), greaterThan(prevSortTime)); prevSortTime = sorter.getSortTimeNanos(); sorter.spill(); // no sort needed assertEquals(sorter.getSortTimeNanos(), prevSortTime); sorter.insertRecord(null, 0, 0, 0, false); UnsafeSorterIterator iter = sorter.getSortedIterator(); assertThat(sorter.getSortTimeNanos(), greaterThan(prevSortTime)); }
@Test public void testSortingEmptyArrays() throws Exception { final UnsafeExternalSorter sorter = newSorter(); sorter.insertRecord(null, 0, 0, 0, false); sorter.insertRecord(null, 0, 0, 0, false); sorter.spill(); sorter.insertRecord(null, 0, 0, 0, false); sorter.spill(); sorter.insertRecord(null, 0, 0, 0, false); sorter.insertRecord(null, 0, 0, 0, false); UnsafeSorterIterator iter = sorter.getSortedIterator(); for (int i = 1; i <= 5; i++) { iter.loadNext(); assertEquals(0, iter.getKeyPrefix()); assertEquals(0, iter.getRecordLength()); } sorter.cleanupResources(); assertSpillFilesWereCleanedUp(); }
@Test public void testSortingEmptyArrays() throws Exception { final UnsafeExternalSorter sorter = newSorter(); sorter.insertRecord(null, 0, 0, 0, false); sorter.insertRecord(null, 0, 0, 0, false); sorter.spill(); sorter.insertRecord(null, 0, 0, 0, false); sorter.spill(); sorter.insertRecord(null, 0, 0, 0, false); sorter.insertRecord(null, 0, 0, 0, false); UnsafeSorterIterator iter = sorter.getSortedIterator(); for (int i = 1; i <= 5; i++) { iter.loadNext(); assertEquals(0, iter.getKeyPrefix()); assertEquals(0, iter.getRecordLength()); } sorter.cleanupResources(); assertSpillFilesWereCleanedUp(); }
@Test public void testSortingEmptyArrays() throws Exception { final UnsafeExternalSorter sorter = newSorter(); sorter.insertRecord(null, 0, 0, 0, false); sorter.insertRecord(null, 0, 0, 0, false); sorter.spill(); sorter.insertRecord(null, 0, 0, 0, false); sorter.spill(); sorter.insertRecord(null, 0, 0, 0, false); sorter.insertRecord(null, 0, 0, 0, false); UnsafeSorterIterator iter = sorter.getSortedIterator(); for (int i = 1; i <= 5; i++) { iter.loadNext(); assertEquals(0, iter.getKeyPrefix()); assertEquals(0, iter.getRecordLength()); } sorter.cleanupResources(); assertSpillFilesWereCleanedUp(); }
@Test public void testGetIterator() throws Exception { final UnsafeExternalSorter sorter = newSorter(); for (int i = 0; i < 100; i++) { insertNumber(sorter, i); } verifyIntIterator(sorter.getIterator(0), 0, 100); verifyIntIterator(sorter.getIterator(79), 79, 100); sorter.spill(); for (int i = 100; i < 200; i++) { insertNumber(sorter, i); } sorter.spill(); verifyIntIterator(sorter.getIterator(79), 79, 200); for (int i = 200; i < 300; i++) { insertNumber(sorter, i); } verifyIntIterator(sorter.getIterator(79), 79, 300); verifyIntIterator(sorter.getIterator(139), 139, 300); verifyIntIterator(sorter.getIterator(279), 279, 300); }
@Test public void testGetIterator() throws Exception { final UnsafeExternalSorter sorter = newSorter(); for (int i = 0; i < 100; i++) { insertNumber(sorter, i); } verifyIntIterator(sorter.getIterator(0), 0, 100); verifyIntIterator(sorter.getIterator(79), 79, 100); sorter.spill(); for (int i = 100; i < 200; i++) { insertNumber(sorter, i); } sorter.spill(); verifyIntIterator(sorter.getIterator(79), 79, 200); for (int i = 200; i < 300; i++) { insertNumber(sorter, i); } verifyIntIterator(sorter.getIterator(79), 79, 300); verifyIntIterator(sorter.getIterator(139), 139, 300); verifyIntIterator(sorter.getIterator(279), 279, 300); }
@Test public void testSortingOnlyByPrefix() throws Exception { final UnsafeExternalSorter sorter = newSorter(); insertNumber(sorter, 5); insertNumber(sorter, 1); insertNumber(sorter, 3); sorter.spill(); insertNumber(sorter, 4); sorter.spill(); insertNumber(sorter, 2); UnsafeSorterIterator iter = sorter.getSortedIterator(); for (int i = 1; i <= 5; i++) { iter.loadNext(); assertEquals(i, iter.getKeyPrefix()); assertEquals(4, iter.getRecordLength()); assertEquals(i, Platform.getInt(iter.getBaseObject(), iter.getBaseOffset())); } sorter.cleanupResources(); assertSpillFilesWereCleanedUp(); }
@Test public void testSortingOnlyByPrefix() throws Exception { final UnsafeExternalSorter sorter = newSorter(); insertNumber(sorter, 5); insertNumber(sorter, 1); insertNumber(sorter, 3); sorter.spill(); insertNumber(sorter, 4); sorter.spill(); insertNumber(sorter, 2); UnsafeSorterIterator iter = sorter.getSortedIterator(); for (int i = 1; i <= 5; i++) { iter.loadNext(); assertEquals(i, iter.getKeyPrefix()); assertEquals(4, iter.getRecordLength()); assertEquals(i, Platform.getInt(iter.getBaseObject(), iter.getBaseOffset())); } sorter.cleanupResources(); assertSpillFilesWereCleanedUp(); }
@Test public void testSortingOnlyByPrefix() throws Exception { final UnsafeExternalSorter sorter = newSorter(); insertNumber(sorter, 5); insertNumber(sorter, 1); insertNumber(sorter, 3); sorter.spill(); insertNumber(sorter, 4); sorter.spill(); insertNumber(sorter, 2); UnsafeSorterIterator iter = sorter.getSortedIterator(); for (int i = 1; i <= 5; i++) { iter.loadNext(); assertEquals(i, iter.getKeyPrefix()); assertEquals(4, iter.getRecordLength()); assertEquals(i, Platform.getInt(iter.getBaseObject(), iter.getBaseOffset())); } sorter.cleanupResources(); assertSpillFilesWereCleanedUp(); }