private static void spillIterator(UnsafeSorterIterator inMemIterator, UnsafeSorterSpillWriter spillWriter) throws IOException { while (inMemIterator.hasNext()) { inMemIterator.loadNext(); final Object baseObject = inMemIterator.getBaseObject(); final long baseOffset = inMemIterator.getBaseOffset(); final int recordLength = inMemIterator.getRecordLength(); spillWriter.write(baseObject, baseOffset, recordLength, inMemIterator.getKeyPrefix()); } spillWriter.close(); }
@Override public long getKeyPrefix() { return upstream.getKeyPrefix(); } }
public SortedIterator clone() { SortedIterator iter = new SortedIterator(numRecords, offset); iter.position = position; iter.baseObject = baseObject; iter.baseOffset = baseOffset; iter.keyPrefix = keyPrefix; iter.recordLength = recordLength; iter.currentPageNumber = currentPageNumber; return iter; }
@Test public void testFillingPage() throws Exception { final UnsafeExternalSorter sorter = newSorter(); byte[] record = new byte[16]; while (sorter.getNumberOfAllocatedPages() < 2) { sorter.insertRecord(record, Platform.BYTE_ARRAY_OFFSET, record.length, 0, false); } sorter.cleanupResources(); assertSpillFilesWereCleanedUp(); }
/** * Frees this sorter's in-memory data structures and cleans up its spill files. */ public void cleanupResources() { synchronized (this) { deleteSpillFiles(); freeMemory(); if (inMemSorter != null) { inMemSorter.free(); inMemSorter = null; } } }
private UnsafeExternalSorter newSorter() throws IOException { return UnsafeExternalSorter.create( taskMemoryManager, blockManager, serializerManager, taskContext, () -> recordComparator, prefixComparator, /* initialSize */ 1024, pageSizeBytes, spillThreshold, shouldUseRadixSort()); }
@Override public Object getBaseObject() { return upstream.getBaseObject(); }
private static void insertRecord( UnsafeExternalSorter sorter, int[] record, long prefix) throws IOException { sorter.insertRecord(record, Platform.INT_ARRAY_OFFSET, record.length * 4, prefix, false); }
private void updatePeakMemoryUsed() { long mem = getMemoryUsage(); if (mem > peakMemoryUsedBytes) { peakMemoryUsedBytes = mem; } }
/** * Return the peak memory used so far, in bytes. */ public long getPeakMemoryUsedBytes() { updatePeakMemoryUsed(); return peakMemoryUsedBytes; }
/** * @return the total amount of time spent sorting data (in-memory only). */ public long getSortTimeNanos() { UnsafeInMemorySorter sorter = inMemSorter; if (sorter != null) { return sorter.getSortTimeNanos(); } return totalSortTimeNanos; }
private static void spillIterator(UnsafeSorterIterator inMemIterator, UnsafeSorterSpillWriter spillWriter) throws IOException { while (inMemIterator.hasNext()) { inMemIterator.loadNext(); final Object baseObject = inMemIterator.getBaseObject(); final long baseOffset = inMemIterator.getBaseOffset(); final int recordLength = inMemIterator.getRecordLength(); spillWriter.write(baseObject, baseOffset, recordLength, inMemIterator.getKeyPrefix()); } spillWriter.close(); }
@Test public void testFillingPage() throws Exception { final UnsafeExternalSorter sorter = newSorter(); byte[] record = new byte[16]; while (sorter.getNumberOfAllocatedPages() < 2) { sorter.insertRecord(record, Platform.BYTE_ARRAY_OFFSET, record.length, 0, false); } sorter.cleanupResources(); assertSpillFilesWereCleanedUp(); }
/** * Frees this sorter's in-memory data structures and cleans up its spill files. */ public void cleanupResources() { synchronized (this) { deleteSpillFiles(); freeMemory(); if (inMemSorter != null) { inMemSorter.free(); inMemSorter = null; } } }
private UnsafeExternalSorter newSorter() throws IOException { return UnsafeExternalSorter.create( taskMemoryManager, blockManager, serializerManager, taskContext, () -> recordComparator, prefixComparator, /* initialSize */ 1024, pageSizeBytes, spillThreshold, shouldUseRadixSort()); }
@Override public long getKeyPrefix() { return upstream.getKeyPrefix(); } }
public SortedIterator clone() { SortedIterator iter = new SortedIterator(numRecords, offset); iter.position = position; iter.baseObject = baseObject; iter.baseOffset = baseOffset; iter.keyPrefix = keyPrefix; iter.recordLength = recordLength; iter.currentPageNumber = currentPageNumber; return iter; }
private void updatePeakMemoryUsed() { long mem = getMemoryUsage(); if (mem > peakMemoryUsedBytes) { peakMemoryUsedBytes = mem; } }
/** * Frees this sorter's in-memory data structures and cleans up its spill files. */ public void cleanupResources() { synchronized (this) { deleteSpillFiles(); freeMemory(); if (inMemSorter != null) { inMemSorter.free(); inMemSorter = null; } } }
private UnsafeExternalSorter newSorter() throws IOException { return UnsafeExternalSorter.create( taskMemoryManager, blockManager, serializerManager, taskContext, recordComparator, prefixComparator, /* initialSize */ 1024, pageSizeBytes, UnsafeExternalSorter.DEFAULT_NUM_ELEMENTS_FOR_SPILL_THRESHOLD, shouldUseRadixSort()); }