@Override public Object getBaseObject() { return upstream.getBaseObject(); }
@Override public Object getBaseObject() { return upstream.getBaseObject(); }
@Override public Object getBaseObject() { return upstream.getBaseObject(); }
@Override public Object getBaseObject() { return spillReader.getBaseObject(); }
@Override public Object getBaseObject() { return current.getBaseObject(); }
@Override public Object getBaseObject() { return current.getBaseObject(); }
@Override public Object getBaseObject() { return spillReader.getBaseObject(); }
@Override public Object getBaseObject() { return current.getBaseObject(); }
@Override public Object getBaseObject() { return spillReader.getBaseObject(); }
UnsafeSorterSpillMerger( RecordComparator recordComparator, PrefixComparator prefixComparator, int numSpills) { Comparator<UnsafeSorterIterator> comparator = (left, right) -> { int prefixComparisonResult = prefixComparator.compare(left.getKeyPrefix(), right.getKeyPrefix()); if (prefixComparisonResult == 0) { return recordComparator.compare( left.getBaseObject(), left.getBaseOffset(), left.getRecordLength(), right.getBaseObject(), right.getBaseOffset(), right.getRecordLength()); } else { return prefixComparisonResult; } }; priorityQueue = new PriorityQueue<>(numSpills, comparator); }
UnsafeSorterSpillMerger( RecordComparator recordComparator, PrefixComparator prefixComparator, int numSpills) { Comparator<UnsafeSorterIterator> comparator = (left, right) -> { int prefixComparisonResult = prefixComparator.compare(left.getKeyPrefix(), right.getKeyPrefix()); if (prefixComparisonResult == 0) { return recordComparator.compare( left.getBaseObject(), left.getBaseOffset(), left.getRecordLength(), right.getBaseObject(), right.getBaseOffset(), right.getRecordLength()); } else { return prefixComparisonResult; } }; priorityQueue = new PriorityQueue<>(numSpills, comparator); }
UnsafeSorterSpillMerger( RecordComparator recordComparator, PrefixComparator prefixComparator, int numSpills) { Comparator<UnsafeSorterIterator> comparator = (left, right) -> { int prefixComparisonResult = prefixComparator.compare(left.getKeyPrefix(), right.getKeyPrefix()); if (prefixComparisonResult == 0) { return recordComparator.compare( left.getBaseObject(), left.getBaseOffset(), left.getRecordLength(), right.getBaseObject(), right.getBaseOffset(), right.getRecordLength()); } else { return prefixComparisonResult; } }; priorityQueue = new PriorityQueue<>(numSpills, comparator); }
private static void spillIterator(UnsafeSorterIterator inMemIterator, UnsafeSorterSpillWriter spillWriter) throws IOException { while (inMemIterator.hasNext()) { inMemIterator.loadNext(); final Object baseObject = inMemIterator.getBaseObject(); final long baseOffset = inMemIterator.getBaseOffset(); final int recordLength = inMemIterator.getRecordLength(); spillWriter.write(baseObject, baseOffset, recordLength, inMemIterator.getKeyPrefix()); } spillWriter.close(); }
private static void spillIterator(UnsafeSorterIterator inMemIterator, UnsafeSorterSpillWriter spillWriter) throws IOException { while (inMemIterator.hasNext()) { inMemIterator.loadNext(); final Object baseObject = inMemIterator.getBaseObject(); final long baseOffset = inMemIterator.getBaseOffset(); final int recordLength = inMemIterator.getRecordLength(); spillWriter.write(baseObject, baseOffset, recordLength, inMemIterator.getKeyPrefix()); } spillWriter.close(); }
private void verifyIntIterator(UnsafeSorterIterator iter, int start, int end) throws IOException { for (int i = start; i < end; i++) { assert (iter.hasNext()); iter.loadNext(); assert (Platform.getInt(iter.getBaseObject(), iter.getBaseOffset()) == i); } } }
private void verifyIntIterator(UnsafeSorterIterator iter, int start, int end) throws IOException { for (int i = start; i < end; i++) { assert (iter.hasNext()); iter.loadNext(); assert (Platform.getInt(iter.getBaseObject(), iter.getBaseOffset()) == i); } } }
@Test public void spillingOccursInResponseToMemoryPressure() throws Exception { final UnsafeExternalSorter sorter = newSorter(); // This should be enough records to completely fill up a data page: final int numRecords = (int) (pageSizeBytes / (4 + 4)); for (int i = 0; i < numRecords; i++) { insertNumber(sorter, numRecords - i); } assertEquals(1, sorter.getNumberOfAllocatedPages()); memoryManager.markExecutionAsOutOfMemoryOnce(); // The insertion of this record should trigger a spill: insertNumber(sorter, 0); // Ensure that spill files were created assertThat(tempDir.listFiles().length, greaterThanOrEqualTo(1)); // Read back the sorted data: UnsafeSorterIterator iter = sorter.getSortedIterator(); int i = 0; while (iter.hasNext()) { iter.loadNext(); assertEquals(i, iter.getKeyPrefix()); assertEquals(4, iter.getRecordLength()); assertEquals(i, Platform.getInt(iter.getBaseObject(), iter.getBaseOffset())); i++; } assertEquals(numRecords + 1, i); sorter.cleanupResources(); assertSpillFilesWereCleanedUp(); }
@Test public void testSortingOnlyByPrefix() throws Exception { final UnsafeExternalSorter sorter = newSorter(); insertNumber(sorter, 5); insertNumber(sorter, 1); insertNumber(sorter, 3); sorter.spill(); insertNumber(sorter, 4); sorter.spill(); insertNumber(sorter, 2); UnsafeSorterIterator iter = sorter.getSortedIterator(); for (int i = 1; i <= 5; i++) { iter.loadNext(); assertEquals(i, iter.getKeyPrefix()); assertEquals(4, iter.getRecordLength()); assertEquals(i, Platform.getInt(iter.getBaseObject(), iter.getBaseOffset())); } sorter.cleanupResources(); assertSpillFilesWereCleanedUp(); }
@Test public void testSortingOnlyByPrefix() throws Exception { final UnsafeExternalSorter sorter = newSorter(); insertNumber(sorter, 5); insertNumber(sorter, 1); insertNumber(sorter, 3); sorter.spill(); insertNumber(sorter, 4); sorter.spill(); insertNumber(sorter, 2); UnsafeSorterIterator iter = sorter.getSortedIterator(); for (int i = 1; i <= 5; i++) { iter.loadNext(); assertEquals(i, iter.getKeyPrefix()); assertEquals(4, iter.getRecordLength()); assertEquals(i, Platform.getInt(iter.getBaseObject(), iter.getBaseOffset())); } sorter.cleanupResources(); assertSpillFilesWereCleanedUp(); }
@Test public void testSortingOnlyByPrefix() throws Exception { final UnsafeExternalSorter sorter = newSorter(); insertNumber(sorter, 5); insertNumber(sorter, 1); insertNumber(sorter, 3); sorter.spill(); insertNumber(sorter, 4); sorter.spill(); insertNumber(sorter, 2); UnsafeSorterIterator iter = sorter.getSortedIterator(); for (int i = 1; i <= 5; i++) { iter.loadNext(); assertEquals(i, iter.getKeyPrefix()); assertEquals(4, iter.getRecordLength()); assertEquals(i, Platform.getInt(iter.getBaseObject(), iter.getBaseOffset())); } sorter.cleanupResources(); assertSpillFilesWereCleanedUp(); }