@Test public void testFillingPage() throws Exception { final UnsafeExternalSorter sorter = newSorter(); byte[] record = new byte[16]; while (sorter.getNumberOfAllocatedPages() < 2) { sorter.insertRecord(record, Platform.BYTE_ARRAY_OFFSET, record.length, 0, false); } sorter.cleanupResources(); assertSpillFilesWereCleanedUp(); }
@Test public void testFillingPage() throws Exception { final UnsafeExternalSorter sorter = newSorter(); byte[] record = new byte[16]; while (sorter.getNumberOfAllocatedPages() < 2) { sorter.insertRecord(record, Platform.BYTE_ARRAY_OFFSET, record.length, 0, false); } sorter.cleanupResources(); assertSpillFilesWereCleanedUp(); }
@Test public void testFillingPage() throws Exception { final UnsafeExternalSorter sorter = newSorter(); byte[] record = new byte[16]; while (sorter.getNumberOfAllocatedPages() < 2) { sorter.insertRecord(record, Platform.BYTE_ARRAY_OFFSET, record.length, 0, false); } sorter.cleanupResources(); assertSpillFilesWereCleanedUp(); }
@Test public void testSortingEmptyArrays() throws Exception { final UnsafeExternalSorter sorter = newSorter(); sorter.insertRecord(null, 0, 0, 0, false); sorter.insertRecord(null, 0, 0, 0, false); sorter.spill(); sorter.insertRecord(null, 0, 0, 0, false); sorter.spill(); sorter.insertRecord(null, 0, 0, 0, false); sorter.insertRecord(null, 0, 0, 0, false); UnsafeSorterIterator iter = sorter.getSortedIterator(); for (int i = 1; i <= 5; i++) { iter.loadNext(); assertEquals(0, iter.getKeyPrefix()); assertEquals(0, iter.getRecordLength()); } sorter.cleanupResources(); assertSpillFilesWereCleanedUp(); }
@Test public void testSortingEmptyArrays() throws Exception { final UnsafeExternalSorter sorter = newSorter(); sorter.insertRecord(null, 0, 0, 0, false); sorter.insertRecord(null, 0, 0, 0, false); sorter.spill(); sorter.insertRecord(null, 0, 0, 0, false); sorter.spill(); sorter.insertRecord(null, 0, 0, 0, false); sorter.insertRecord(null, 0, 0, 0, false); UnsafeSorterIterator iter = sorter.getSortedIterator(); for (int i = 1; i <= 5; i++) { iter.loadNext(); assertEquals(0, iter.getKeyPrefix()); assertEquals(0, iter.getRecordLength()); } sorter.cleanupResources(); assertSpillFilesWereCleanedUp(); }
@Test public void testSortingEmptyArrays() throws Exception { final UnsafeExternalSorter sorter = newSorter(); sorter.insertRecord(null, 0, 0, 0, false); sorter.insertRecord(null, 0, 0, 0, false); sorter.spill(); sorter.insertRecord(null, 0, 0, 0, false); sorter.spill(); sorter.insertRecord(null, 0, 0, 0, false); sorter.insertRecord(null, 0, 0, 0, false); UnsafeSorterIterator iter = sorter.getSortedIterator(); for (int i = 1; i <= 5; i++) { iter.loadNext(); assertEquals(0, iter.getKeyPrefix()); assertEquals(0, iter.getRecordLength()); } sorter.cleanupResources(); assertSpillFilesWereCleanedUp(); }
@Test public void spillingOccursInResponseToMemoryPressure() throws Exception { final UnsafeExternalSorter sorter = newSorter(); // This should be enough records to completely fill up a data page: final int numRecords = (int) (pageSizeBytes / (4 + 4)); for (int i = 0; i < numRecords; i++) { insertNumber(sorter, numRecords - i); } assertEquals(1, sorter.getNumberOfAllocatedPages()); memoryManager.markExecutionAsOutOfMemoryOnce(); // The insertion of this record should trigger a spill: insertNumber(sorter, 0); // Ensure that spill files were created assertThat(tempDir.listFiles().length, greaterThanOrEqualTo(1)); // Read back the sorted data: UnsafeSorterIterator iter = sorter.getSortedIterator(); int i = 0; while (iter.hasNext()) { iter.loadNext(); assertEquals(i, iter.getKeyPrefix()); assertEquals(4, iter.getRecordLength()); assertEquals(i, Platform.getInt(iter.getBaseObject(), iter.getBaseOffset())); i++; } assertEquals(numRecords + 1, i); sorter.cleanupResources(); assertSpillFilesWereCleanedUp(); }
@Test public void spillingOccursInResponseToMemoryPressure() throws Exception { final UnsafeExternalSorter sorter = newSorter(); // This should be enough records to completely fill up a data page: final int numRecords = (int) (pageSizeBytes / (4 + 4)); for (int i = 0; i < numRecords; i++) { insertNumber(sorter, numRecords - i); } assertEquals(1, sorter.getNumberOfAllocatedPages()); memoryManager.markExecutionAsOutOfMemoryOnce(); // The insertion of this record should trigger a spill: insertNumber(sorter, 0); // Ensure that spill files were created assertThat(tempDir.listFiles().length, greaterThanOrEqualTo(1)); // Read back the sorted data: UnsafeSorterIterator iter = sorter.getSortedIterator(); int i = 0; while (iter.hasNext()) { iter.loadNext(); assertEquals(i, iter.getKeyPrefix()); assertEquals(4, iter.getRecordLength()); assertEquals(i, Platform.getInt(iter.getBaseObject(), iter.getBaseOffset())); i++; } assertEquals(numRecords + 1, i); sorter.cleanupResources(); assertSpillFilesWereCleanedUp(); }
@Test public void spillingOccursInResponseToMemoryPressure() throws Exception { final UnsafeExternalSorter sorter = newSorter(); // This should be enough records to completely fill up a data page: final int numRecords = (int) (pageSizeBytes / (4 + 4)); for (int i = 0; i < numRecords; i++) { insertNumber(sorter, numRecords - i); } assertEquals(1, sorter.getNumberOfAllocatedPages()); memoryManager.markExecutionAsOutOfMemoryOnce(); // The insertion of this record should trigger a spill: insertNumber(sorter, 0); // Ensure that spill files were created assertThat(tempDir.listFiles().length, greaterThanOrEqualTo(1)); // Read back the sorted data: UnsafeSorterIterator iter = sorter.getSortedIterator(); int i = 0; while (iter.hasNext()) { iter.loadNext(); assertEquals(i, iter.getKeyPrefix()); assertEquals(4, iter.getRecordLength()); assertEquals(i, Platform.getInt(iter.getBaseObject(), iter.getBaseOffset())); i++; } assertEquals(numRecords + 1, i); sorter.cleanupResources(); assertSpillFilesWereCleanedUp(); }
@Test public void forcedSpillingWithNotReadIterator() throws Exception { final UnsafeExternalSorter sorter = newSorter(); long[] record = new long[100]; int recordSize = record.length * 8; int n = (int) pageSizeBytes / recordSize * 3; for (int i = 0; i < n; i++) { record[0] = (long) i; sorter.insertRecord(record, Platform.LONG_ARRAY_OFFSET, recordSize, 0, false); } assertTrue(sorter.getNumberOfAllocatedPages() >= 2); UnsafeExternalSorter.SpillableIterator iter = (UnsafeExternalSorter.SpillableIterator) sorter.getSortedIterator(); assertTrue(iter.spill() > 0); assertEquals(0, iter.spill()); for (int i = 0; i < n; i++) { iter.hasNext(); iter.loadNext(); assertEquals(i, Platform.getLong(iter.getBaseObject(), iter.getBaseOffset())); } sorter.cleanupResources(); assertSpillFilesWereCleanedUp(); }
@Test public void forcedSpillingWithNotReadIterator() throws Exception { final UnsafeExternalSorter sorter = newSorter(); long[] record = new long[100]; int recordSize = record.length * 8; int n = (int) pageSizeBytes / recordSize * 3; for (int i = 0; i < n; i++) { record[0] = (long) i; sorter.insertRecord(record, Platform.LONG_ARRAY_OFFSET, recordSize, 0, false); } assertTrue(sorter.getNumberOfAllocatedPages() >= 2); UnsafeExternalSorter.SpillableIterator iter = (UnsafeExternalSorter.SpillableIterator) sorter.getSortedIterator(); assertTrue(iter.spill() > 0); assertEquals(0, iter.spill()); for (int i = 0; i < n; i++) { iter.hasNext(); iter.loadNext(); assertEquals(i, Platform.getLong(iter.getBaseObject(), iter.getBaseOffset())); } sorter.cleanupResources(); assertSpillFilesWereCleanedUp(); }
@Test public void forcedSpillingWithNotReadIterator() throws Exception { final UnsafeExternalSorter sorter = newSorter(); long[] record = new long[100]; int recordSize = record.length * 8; int n = (int) pageSizeBytes / recordSize * 3; for (int i = 0; i < n; i++) { record[0] = (long) i; sorter.insertRecord(record, Platform.LONG_ARRAY_OFFSET, recordSize, 0, false); } assertTrue(sorter.getNumberOfAllocatedPages() >= 2); UnsafeExternalSorter.SpillableIterator iter = (UnsafeExternalSorter.SpillableIterator) sorter.getSortedIterator(); assertTrue(iter.spill() > 0); assertEquals(0, iter.spill()); for (int i = 0; i < n; i++) { iter.hasNext(); iter.loadNext(); assertEquals(i, Platform.getLong(iter.getBaseObject(), iter.getBaseOffset())); } sorter.cleanupResources(); assertSpillFilesWereCleanedUp(); }
@Test public void testSortingOnlyByPrefix() throws Exception { final UnsafeExternalSorter sorter = newSorter(); insertNumber(sorter, 5); insertNumber(sorter, 1); insertNumber(sorter, 3); sorter.spill(); insertNumber(sorter, 4); sorter.spill(); insertNumber(sorter, 2); UnsafeSorterIterator iter = sorter.getSortedIterator(); for (int i = 1; i <= 5; i++) { iter.loadNext(); assertEquals(i, iter.getKeyPrefix()); assertEquals(4, iter.getRecordLength()); assertEquals(i, Platform.getInt(iter.getBaseObject(), iter.getBaseOffset())); } sorter.cleanupResources(); assertSpillFilesWereCleanedUp(); }
@Test public void testSortingOnlyByPrefix() throws Exception { final UnsafeExternalSorter sorter = newSorter(); insertNumber(sorter, 5); insertNumber(sorter, 1); insertNumber(sorter, 3); sorter.spill(); insertNumber(sorter, 4); sorter.spill(); insertNumber(sorter, 2); UnsafeSorterIterator iter = sorter.getSortedIterator(); for (int i = 1; i <= 5; i++) { iter.loadNext(); assertEquals(i, iter.getKeyPrefix()); assertEquals(4, iter.getRecordLength()); assertEquals(i, Platform.getInt(iter.getBaseObject(), iter.getBaseOffset())); } sorter.cleanupResources(); assertSpillFilesWereCleanedUp(); }
@Test public void testSortingOnlyByPrefix() throws Exception { final UnsafeExternalSorter sorter = newSorter(); insertNumber(sorter, 5); insertNumber(sorter, 1); insertNumber(sorter, 3); sorter.spill(); insertNumber(sorter, 4); sorter.spill(); insertNumber(sorter, 2); UnsafeSorterIterator iter = sorter.getSortedIterator(); for (int i = 1; i <= 5; i++) { iter.loadNext(); assertEquals(i, iter.getKeyPrefix()); assertEquals(4, iter.getRecordLength()); assertEquals(i, Platform.getInt(iter.getBaseObject(), iter.getBaseOffset())); } sorter.cleanupResources(); assertSpillFilesWereCleanedUp(); }
@Test public void testDiskSpilledBytes() throws Exception { final UnsafeExternalSorter sorter = newSorter(); long[] record = new long[100]; int recordSize = record.length * 8; int n = (int) pageSizeBytes / recordSize * 3; for (int i = 0; i < n; i++) { record[0] = (long) i; sorter.insertRecord(record, Platform.LONG_ARRAY_OFFSET, recordSize, 0, false); } // We will have at-least 2 memory pages allocated because of rounding happening due to // integer division of pageSizeBytes and recordSize. assertTrue(sorter.getNumberOfAllocatedPages() >= 2); assertTrue(taskContext.taskMetrics().diskBytesSpilled() == 0); UnsafeExternalSorter.SpillableIterator iter = (UnsafeExternalSorter.SpillableIterator) sorter.getSortedIterator(); assertTrue(iter.spill() > 0); assertTrue(taskContext.taskMetrics().diskBytesSpilled() > 0); assertEquals(0, iter.spill()); // Even if we did not spill second time, the disk spilled bytes should still be non-zero assertTrue(taskContext.taskMetrics().diskBytesSpilled() > 0); sorter.cleanupResources(); assertSpillFilesWereCleanedUp(); }
@Test public void testDiskSpilledBytes() throws Exception { final UnsafeExternalSorter sorter = newSorter(); long[] record = new long[100]; int recordSize = record.length * 8; int n = (int) pageSizeBytes / recordSize * 3; for (int i = 0; i < n; i++) { record[0] = (long) i; sorter.insertRecord(record, Platform.LONG_ARRAY_OFFSET, recordSize, 0, false); } // We will have at-least 2 memory pages allocated because of rounding happening due to // integer division of pageSizeBytes and recordSize. assertTrue(sorter.getNumberOfAllocatedPages() >= 2); assertTrue(taskContext.taskMetrics().diskBytesSpilled() == 0); UnsafeExternalSorter.SpillableIterator iter = (UnsafeExternalSorter.SpillableIterator) sorter.getSortedIterator(); assertTrue(iter.spill() > 0); assertTrue(taskContext.taskMetrics().diskBytesSpilled() > 0); assertEquals(0, iter.spill()); // Even if we did not spill second time, the disk spilled bytes should still be non-zero assertTrue(taskContext.taskMetrics().diskBytesSpilled() > 0); sorter.cleanupResources(); assertSpillFilesWereCleanedUp(); }