/** * Inserts a record to be sorted. * * @param recordPointer a pointer to the record, encoded by the task memory manager. Due to * certain pointer compression techniques used by the sorter, the sort can * only operate on pointers that point to locations in the first * {@link PackedRecordPointer#MAXIMUM_PAGE_SIZE_BYTES} bytes of a data page. * @param partitionId the partition id, which must be less than or equal to * {@link PackedRecordPointer#MAXIMUM_PARTITION_ID}. */ public void insertRecord(long recordPointer, int partitionId) { if (!hasSpaceForAnotherRecord()) { throw new IllegalStateException("There is no space for new record"); } array.set(pos, PackedRecordPointer.packPointer(recordPointer, partitionId)); pos++; }
/** * Inserts a record to be sorted. * * @param recordPointer a pointer to the record, encoded by the task memory manager. Due to * certain pointer compression techniques used by the sorter, the sort can * only operate on pointers that point to locations in the first * {@link PackedRecordPointer#MAXIMUM_PAGE_SIZE_BYTES} bytes of a data page. * @param partitionId the partition id, which must be less than or equal to * {@link PackedRecordPointer#MAXIMUM_PARTITION_ID}. */ public void insertRecord(long recordPointer, int partitionId) { if (!hasSpaceForAnotherRecord()) { throw new IllegalStateException("There is no space for new record"); } array.set(pos, PackedRecordPointer.packPointer(recordPointer, partitionId)); pos++; }
/** * Inserts a record to be sorted. * * @param recordPointer a pointer to the record, encoded by the task memory manager. Due to * certain pointer compression techniques used by the sorter, the sort can * only operate on pointers that point to locations in the first * {@link PackedRecordPointer#MAXIMUM_PAGE_SIZE_BYTES} bytes of a data page. * @param partitionId the partition id, which must be less than or equal to * {@link PackedRecordPointer#MAXIMUM_PARTITION_ID}. */ public void insertRecord(long recordPointer, int partitionId) { if (!hasSpaceForAnotherRecord()) { throw new IllegalStateException("There is no space for new record"); } array.set(pos, PackedRecordPointer.packPointer(recordPointer, partitionId)); pos++; }
/** * Checks whether there is enough space to insert an additional record in to the sort pointer * array and grows the array if additional space is required. If the required space cannot be * obtained, then the in-memory data will be spilled to disk. */ private void growPointerArrayIfNecessary() throws IOException { assert(inMemSorter != null); if (!inMemSorter.hasSpaceForAnotherRecord()) { long used = inMemSorter.getMemoryUsage(); LongArray array; try { // could trigger spilling array = allocateArray(used / 8 * 2); } catch (OutOfMemoryError e) { // should have trigger spilling if (!inMemSorter.hasSpaceForAnotherRecord()) { logger.error("Unable to grow the pointer array"); throw e; } return; } // check if spilling is triggered or not if (inMemSorter.hasSpaceForAnotherRecord()) { freeArray(array); } else { inMemSorter.expandPointerArray(array); } } }
if (!inMemSorter.hasSpaceForAnotherRecord()) { long used = inMemSorter.getMemoryUsage(); LongArray array; } catch (SparkOutOfMemoryError e) { if (!inMemSorter.hasSpaceForAnotherRecord()) { logger.error("Unable to grow the pointer array"); throw e; if (inMemSorter.hasSpaceForAnotherRecord()) { freeArray(array); } else {
if (!inMemSorter.hasSpaceForAnotherRecord()) { long used = inMemSorter.getMemoryUsage(); LongArray array; } catch (SparkOutOfMemoryError e) { if (!inMemSorter.hasSpaceForAnotherRecord()) { logger.error("Unable to grow the pointer array"); throw e; if (inMemSorter.hasSpaceForAnotherRecord()) { freeArray(array); } else {
@Test public void testSortingManyNumbers() throws Exception { ShuffleInMemorySorter sorter = new ShuffleInMemorySorter(consumer, 4, shouldUseRadixSort()); int[] numbersToSort = new int[128000]; Random random = new Random(16); for (int i = 0; i < numbersToSort.length; i++) { if (!sorter.hasSpaceForAnotherRecord()) { sorter.expandPointerArray(consumer.allocateArray(sorter.getMemoryUsage() / 8 * 2)); } numbersToSort[i] = random.nextInt(PackedRecordPointer.MAXIMUM_PARTITION_ID + 1); sorter.insertRecord(0, numbersToSort[i]); } Arrays.sort(numbersToSort); int[] sorterResult = new int[numbersToSort.length]; ShuffleInMemorySorter.ShuffleSorterIterator iter = sorter.getSortedIterator(); int j = 0; while (iter.hasNext()) { iter.loadNext(); sorterResult[j] = iter.packedRecordPointer.getPartitionId(); j += 1; } Assert.assertArrayEquals(numbersToSort, sorterResult); } }
@Test public void testSortingManyNumbers() throws Exception { ShuffleInMemorySorter sorter = new ShuffleInMemorySorter(consumer, 4, shouldUseRadixSort()); int[] numbersToSort = new int[128000]; Random random = new Random(16); for (int i = 0; i < numbersToSort.length; i++) { if (!sorter.hasSpaceForAnotherRecord()) { sorter.expandPointerArray(consumer.allocateArray(sorter.getMemoryUsage() / 8 * 2)); } numbersToSort[i] = random.nextInt(PackedRecordPointer.MAXIMUM_PARTITION_ID + 1); sorter.insertRecord(0, numbersToSort[i]); } Arrays.sort(numbersToSort); int[] sorterResult = new int[numbersToSort.length]; ShuffleInMemorySorter.ShuffleSorterIterator iter = sorter.getSortedIterator(); int j = 0; while (iter.hasNext()) { iter.loadNext(); sorterResult[j] = iter.packedRecordPointer.getPartitionId(); j += 1; } Assert.assertArrayEquals(numbersToSort, sorterResult); } }
@Test public void testSortingManyNumbers() throws Exception { ShuffleInMemorySorter sorter = new ShuffleInMemorySorter(consumer, 4, shouldUseRadixSort()); int[] numbersToSort = new int[128000]; Random random = new Random(16); for (int i = 0; i < numbersToSort.length; i++) { if (!sorter.hasSpaceForAnotherRecord()) { sorter.expandPointerArray(consumer.allocateArray(sorter.getMemoryUsage() / 8 * 2)); } numbersToSort[i] = random.nextInt(PackedRecordPointer.MAXIMUM_PARTITION_ID + 1); sorter.insertRecord(0, numbersToSort[i]); } Arrays.sort(numbersToSort); int[] sorterResult = new int[numbersToSort.length]; ShuffleInMemorySorter.ShuffleSorterIterator iter = sorter.getSortedIterator(); int j = 0; while (iter.hasNext()) { iter.loadNext(); sorterResult[j] = iter.packedRecordPointer.getPartitionId(); j += 1; } Assert.assertArrayEquals(numbersToSort, sorterResult); } }
if (!sorter.hasSpaceForAnotherRecord()) { sorter.expandPointerArray( consumer.allocateArray(sorter.getMemoryUsage() / 8 * 2));
if (!sorter.hasSpaceForAnotherRecord()) { sorter.expandPointerArray( consumer.allocateArray(sorter.getMemoryUsage() / 8 * 2));
if (!sorter.hasSpaceForAnotherRecord()) { sorter.expandPointerArray( consumer.allocateArray(sorter.getMemoryUsage() / 8 * 2));