private UnsafeExternalSorter newSorter() throws IOException { return UnsafeExternalSorter.create( taskMemoryManager, blockManager, serializerManager, taskContext, () -> recordComparator, prefixComparator, /* initialSize */ 1024, pageSizeBytes, spillThreshold, shouldUseRadixSort()); }
sorter = UnsafeExternalSorter.create( taskMemoryManager, blockManager,
private UnsafeExternalSorter newSorter() throws IOException { return UnsafeExternalSorter.create( taskMemoryManager, blockManager, serializerManager, taskContext, () -> recordComparator, prefixComparator, /* initialSize */ 1024, pageSizeBytes, spillThreshold, shouldUseRadixSort()); }
sorter = UnsafeExternalSorter.create( taskMemoryManager, blockManager,
private UnsafeExternalSorter newSorter() throws IOException { return UnsafeExternalSorter.create( taskMemoryManager, blockManager, serializerManager, taskContext, recordComparator, prefixComparator, /* initialSize */ 1024, pageSizeBytes, UnsafeExternalSorter.DEFAULT_NUM_ELEMENTS_FOR_SPILL_THRESHOLD, shouldUseRadixSort()); }
sorter = UnsafeExternalSorter.create( taskMemoryManager, blockManager,
@Test public void forcedSpillingWithoutComparator() throws Exception { final UnsafeExternalSorter sorter = UnsafeExternalSorter.create( taskMemoryManager, blockManager,
sorter = UnsafeExternalSorter.create( taskMemoryManager,
@Test public void forcedSpillingWithoutComparator() throws Exception { final UnsafeExternalSorter sorter = UnsafeExternalSorter.create( taskMemoryManager, blockManager,
sorter = UnsafeExternalSorter.create( taskMemoryManager, blockManager,
@Test public void forcedSpillingWithoutComparator() throws Exception { final UnsafeExternalSorter sorter = UnsafeExternalSorter.create( taskMemoryManager, blockManager,
final long pageSizeBytes = 256; final long numRecordsPerPage = pageSizeBytes / recordLengthBytes; final UnsafeExternalSorter sorter = UnsafeExternalSorter.create( taskMemoryManager, blockManager,
final long pageSizeBytes = 256; final long numRecordsPerPage = pageSizeBytes / recordLengthBytes; final UnsafeExternalSorter sorter = UnsafeExternalSorter.create( taskMemoryManager, blockManager,
final long pageSizeBytes = 256; final long numRecordsPerPage = pageSizeBytes / recordLengthBytes; final UnsafeExternalSorter sorter = UnsafeExternalSorter.create( taskMemoryManager, blockManager,
public UnsafeExternalRowSorter( StructType schema, Comparator<InternalRow> ordering, PrefixComparator prefixComparator, PrefixComputer prefixComputer, long pageSizeBytes) throws IOException { this.schema = schema; this.prefixComputer = prefixComputer; //final SparkEnv sparkEnv = SparkEnv.get(); final TaskContext taskContext = TaskContext.get(); sorter = UnsafeExternalSorter.create( taskContext.taskMemoryManager(), taskContext, new RowComparator(ordering, schema.size()), prefixComparator, /* initialSize */ 4096, pageSizeBytes ); }
private UnsafeExternalRowSorter( StructType schema, Supplier<RecordComparator> recordComparatorSupplier, PrefixComparator prefixComparator, PrefixComputer prefixComputer, long pageSizeBytes, boolean canUseRadixSort) throws IOException { this.schema = schema; this.prefixComputer = prefixComputer; final SparkEnv sparkEnv = SparkEnv.get(); final TaskContext taskContext = TaskContext.get(); sorter = UnsafeExternalSorter.create( taskContext.taskMemoryManager(), sparkEnv.blockManager(), sparkEnv.serializerManager(), taskContext, recordComparatorSupplier.get(), prefixComparator, sparkEnv.conf().getInt("spark.shuffle.sort.initialBufferSize", DEFAULT_INITIAL_SORT_BUFFER_SIZE), pageSizeBytes, SparkEnv.get().conf().getLong("spark.shuffle.spill.numElementsForceSpillThreshold", UnsafeExternalSorter.DEFAULT_NUM_ELEMENTS_FOR_SPILL_THRESHOLD), canUseRadixSort ); }
private UnsafeExternalRowSorter( StructType schema, Supplier<RecordComparator> recordComparatorSupplier, PrefixComparator prefixComparator, PrefixComputer prefixComputer, long pageSizeBytes, boolean canUseRadixSort) throws IOException { this.schema = schema; this.prefixComputer = prefixComputer; final SparkEnv sparkEnv = SparkEnv.get(); final TaskContext taskContext = TaskContext.get(); sorter = UnsafeExternalSorter.create( taskContext.taskMemoryManager(), sparkEnv.blockManager(), sparkEnv.serializerManager(), taskContext, recordComparatorSupplier, prefixComparator, sparkEnv.conf().getInt("spark.shuffle.sort.initialBufferSize", DEFAULT_INITIAL_SORT_BUFFER_SIZE), pageSizeBytes, (int) SparkEnv.get().conf().get( package$.MODULE$.SHUFFLE_SPILL_NUM_ELEMENTS_FORCE_SPILL_THRESHOLD()), canUseRadixSort ); }
private UnsafeExternalRowSorter( StructType schema, Supplier<RecordComparator> recordComparatorSupplier, PrefixComparator prefixComparator, PrefixComputer prefixComputer, long pageSizeBytes, boolean canUseRadixSort) throws IOException { this.schema = schema; this.prefixComputer = prefixComputer; final SparkEnv sparkEnv = SparkEnv.get(); final TaskContext taskContext = TaskContext.get(); sorter = UnsafeExternalSorter.create( taskContext.taskMemoryManager(), sparkEnv.blockManager(), sparkEnv.serializerManager(), taskContext, recordComparatorSupplier, prefixComparator, sparkEnv.conf().getInt("spark.shuffle.sort.initialBufferSize", DEFAULT_INITIAL_SORT_BUFFER_SIZE), pageSizeBytes, (int) SparkEnv.get().conf().get( package$.MODULE$.SHUFFLE_SPILL_NUM_ELEMENTS_FORCE_SPILL_THRESHOLD()), canUseRadixSort ); }