@Override public void close() { cleanupResources(); } }
public boolean hasNext() throws IOException { if (currentRows.isEmpty()) { processNext(); } return !currentRows.isEmpty(); }
/** * Returns a sorted iterator. It is the caller's responsibility to call `cleanupResources()` * after consuming this iterator. */ public KVSorterIterator sortedIterator() throws IOException { try { final UnsafeSorterIterator underlying = sorter.getSortedIterator(); if (!underlying.hasNext()) { // Since we won't ever call next() on an empty iterator, we need to clean up resources // here in order to prevent memory leaks. cleanupResources(); } return new KVSorterIterator(underlying); } catch (IOException e) { cleanupResources(); throw e; } }
/** * Inserts a key-value record into the sorter. If the sorter no longer has enough memory to hold * the record, the sorter sorts the existing records in-memory, writes them out as partially * sorted runs, and then reallocates memory to hold the new record. */ public void insertKV(UnsafeRow key, UnsafeRow value) throws IOException { final UnsafeExternalRowSorter.PrefixComputer.Prefix prefix = prefixComputer.computePrefix(key); sorter.insertKVRecord( key.getBaseObject(), key.getBaseOffset(), key.getSizeInBytes(), value.getBaseObject(), value.getBaseOffset(), value.getSizeInBytes(), prefix.value, prefix.isNull); }
/** * Return the aggregation buffer for the current group. For efficiency, all calls to this method * return the same object. If additional memory could not be allocated, then this method will * signal an error by returning null. */ public UnsafeRow getAggregationBuffer(InternalRow groupingKey) { final UnsafeRow unsafeGroupingKeyRow = this.groupingKeyProjection.apply(groupingKey); return getAggregationBufferFromUnsafeRow(unsafeGroupingKeyRow); }
/** * Sorts the map's records in place, spill them to disk, and returns an [[UnsafeKVExternalSorter]] * * Note that the map will be reset for inserting new records, and the returned sorter can NOT be * used to insert records. */ public UnsafeKVExternalSorter destructAndCreateExternalSorter() throws IOException { return new UnsafeKVExternalSorter( groupingKeySchema, aggregationBufferSchema, SparkEnv.get().blockManager(), SparkEnv.get().serializerManager(), map.getPageSizeBytes(), SparkEnv.get().conf().getLong("spark.shuffle.spill.numElementsForceSpillThreshold", UnsafeExternalSorter.DEFAULT_NUM_ELEMENTS_FOR_SPILL_THRESHOLD), map); } }
/** * Returns a sorted iterator. It is the caller's responsibility to call `cleanupResources()` * after consuming this iterator. */ public KVSorterIterator sortedIterator() throws IOException { try { final UnsafeSorterIterator underlying = sorter.getSortedIterator(); if (!underlying.hasNext()) { // Since we won't ever call next() on an empty iterator, we need to clean up resources // here in order to prevent memory leaks. cleanupResources(); } return new KVSorterIterator(underlying); } catch (IOException e) { cleanupResources(); throw e; } }
/** * Inserts a key-value record into the sorter. If the sorter no longer has enough memory to hold * the record, the sorter sorts the existing records in-memory, writes them out as partially * sorted runs, and then reallocates memory to hold the new record. */ public void insertKV(UnsafeRow key, UnsafeRow value) throws IOException { final UnsafeExternalRowSorter.PrefixComputer.Prefix prefix = prefixComputer.computePrefix(key); sorter.insertKVRecord( key.getBaseObject(), key.getBaseOffset(), key.getSizeInBytes(), value.getBaseObject(), value.getBaseOffset(), value.getSizeInBytes(), prefix.value, prefix.isNull); }
/** * Return the aggregation buffer for the current group. For efficiency, all calls to this method * return the same object. If additional memory could not be allocated, then this method will * signal an error by returning null. */ public UnsafeRow getAggregationBuffer(InternalRow groupingKey) { final UnsafeRow unsafeGroupingKeyRow = this.groupingKeyProjection.apply(groupingKey); return getAggregationBufferFromUnsafeRow(unsafeGroupingKeyRow); }
@Override public void close() { cleanupResources(); } }
/** * Sorts the map's records in place, spill them to disk, and returns an [[UnsafeKVExternalSorter]] * * Note that the map will be reset for inserting new records, and the returned sorter can NOT be * used to insert records. */ public UnsafeKVExternalSorter destructAndCreateExternalSorter() throws IOException { return new UnsafeKVExternalSorter( groupingKeySchema, aggregationBufferSchema, SparkEnv.get().blockManager(), SparkEnv.get().serializerManager(), map.getPageSizeBytes(), (int) SparkEnv.get().conf().get( package$.MODULE$.SHUFFLE_SPILL_NUM_ELEMENTS_FORCE_SPILL_THRESHOLD()), map); } }
public boolean hasNext() throws IOException { if (currentRows.isEmpty()) { processNext(); } return !currentRows.isEmpty(); }
/** * Returns a sorted iterator. It is the caller's responsibility to call `cleanupResources()` * after consuming this iterator. */ public KVSorterIterator sortedIterator() throws IOException { try { final UnsafeSorterIterator underlying = sorter.getSortedIterator(); if (!underlying.hasNext()) { // Since we won't ever call next() on an empty iterator, we need to clean up resources // here in order to prevent memory leaks. cleanupResources(); } return new KVSorterIterator(underlying); } catch (IOException e) { cleanupResources(); throw e; } }
/** * Inserts a key-value record into the sorter. If the sorter no longer has enough memory to hold * the record, the sorter sorts the existing records in-memory, writes them out as partially * sorted runs, and then reallocates memory to hold the new record. */ public void insertKV(UnsafeRow key, UnsafeRow value) throws IOException { final UnsafeExternalRowSorter.PrefixComputer.Prefix prefix = prefixComputer.computePrefix(key); sorter.insertKVRecord( key.getBaseObject(), key.getBaseOffset(), key.getSizeInBytes(), value.getBaseObject(), value.getBaseOffset(), value.getSizeInBytes(), prefix.value, prefix.isNull); }
/** * Return the aggregation buffer for the current group. For efficiency, all calls to this method * return the same object. If additional memory could not be allocated, then this method will * signal an error by returning null. */ public UnsafeRow getAggregationBuffer(InternalRow groupingKey) { final UnsafeRow unsafeGroupingKeyRow = this.groupingKeyProjection.apply(groupingKey); return getAggregationBufferFromUnsafeRow(unsafeGroupingKeyRow); }
@Override public void close() { cleanupResources(); } }
public boolean hasNext() throws IOException { if (currentRows.isEmpty()) { processNext(); } return !currentRows.isEmpty(); }
public UnsafeRow getAggregationBufferFromUnsafeRow(UnsafeRow key) { return getAggregationBufferFromUnsafeRow(key, key.hashCode()); }
public UnsafeRow getAggregationBufferFromUnsafeRow(UnsafeRow key) { return getAggregationBufferFromUnsafeRow(key, key.hashCode()); }
public UnsafeRow getAggregationBufferFromUnsafeRow(UnsafeRow key) { return getAggregationBufferFromUnsafeRow(key, key.hashCode()); }