/** * Write a record to the sorter. */ public void insertRecord( Object recordBase, long recordOffset, int length, long prefix, boolean prefixIsNull) throws IOException { assert(inMemSorter != null); if (inMemSorter.numRecords() >= numElementsForSpillThreshold) { logger.info("Spilling data because number of spilledRecords crossed the threshold " + numElementsForSpillThreshold); spill(); } growPointerArrayIfNecessary(); int uaoSize = UnsafeAlignedOffset.getUaoSize(); // Need 4 bytes to store the record length. final int required = length + uaoSize; acquireNewPageIfNecessary(required); final Object base = currentPage.getBaseObject(); final long recordAddress = taskMemoryManager.encodePageNumberAndOffset(currentPage, pageCursor); UnsafeAlignedOffset.putSize(base, pageCursor, length); pageCursor += uaoSize; Platform.copyMemory(recordBase, recordOffset, base, pageCursor, length); pageCursor += length; inMemSorter.insertRecord(recordAddress, prefix, prefixIsNull); }
/** * Write a record to the sorter. */ public void insertRecord( Object recordBase, long recordOffset, int length, long prefix, boolean prefixIsNull) throws IOException { assert(inMemSorter != null); if (inMemSorter.numRecords() >= numElementsForSpillThreshold) { logger.info("Spilling data because number of spilledRecords crossed the threshold " + numElementsForSpillThreshold); spill(); } growPointerArrayIfNecessary(); int uaoSize = UnsafeAlignedOffset.getUaoSize(); // Need 4 or 8 bytes to store the record length. final int required = length + uaoSize; acquireNewPageIfNecessary(required); final Object base = currentPage.getBaseObject(); final long recordAddress = taskMemoryManager.encodePageNumberAndOffset(currentPage, pageCursor); UnsafeAlignedOffset.putSize(base, pageCursor, length); pageCursor += uaoSize; Platform.copyMemory(recordBase, recordOffset, base, pageCursor, length); pageCursor += length; inMemSorter.insertRecord(recordAddress, prefix, prefixIsNull); }
/** * Write a record to the sorter. */ public void insertRecord( Object recordBase, long recordOffset, int length, long prefix, boolean prefixIsNull) throws IOException { assert(inMemSorter != null); if (inMemSorter.numRecords() >= numElementsForSpillThreshold) { logger.info("Spilling data because number of spilledRecords crossed the threshold " + numElementsForSpillThreshold); spill(); } growPointerArrayIfNecessary(); int uaoSize = UnsafeAlignedOffset.getUaoSize(); // Need 4 or 8 bytes to store the record length. final int required = length + uaoSize; acquireNewPageIfNecessary(required); final Object base = currentPage.getBaseObject(); final long recordAddress = taskMemoryManager.encodePageNumberAndOffset(currentPage, pageCursor); UnsafeAlignedOffset.putSize(base, pageCursor, length); pageCursor += uaoSize; Platform.copyMemory(recordBase, recordOffset, base, pageCursor, length); pageCursor += length; inMemSorter.insertRecord(recordAddress, prefix, prefixIsNull); }
/** * Write a key-value record to the sorter. The key and value will be put together in-memory, * using the following format: * * record length (4 bytes), key length (4 bytes), key data, value data * * record length = key length + value length + 4 */ public void insertKVRecord(Object keyBase, long keyOffset, int keyLen, Object valueBase, long valueOffset, int valueLen, long prefix, boolean prefixIsNull) throws IOException { growPointerArrayIfNecessary(); int uaoSize = UnsafeAlignedOffset.getUaoSize(); final int required = keyLen + valueLen + (2 * uaoSize); acquireNewPageIfNecessary(required); final Object base = currentPage.getBaseObject(); final long recordAddress = taskMemoryManager.encodePageNumberAndOffset(currentPage, pageCursor); UnsafeAlignedOffset.putSize(base, pageCursor, keyLen + valueLen + uaoSize); pageCursor += uaoSize; UnsafeAlignedOffset.putSize(base, pageCursor, keyLen); pageCursor += uaoSize; Platform.copyMemory(keyBase, keyOffset, base, pageCursor, keyLen); pageCursor += keyLen; Platform.copyMemory(valueBase, valueOffset, base, pageCursor, valueLen); pageCursor += valueLen; assert(inMemSorter != null); inMemSorter.insertRecord(recordAddress, prefix, prefixIsNull); }
/** * Write a key-value record to the sorter. The key and value will be put together in-memory, * using the following format: * * record length (4 bytes), key length (4 bytes), key data, value data * * record length = key length + value length + 4 */ public void insertKVRecord(Object keyBase, long keyOffset, int keyLen, Object valueBase, long valueOffset, int valueLen, long prefix, boolean prefixIsNull) throws IOException { growPointerArrayIfNecessary(); int uaoSize = UnsafeAlignedOffset.getUaoSize(); final int required = keyLen + valueLen + (2 * uaoSize); acquireNewPageIfNecessary(required); final Object base = currentPage.getBaseObject(); final long recordAddress = taskMemoryManager.encodePageNumberAndOffset(currentPage, pageCursor); UnsafeAlignedOffset.putSize(base, pageCursor, keyLen + valueLen + uaoSize); pageCursor += uaoSize; UnsafeAlignedOffset.putSize(base, pageCursor, keyLen); pageCursor += uaoSize; Platform.copyMemory(keyBase, keyOffset, base, pageCursor, keyLen); pageCursor += keyLen; Platform.copyMemory(valueBase, valueOffset, base, pageCursor, valueLen); pageCursor += valueLen; assert(inMemSorter != null); inMemSorter.insertRecord(recordAddress, prefix, prefixIsNull); }
/** * Write a key-value record to the sorter. The key and value will be put together in-memory, * using the following format: * * record length (4 bytes), key length (4 bytes), key data, value data * * record length = key length + value length + 4 */ public void insertKVRecord(Object keyBase, long keyOffset, int keyLen, Object valueBase, long valueOffset, int valueLen, long prefix, boolean prefixIsNull) throws IOException { growPointerArrayIfNecessary(); int uaoSize = UnsafeAlignedOffset.getUaoSize(); final int required = keyLen + valueLen + (2 * uaoSize); acquireNewPageIfNecessary(required); final Object base = currentPage.getBaseObject(); final long recordAddress = taskMemoryManager.encodePageNumberAndOffset(currentPage, pageCursor); UnsafeAlignedOffset.putSize(base, pageCursor, keyLen + valueLen + uaoSize); pageCursor += uaoSize; UnsafeAlignedOffset.putSize(base, pageCursor, keyLen); pageCursor += uaoSize; Platform.copyMemory(keyBase, keyOffset, base, pageCursor, keyLen); pageCursor += keyLen; Platform.copyMemory(valueBase, valueOffset, base, pageCursor, valueLen); pageCursor += valueLen; assert(inMemSorter != null); inMemSorter.insertRecord(recordAddress, prefix, prefixIsNull); }
final String str = getStringFromDataPage(baseObject, position + 4, recordLength); final int partitionId = hashPartitioner.getPartition(str); sorter.insertRecord(address, partitionId, false); position += 4 + recordLength;
final String str = getStringFromDataPage(baseObject, position + 4, recordLength); final int partitionId = hashPartitioner.getPartition(str); sorter.insertRecord(address, partitionId, false); position += 4 + recordLength;
final String str = getStringFromDataPage(baseObject, position + 4, recordLength); final int partitionId = hashPartitioner.getPartition(str); sorter.insertRecord(address, partitionId, false); position += 4 + recordLength;
prefixComputer.computePrefix(row); inMemSorter.insertRecord(address, prefix.value, prefix.isNull);
prefixComputer.computePrefix(row); inMemSorter.insertRecord(address, prefix.value, prefix.isNull);
prefixComputer.computePrefix(row); inMemSorter.insertRecord(address, prefix.value, prefix.isNull);
/** * Write a record to the sorter. */ public void insertRecord(Object recordBase, long recordOffset, int length, long prefix) throws IOException { growPointerArrayIfNecessary(); // Need 4 bytes to store the record length. final int required = length + 4; acquireNewPageIfNecessary(required); final Object base = currentPage.getBaseObject(); final long recordAddress = taskMemoryManager.encodePageNumberAndOffset(currentPage, pageCursor); Platform.putInt(base, pageCursor, length); pageCursor += 4; Platform.copyMemory(recordBase, recordOffset, base, pageCursor, length); pageCursor += length; assert (inMemSorter != null); inMemSorter.insertRecord(recordAddress, prefix); }
/** * Write a key-value record to the sorter. The key and value will be put together in-memory, * using the following format: * * record length (4 bytes), key length (4 bytes), key data, value data * * record length = key length + value length + 4 */ public void insertKVRecord(Object keyBase, long keyOffset, int keyLen, Object valueBase, long valueOffset, int valueLen, long prefix) throws IOException { growPointerArrayIfNecessary(); final int required = keyLen + valueLen + 4 + 4; acquireNewPageIfNecessary(required); final Object base = currentPage.getBaseObject(); final long recordAddress = taskMemoryManager.encodePageNumberAndOffset(currentPage, pageCursor); Platform.putInt(base, pageCursor, keyLen + valueLen + 4); pageCursor += 4; Platform.putInt(base, pageCursor, keyLen); pageCursor += 4; Platform.copyMemory(keyBase, keyOffset, base, pageCursor, keyLen); pageCursor += keyLen; Platform.copyMemory(valueBase, valueOffset, base, pageCursor, valueLen); pageCursor += valueLen; assert (inMemSorter != null); inMemSorter.insertRecord(recordAddress, prefix); }
final long prefix = prefixComputer.computePrefix(row); inMemSorter.insertRecord(address, prefix);
prefixComputer.computePrefix(row); inMemSorter.insertRecord(address, prefix.value, prefix.isNull);