/** * Inserts a key-value record into the sorter. If the sorter no longer has enough memory to hold * the record, the sorter sorts the existing records in-memory, writes them out as partially * sorted runs, and then reallocates memory to hold the new record. */ public void insertKV(UnsafeRow key, UnsafeRow value) throws IOException { final UnsafeExternalRowSorter.PrefixComputer.Prefix prefix = prefixComputer.computePrefix(key); sorter.insertKVRecord( key.getBaseObject(), key.getBaseOffset(), key.getSizeInBytes(), value.getBaseObject(), value.getBaseOffset(), value.getSizeInBytes(), prefix.value, prefix.isNull); }
@Test public void testBinaryComparatorWhenOnlyTheLastColumnDiffers() throws Exception { int numFields = 4; UnsafeRow row1 = new UnsafeRow(numFields); byte[] data1 = new byte[100]; row1.pointTo(data1, computeSizeInBytes(numFields * 8)); row1.setInt(0, 11); row1.setDouble(1, 3.14); row1.setInt(2, -1); row1.setLong(3, 0); UnsafeRow row2 = new UnsafeRow(numFields); byte[] data2 = new byte[100]; row2.pointTo(data2, computeSizeInBytes(numFields * 8)); row2.setInt(0, 11); row2.setDouble(1, 3.14); row2.setInt(2, -1); row2.setLong(3, 1); insertRow(row1); insertRow(row2); assert(compare(0, 1) < 0); } }
@Test public void testBinaryComparatorForNullColumns() throws Exception { int numFields = 3; UnsafeRow row1 = new UnsafeRow(numFields); byte[] data1 = new byte[100]; row1.pointTo(data1, computeSizeInBytes(numFields * 8)); for (int i = 0; i < numFields; i++) { row1.setNullAt(i); } UnsafeRow row2 = new UnsafeRow(numFields); byte[] data2 = new byte[100]; row2.pointTo(data2, computeSizeInBytes(numFields * 8)); for (int i = 0; i < numFields - 1; i++) { row2.setNullAt(i); } row2.setDouble(numFields - 1, 3.14); insertRow(row1); insertRow(row2); assert(compare(0, 0) == 0); assert(compare(0, 1) > 0); }
public UnsafeRow getAggregationBufferFromUnsafeRow(UnsafeRow key, int hash) { // Probe our map using the serialized key final BytesToBytesMap.Location loc = map.lookup( key.getBaseObject(), key.getBaseOffset(), key.getSizeInBytes(), hash); if (!loc.isDefined()) { // This is the first time that we've seen this grouping key, so we'll insert a copy of the // empty aggregation buffer into the map: boolean putSucceeded = loc.append( key.getBaseObject(), key.getBaseOffset(), key.getSizeInBytes(), emptyAggregationBuffer, Platform.BYTE_ARRAY_OFFSET, emptyAggregationBuffer.length ); if (!putSucceeded) { return null; } } // Reset the pointer to point to the value that we just stored or looked up: currentAggregationBuffer.pointTo( loc.getValueBase(), loc.getValueOffset(), loc.getValueLength() ); return currentAggregationBuffer; }
@Override public Object get(int ordinal, DataType dataType) { if (isNullAt(ordinal) || dataType instanceof NullType) { return null; } else if (dataType instanceof BooleanType) { return getBoolean(ordinal); } else if (dataType instanceof ByteType) { return getByte(ordinal); } else if (dataType instanceof ShortType) { return getShort(ordinal); } else if (dataType instanceof IntegerType) { return getInt(ordinal); } else if (dataType instanceof LongType) { return getLong(ordinal); } else if (dataType instanceof FloatType) { return getFloat(ordinal); } else if (dataType instanceof DoubleType) { return getDouble(ordinal); } else if (dataType instanceof DecimalType) { DecimalType dt = (DecimalType) dataType; return getDecimal(ordinal, dt.precision(), dt.scale()); } else if (dataType instanceof DateType) { return getInt(ordinal); } else if (dataType instanceof TimestampType) { return getLong(ordinal); } else if (dataType instanceof BinaryType) { return getBinary(ordinal); } else if (dataType instanceof StringType) { return getUTF8String(ordinal); } else if (dataType instanceof CalendarIntervalType) {
@Test public void testBinaryComparatorWhenSubtractionIsDivisibleByMaxIntValue() throws Exception { int numFields = 1; UnsafeRow row1 = new UnsafeRow(numFields); byte[] data1 = new byte[100]; row1.pointTo(data1, computeSizeInBytes(numFields * 8)); row1.setLong(0, 11); UnsafeRow row2 = new UnsafeRow(numFields); byte[] data2 = new byte[100]; row2.pointTo(data2, computeSizeInBytes(numFields * 8)); row2.setLong(0, 11L + Integer.MAX_VALUE); insertRow(row1); insertRow(row2); assert(compare(0, 1) < 0); }
@Test public void testBinaryComparatorForArrayColumn() throws Exception { int numFields = 1; UnsafeRow row1 = new UnsafeRow(numFields); byte[] data1 = new byte[100]; UnsafeArrayData arrayData1 = UnsafeArrayData.fromPrimitiveArray(new int[]{11, 42, -1}); row1.pointTo(data1, computeSizeInBytes(numFields * 8 + arrayData1.getSizeInBytes())); row1.setLong(0, (relativeOffset(numFields) << 32) | (long) arrayData1.getSizeInBytes()); Platform.copyMemory(arrayData1.getBaseObject(), arrayData1.getBaseOffset(), data1, row1.getBaseOffset() + relativeOffset(numFields), arrayData1.getSizeInBytes()); UnsafeRow row2 = new UnsafeRow(numFields); byte[] data2 = new byte[100]; UnsafeArrayData arrayData2 = UnsafeArrayData.fromPrimitiveArray(new int[]{22}); row2.pointTo(data2, computeSizeInBytes(numFields * 8 + arrayData2.getSizeInBytes())); row2.setLong(0, (relativeOffset(numFields) << 32) | (long) arrayData2.getSizeInBytes()); Platform.copyMemory(arrayData2.getBaseObject(), arrayData2.getBaseOffset(), data2, row2.getBaseOffset() + relativeOffset(numFields), arrayData2.getSizeInBytes()); insertRow(row1); insertRow(row2); assert(compare(0, 0) == 0); assert(compare(0, 1) > 0); }
/** * Returns the value row by two steps: * 1) looking up the key row with the same id (skipped if the key row is cached) * 2) retrieve the value row by reusing the metadata from step 1) * In most times, 1) is skipped because `getKeyRow(id)` is often called before `getValueRow(id)`. */ @Override public UnsafeRow getValueFromKey(int rowId) { if (keyRowId != rowId) { getKeyRow(rowId); } assert(rowId >= 0); long offset = keyRow.getBaseOffset(); int klen = keyRow.getSizeInBytes(); int vlen = Platform.getInt(base, offset - 8) - klen - 4; valueRow.pointTo(base, offset + klen, vlen + 4); return valueRow; }
@Override public UnsafeRow getStruct(int ordinal, int numFields) { if (isNullAt(ordinal)) { return null; } else { final long offsetAndSize = getLong(ordinal); final int offset = (int) (offsetAndSize >> 32); final int size = (int) offsetAndSize; final UnsafeRow row = new UnsafeRow(numFields); row.pointTo(baseObject, baseOffset + offset, size); return row; } }
assertIndexIsValid(ordinal); if (precision <= Decimal.MAX_LONG_DIGITS()) { setNullAt(ordinal); } else { setLong(ordinal, value.toUnscaledLong()); long cursor = getLong(ordinal) >>> 32; assert cursor > 0 : "invalid cursor " + cursor; setNullAt(ordinal); Platform.putLong(baseObject, getFieldOffset(ordinal), cursor << 32); } else { setLong(ordinal, (cursor << 32) | ((long) bytes.length));
@Test public void testBinaryComparatorForSingleColumnRow() throws Exception { int numFields = 1; UnsafeRow row1 = new UnsafeRow(numFields); byte[] data1 = new byte[100]; row1.pointTo(data1, computeSizeInBytes(numFields * 8)); row1.setInt(0, 11); UnsafeRow row2 = new UnsafeRow(numFields); byte[] data2 = new byte[100]; row2.pointTo(data2, computeSizeInBytes(numFields * 8)); row2.setInt(0, 42); insertRow(row1); insertRow(row2); assert(compare(0, 0) == 0); assert(compare(0, 1) < 0); }
@Test public void testBinaryComparatorForMultipleColumnRow() throws Exception { int numFields = 5; UnsafeRow row1 = new UnsafeRow(numFields); byte[] data1 = new byte[100]; row1.pointTo(data1, computeSizeInBytes(numFields * 8)); for (int i = 0; i < numFields; i++) { row1.setDouble(i, i * 3.14); } UnsafeRow row2 = new UnsafeRow(numFields); byte[] data2 = new byte[100]; row2.pointTo(data2, computeSizeInBytes(numFields * 8)); for (int i = 0; i < numFields; i++) { row2.setDouble(i, 198.7 / (i + 1)); } insertRow(row1); insertRow(row2); assert(compare(0, 0) == 0); assert(compare(0, 1) < 0); }
long pageSizeBytes) { this.aggregationBufferSchema = aggregationBufferSchema; this.currentAggregationBuffer = new UnsafeRow(aggregationBufferSchema.length()); this.groupingKeyProjection = UnsafeProjection.create(groupingKeySchema); this.groupingKeySchema = groupingKeySchema; this.emptyAggregationBuffer = valueProjection.apply(emptyAggregationBuffer).getBytes();
final int sizeInBytes = ((UnsafeRow) value).getSizeInBytes(); ((UnsafeRow) value).writeToMemory(holder.buffer, holder.cursor); holder.cursor += sizeInBytes; result.setTotalSize(holder.totalSize()); return result;
@Override public int compare( Object baseObj1, long baseOff1, int baseLen1, Object baseObj2, long baseOff2, int baseLen2) { // Note that since ordering doesn't need the total length of the record, we just pass 0 // into the row. row1.pointTo(baseObj1, baseOff1 + 4, 0); row2.pointTo(baseObj2, baseOff2 + 4, 0); return ordering.compare(row1, row2); } }
private boolean checkKey(UnsafeRow row, long k1, String k2) { return (row.getLong(0) == k1) && (row.getUTF8String(1).equals(UTF8String.fromString(k2))); }
@Override public void writeExternal(ObjectOutput out) throws IOException { byte[] bytes = getBytes(); out.writeInt(bytes.length); out.writeInt(this.numFields); out.write(bytes); }
@Override public Decimal getDecimal(int ordinal, int precision, int scale) { if (isNullAt(ordinal)) { return null; } if (precision <= Decimal.MAX_LONG_DIGITS()) { return Decimal.createUnsafe(getLong(ordinal), precision, scale); } else { byte[] bytes = getBinary(ordinal); BigInteger bigInteger = new BigInteger(bytes); BigDecimal javaDecimal = new BigDecimal(bigInteger, scale); return Decimal.apply(javaDecimal, precision, scale); } }