@Override public void decodeValueInto(BlockBuilder builder, Slice slice, int offset, int length) { long longBits = slice.getLong(offset); // the file format uses big endian double value = Double.longBitsToDouble(Long.reverseBytes(longBits)); type.writeDouble(builder, value); } }
public long getLong(int columnIndex) { return pageSlice.getLong(getOffset(columnIndex)); }
@Override public long getLong(int position, int offset) { checkReadablePosition(position); return getRawSlice(position).getLong(getPositionOffset(position) + offset); }
@Override public long getLong(int position, int offset) { checkReadablePosition(position); return getRawSlice().getLong(valueOffset(position) + offset); }
@Description("decode the 64-bit big-endian binary in IEEE 754 double-precision floating-point format") @ScalarFunction("from_ieee754_64") @SqlType(StandardTypes.DOUBLE) public static double fromIEEE754Binary64(@SqlType(StandardTypes.VARBINARY) Slice slice) { checkCondition(slice.length() == Double.BYTES, INVALID_FUNCTION_ARGUMENT, "Input floating-point value must be exactly 8 bytes long"); return Double.longBitsToDouble(Long.reverseBytes(slice.getLong(0))); }
public static Model deserialize(Slice slice) { int version = slice.getInt(VERSION_OFFSET); checkArgument(version == CURRENT_FORMAT_VERSION, format("Unsupported version: %d", version)); byte[] modelHashBytes = slice.getBytes(HASH_OFFSET, 32); HashCode expectedHash = HashCode.fromBytes(modelHashBytes); HashCode actualHash = Hashing.sha256().hashBytes(slice.getBytes(ALGORITHM_OFFSET, slice.length() - ALGORITHM_OFFSET)); checkArgument(actualHash.equals(expectedHash), "model hash does not match data"); int id = slice.getInt(ALGORITHM_OFFSET); Class<? extends Model> algorithm = MODEL_SERIALIZATION_IDS.inverse().get(id); requireNonNull(algorithm, format("Unsupported algorith %d", id)); int hyperparameterLength = slice.getInt(HYPERPARAMETER_LENGTH_OFFSET); byte[] hyperparameterBytes = slice.getBytes(HYPERPARAMETERS_OFFSET, hyperparameterLength); int dataLengthOffset = HYPERPARAMETERS_OFFSET + hyperparameterLength; long dataLength = slice.getLong(dataLengthOffset); int dataOffset = dataLengthOffset + SIZE_OF_LONG; byte[] data = slice.getBytes(dataOffset, (int) dataLength); try { Method deserialize = algorithm.getMethod("deserialize", byte[].class); return (Model) deserialize.invoke(null, new Object[] {data}); } catch (NoSuchMethodException | InvocationTargetException | IllegalAccessException e) { throw new RuntimeException(e); } }
@Override public Block decodeColumn(ColumnData columnData) { int size = columnData.rowCount(); BlockBuilder builder = type.createBlockBuilder(null, size); Slice slice = columnData.getSlice(); for (int i = 0; i < size; i++) { int length = columnData.getLength(i); if (length != 0) { checkState(length == SIZE_OF_DOUBLE, "Double should be 8 bytes"); long longBits = slice.getLong(columnData.getOffset(i)); // the file format uses big endian type.writeDouble(builder, Double.longBitsToDouble(Long.reverseBytes(longBits))); } else { builder.appendNull(); } } return builder.build(); }
@Description("decode bigint value from a 64-bit 2's complement big endian varbinary") @ScalarFunction("from_big_endian_64") @SqlType(StandardTypes.BIGINT) public static long fromBigEndian64(@SqlType(StandardTypes.VARBINARY) Slice slice) { if (slice.length() != Long.BYTES) { throw new PrestoException(INVALID_FUNCTION_ARGUMENT, "expected 8-byte input, but got instead: " + slice.length()); } return Long.reverseBytes(slice.getLong(0)); }
public Timestamp getTimestamp(int columnIndex) { if (isNull(columnIndex)) { return null; } int offset = getOffset(columnIndex); long sec = pageSlice.getLong(offset); int nsec = pageSlice.getInt(offset + 8); return Timestamp.ofEpochSecond(sec, nsec); }
private static void writeValues(Slice[] expectedValues, BlockBuilder blockBuilder) { for (Slice expectedValue : expectedValues) { if (expectedValue == null) { blockBuilder.appendNull(); } else { blockBuilder.writeLong(expectedValue.getLong(0)).closeEntry(); } } }
private static List<Long> getSyncPositionsSimple(RcFileReader recordReader, File file) throws IOException { List<Long> syncPositions = new ArrayList<>(); Slice sync = recordReader.getSync(); long syncFirst = sync.getLong(0); long syncSecond = sync.getLong(8); long syncPosition = 0; try (RcFileDataSource dataSource = new FileRcFileDataSource(file)) { while (syncPosition >= 0) { syncPosition = findFirstSyncPosition(dataSource, syncPosition, file.length() - syncPosition, syncFirst, syncSecond); if (syncPosition > 0) { assertEquals(findFirstSyncPosition(dataSource, syncPosition, 1, syncFirst, syncSecond), syncPosition); assertEquals(findFirstSyncPosition(dataSource, syncPosition, 2, syncFirst, syncSecond), syncPosition); assertEquals(findFirstSyncPosition(dataSource, syncPosition, 10, syncFirst, syncSecond), syncPosition); assertEquals(findFirstSyncPosition(dataSource, syncPosition - 1, 1, syncFirst, syncSecond), -1); assertEquals(findFirstSyncPosition(dataSource, syncPosition - 2, 2, syncFirst, syncSecond), -1); assertEquals(findFirstSyncPosition(dataSource, syncPosition + 1, 1, syncFirst, syncSecond), -1); syncPositions.add(syncPosition); syncPosition++; } } } return syncPositions; }
assertEquals(block.getLong(position, offset), expectedSliceValue.getLong(offset));
@Override public long readLong() { long v = slice.getLong(position); position += SIZE_OF_LONG; return v; }
@Override public long readLong() { long v = slice.getLong(position); position += SIZE_OF_LONG; return v; }
@Test(invocationCount = 100) public void test64ReturnsMsb() throws Exception { byte[] data = randomBytes(ThreadLocalRandom.current().nextInt(200)); long expected = Murmur3Hash128.hash(Slices.wrappedBuffer(data)).getLong(0); long actual = Murmur3Hash128.hash64(Slices.wrappedBuffer(data)); assertEquals(actual, expected); }
@Test(invocationCount = 100) public void testMoreThan16Bytes64() throws Exception { byte[] data = randomBytes(131); long expected = Murmur3Hash128.hash(Slices.wrappedBuffer(data)).getLong(0); long actual = Murmur3Hash128.hash64(Slices.wrappedBuffer(data)); assertEquals(actual, expected); }
@Test(invocationCount = 100) public void testOffsetAndLength64() throws Exception { byte[] data = randomBytes(131); int offset = 13; int length = 55; long expected = Murmur3Hash128.hash(Slices.wrappedBuffer(data), offset, length).getLong(0); long actual = Murmur3Hash128.hash64(Slices.wrappedBuffer(data), offset, length); assertEquals(actual, expected); }
@Test(invocationCount = 100) public void testMoreThan16Bytes64() throws Exception { byte[] data = randomBytes(131); long expected = Murmur3Hash128.hash(Slices.wrappedBuffer(data)).getLong(0); long actual = Murmur3Hash128.hash64(Slices.wrappedBuffer(data)); assertEquals(actual, expected); }
@Test public void testTail64() throws Exception { for (int i = 0; i < 16; i++) { byte[] data = randomBytes(50 + i); long expected = Murmur3Hash128.hash(Slices.wrappedBuffer(data)).getLong(0); long actual = Murmur3Hash128.hash64(Slices.wrappedBuffer(data)); assertEquals(actual, expected); } }