@Override public final int getValueLength(Slice slice, int offset) { return Integer.reverseBytes(slice.getInt(offset)); }
@Override public void decodeValueInto(BlockBuilder builder, Slice slice, int offset, int length) { int intBits = slice.getInt(offset); // the file format uses big endian type.writeLong(builder, Integer.reverseBytes(intBits)); } }
public static List<Model> deserializeModels(byte[] bytes) { Slice slice = Slices.wrappedBuffer(bytes); int numModels = slice.getInt(0); int offset = SIZE_OF_INT + SIZE_OF_INT * numModels; ImmutableList.Builder<Model> models = ImmutableList.builder(); for (int i = 0; i < numModels; i++) { int length = slice.getInt(SIZE_OF_INT * (i + 1)); models.add(deserialize(slice.getBytes(offset, length))); offset += length; } return models.build(); }
public boolean nextRecord() { if (pageRecordCount <= readCount) { return false; } if (readCount > 0) { // advance position excepting the first record int lastRecordSize = pageSlice.getInt(position); position += lastRecordSize; } readCount++; pageSlice.getBytes(position + 4, nullBitSet, 0, nullBitSet.length); return true; }
private static Slice decompressLZO(Slice input, int uncompressedSize) { LzoDecompressor lzoDecompressor = new LzoDecompressor(); long totalDecompressedCount = 0; // over allocate buffer which makes decompression easier byte[] output = new byte[uncompressedSize + SIZE_OF_LONG]; int outputOffset = 0; int inputOffset = 0; int cumulativeUncompressedBlockLength = 0; while (totalDecompressedCount < uncompressedSize) { if (totalDecompressedCount == cumulativeUncompressedBlockLength) { cumulativeUncompressedBlockLength += Integer.reverseBytes(input.getInt(inputOffset)); inputOffset += SIZE_OF_INT; } int compressedChunkLength = Integer.reverseBytes(input.getInt(inputOffset)); inputOffset += SIZE_OF_INT; int decompressionSize = decompress(lzoDecompressor, input, inputOffset, compressedChunkLength, output, outputOffset); totalDecompressedCount += decompressionSize; outputOffset += decompressionSize; inputOffset += compressedChunkLength; } checkArgument(outputOffset == uncompressedSize); return wrappedBuffer(output, 0, uncompressedSize); }
public static Model deserialize(Slice slice) { int version = slice.getInt(VERSION_OFFSET); checkArgument(version == CURRENT_FORMAT_VERSION, format("Unsupported version: %d", version)); byte[] modelHashBytes = slice.getBytes(HASH_OFFSET, 32); HashCode expectedHash = HashCode.fromBytes(modelHashBytes); HashCode actualHash = Hashing.sha256().hashBytes(slice.getBytes(ALGORITHM_OFFSET, slice.length() - ALGORITHM_OFFSET)); checkArgument(actualHash.equals(expectedHash), "model hash does not match data"); int id = slice.getInt(ALGORITHM_OFFSET); Class<? extends Model> algorithm = MODEL_SERIALIZATION_IDS.inverse().get(id); requireNonNull(algorithm, format("Unsupported algorith %d", id)); int hyperparameterLength = slice.getInt(HYPERPARAMETER_LENGTH_OFFSET); byte[] hyperparameterBytes = slice.getBytes(HYPERPARAMETERS_OFFSET, hyperparameterLength); int dataLengthOffset = HYPERPARAMETERS_OFFSET + hyperparameterLength; long dataLength = slice.getLong(dataLengthOffset); int dataOffset = dataLengthOffset + SIZE_OF_LONG; byte[] data = slice.getBytes(dataOffset, (int) dataLength); try { Method deserialize = algorithm.getMethod("deserialize", byte[].class); return (Model) deserialize.invoke(null, new Object[] {data}); } catch (NoSuchMethodException | InvocationTargetException | IllegalAccessException e) { throw new RuntimeException(e); } }
@Override public int getInt(int position, int offset) { checkReadablePosition(position); return getRawSlice(position).getInt(getPositionOffset(position) + offset); }
private static int[] toInt8Array(byte[] bytes) { Slice slice = Slices.wrappedBuffer(bytes); int[] ints = new int[8]; for (int i = 0; i < ints.length; i++) { ints[i] = slice.getInt(i * Integer.SIZE / Byte.SIZE); } return ints; }
@Override public int getInt(int position, int offset) { checkReadablePosition(position); return getRawSlice().getInt(valueOffset(position) + offset); }
@Description("decode the 32-bit big-endian binary in IEEE 754 single-precision floating-point format") @ScalarFunction("from_ieee754_32") @SqlType(StandardTypes.REAL) public static long fromIEEE754Binary32(@SqlType(StandardTypes.VARBINARY) Slice slice) { checkCondition(slice.length() == Integer.BYTES, INVALID_FUNCTION_ARGUMENT, "Input floating-point value must be exactly 4 bytes long"); return Integer.reverseBytes(slice.getInt(0)); }
@ScalarOperator(CAST) @SqlType(StandardTypes.HYPER_LOG_LOG) public static Slice castToHyperLogLog(@SqlType(SetDigestType.NAME) Slice slice) { checkArgument(slice.getByte(0) == 1, "Legacy version of SetDigest cannot cast to HyperLogLog"); int hllLength = slice.getInt(SIZE_OF_BYTE); return Slices.wrappedBuffer(slice.getBytes(SIZE_OF_BYTE + SIZE_OF_INT, hllLength)); } }
public String getString(int columnIndex) { if (isNull(columnIndex)) { return null; } int index = pageSlice.getInt(getOffset(columnIndex)); return page.getStringReference(index); }
public Value getJson(int columnIndex) { if (isNull(columnIndex)) { return null; } int index = pageSlice.getInt(getOffset(columnIndex)); return page.getValueReference(index); }
@Description("decode bigint value from a 32-bit 2's complement big endian varbinary") @ScalarFunction("from_big_endian_32") @SqlType(StandardTypes.INTEGER) public static long fromBigEndian32(@SqlType(StandardTypes.VARBINARY) Slice slice) { if (slice.length() != Integer.BYTES) { throw new PrestoException(INVALID_FUNCTION_ARGUMENT, "expected 4-byte input, but got instead: " + slice.length()); } return Integer.reverseBytes(slice.getInt(0)); }
@Override public Block decodeColumn(ColumnData columnData) { int size = columnData.rowCount(); BlockBuilder builder = type.createBlockBuilder(null, size); Slice slice = columnData.getSlice(); for (int i = 0; i < size; i++) { int length = columnData.getLength(i); if (length != 0) { checkState(length == SIZE_OF_FLOAT, "Float should be 4 bytes"); int intBits = slice.getInt(columnData.getOffset(i)); // the file format uses big endian type.writeLong(builder, Integer.reverseBytes(intBits)); } else { builder.appendNull(); } } return builder.build(); }
public Timestamp getTimestamp(int columnIndex) { if (isNull(columnIndex)) { return null; } int offset = getOffset(columnIndex); long sec = pageSlice.getLong(offset); int nsec = pageSlice.getInt(offset + 8); return Timestamp.ofEpochSecond(sec, nsec); }
private static void verifySlice() { Slice slice = Slices.wrappedBuffer(new byte[5]); slice.setByte(4, 0xDE); slice.setByte(3, 0xAD); slice.setByte(2, 0xBE); slice.setByte(1, 0xEF); if (slice.getInt(1) != 0xDEADBEEF) { failRequirement("Slice library produced an unexpected result"); } }
private static void writeValues(Slice[] expectedValues, BlockBuilder blockBuilder) { for (Slice expectedValue : expectedValues) { if (expectedValue == null) { blockBuilder.appendNull(); } else { blockBuilder.writeInt(expectedValue.getInt(0)).closeEntry(); } } }
public static int getRecordCount(Page page) { Buffer pageBuffer = page.buffer(); Slice pageSlice = Slices.wrappedBuffer(pageBuffer.array(), pageBuffer.offset(), pageBuffer.limit()); return pageSlice.getInt(0); // see page format }
public void setPage(Page page) { this.page.buffer().release(); this.page = SENTINEL; Buffer pageBuffer = page.buffer(); Slice pageSlice = Slices.wrappedBuffer(pageBuffer.array(), pageBuffer.offset(), pageBuffer.limit()); pageRecordCount = pageSlice.getInt(0); // see page format readCount = 0; position = PageFormat.PAGE_HEADER_SIZE; this.page = page; this.pageSlice = pageSlice; }