@Override public void writeInt(int value) { ensureWritableBytes(SIZE_OF_INT); slice.setInt(bufferPosition, value); bufferPosition += SIZE_OF_INT; }
public void addRowGroup(int rowCount) { longSlice.setInt(0, rowCount); rowGroupHash.update(longBuffer, 0, Integer.BYTES); }
@Override public void writeInt(int value) { ensureWritableBytes(SIZE_OF_INT); slice.setInt(bufferPosition, value); bufferPosition += SIZE_OF_INT; }
@Override public void writeInt(int value) { ensureWritableBytes(SIZE_OF_INT); slice.setInt(bufferPosition, value); bufferPosition += SIZE_OF_INT; }
public void addStripe(int rowCount) { longSlice.setInt(0, rowCount); stripeHash.update(longBuffer, 0, Integer.BYTES); }
@Override public void writeInt(int value) { ensureWritableBytes(SIZE_OF_INT); slice.setInt(bufferPosition, value); bufferPosition += SIZE_OF_INT; }
private void writeString(int columnIndex, String value) { int index = stringReferences.size(); stringReferences.add(value); bufferSlice.setInt(getOffset(columnIndex), index); referenceSize += value.length() * 2 + 4; // assuming size of char = size of byte * 2 + length clearNull(columnIndex); }
public static byte[] serializeModels(Model... models) { List<byte[]> serializedModels = new ArrayList<>(); int size = SIZE_OF_INT + SIZE_OF_INT * models.length; for (Model model : models) { byte[] bytes = serialize(model).getBytes(); size += bytes.length; serializedModels.add(bytes); } Slice slice = Slices.allocate(size); slice.setInt(0, models.length); for (int i = 0; i < models.length; i++) { slice.setInt(SIZE_OF_INT * (i + 1), serializedModels.get(i).length); } int offset = SIZE_OF_INT + SIZE_OF_INT * models.length; for (byte[] bytes : serializedModels) { slice.setBytes(offset, bytes); offset += bytes.length; } return slice.getBytes(); }
private void writeJson(int columnIndex, Value value) { int index = valueReferences.size(); valueReferences.add(value.immutableValue()); bufferSlice.setInt(getOffset(columnIndex), index); referenceSize += 256; // TODO how to estimate size of the value? clearNull(columnIndex); }
@Description("encode value as a 32-bit 2's complement big endian varbinary") @ScalarFunction("to_big_endian_32") @SqlType(StandardTypes.VARBINARY) public static Slice toBigEndian32(@SqlType(StandardTypes.INTEGER) long value) { Slice slice = Slices.allocate(Integer.BYTES); slice.setInt(0, Integer.reverseBytes((int) value)); return slice; }
@Description("encode value as a big endian varbinary according to IEEE 754 single-precision floating-point format") @ScalarFunction("to_ieee754_32") @SqlType(StandardTypes.VARBINARY) public static Slice toIEEE754Binary32(@SqlType(StandardTypes.REAL) long value) { Slice slice = Slices.allocate(Float.BYTES); slice.setInt(0, Integer.reverseBytes((int) value)); return slice; }
public void addRecord() { // record row.write(this); // record header bufferSlice.setInt(position, nextVariableLengthDataOffset); // nextVariableLengthDataOffset means record size bufferSlice.setBytes(position + 4, nullBitSet); count++; this.position += nextVariableLengthDataOffset; this.nextVariableLengthDataOffset = fixedRecordSize; Arrays.fill(nullBitSet, (byte) -1); // flush if next record will not fit in this buffer if (buffer.capacity() < position + nextVariableLengthDataOffset + referenceSize) { flush(); } }
@Description("compute SpookyHashV2 32-bit hash") @ScalarFunction @SqlType(StandardTypes.VARBINARY) public static Slice spookyHashV2_32(@SqlType(StandardTypes.VARBINARY) Slice slice) { Slice hash = Slices.allocate(Integer.BYTES); hash.setInt(0, Integer.reverseBytes(SpookyHashV2.hash32(slice, 0, slice.length(), 0))); return hash; }
private void writeTimestamp(int columnIndex, Timestamp value) { int offset = getOffset(columnIndex); bufferSlice.setLong(offset, value.getEpochSecond()); bufferSlice.setInt(offset + 8, value.getNano()); clearNull(columnIndex); }
private void doFlush() { if (buffer != null && count > 0) { // write page header bufferSlice.setInt(0, count); buffer.limit(position); // flush page Page page = Page.wrap(buffer) .setStringReferences(stringReferences) .setValueReferences(valueReferences); buffer = null; bufferSlice = null; output.add(page); } }
private static List<Long> getSyncPositionsBruteForce(RcFileReader recordReader, File file) { Slice slice = Slices.allocate((int) file.length()); try (InputStream in = new FileInputStream(file)) { slice.setBytes(0, in, slice.length()); } catch (IOException e) { throw new UncheckedIOException(e); } List<Long> syncPositionsBruteForce = new ArrayList<>(); Slice sync = Slices.allocate(SIZE_OF_INT + SIZE_OF_LONG + SIZE_OF_LONG); sync.setInt(0, -1); sync.setBytes(SIZE_OF_INT, recordReader.getSync()); long syncPosition = 0; while (syncPosition >= 0) { syncPosition = slice.indexOf(sync, (int) syncPosition); if (syncPosition > 0) { syncPositionsBruteForce.add(syncPosition); syncPosition++; } } return syncPositionsBruteForce; }
private static void writeGeometryCollection(Geometry collection, DynamicSliceOutput output) { output.appendByte(GeometrySerializationType.GEOMETRY_COLLECTION.code()); for (int geometryIndex = 0; geometryIndex < collection.getNumGeometries(); geometryIndex++) { Geometry geometry = collection.getGeometryN(geometryIndex); int startPosition = output.size(); // leave 4 bytes for the shape length output.appendInt(0); writeGeometry(geometry, output); int endPosition = output.size(); int length = endPosition - startPosition - Integer.BYTES; output.getUnderlyingSlice().setInt(startPosition, length); } }
private static void writeGeometryCollection(DynamicSliceOutput output, OGCGeometryCollection collection) { output.appendByte(GeometrySerializationType.GEOMETRY_COLLECTION.code()); for (int geometryIndex = 0; geometryIndex < collection.numGeometries(); geometryIndex++) { OGCGeometry geometry = collection.geometryN(geometryIndex); int startPosition = output.size(); // leave 4 bytes for the shape length output.appendInt(0); writeGeometry(output, geometry); int endPosition = output.size(); int length = endPosition - startPosition - Integer.BYTES; output.getUnderlyingSlice().setInt(startPosition, length); } }
public void addStripe(int rowCount) { longSlice.setInt(0, rowCount); stripeHash.update(longBuffer, 0, Integer.BYTES); }