output.write(2);
private static void unrolledBitPack4(long[] input, int offset, int len, SliceOutput output) { final int numHops = 2; final int remainder = len % numHops; final int endOffset = offset + len; final int endUnroll = endOffset - remainder; int val = 0; for (int i = offset; i < endUnroll; i = i + numHops) { val = (int) (val | ((input[i] & 15) << 4) | (input[i + 1]) & 15); output.write(val); val = 0; } if (remainder > 0) { int startShift = 4; for (int i = endUnroll; i < endOffset; i++) { val = (int) (val | (input[i] & 15) << startShift); startShift -= 4; } output.write(val); } }
private static void unrolledBitPack2(long[] input, int offset, int len, SliceOutput output) { final int numHops = 4; final int remainder = len % numHops; final int endOffset = offset + len; final int endUnroll = endOffset - remainder; int val = 0; for (int i = offset; i < endUnroll; i = i + numHops) { val = (int) (val | ((input[i] & 3) << 6) | ((input[i + 1] & 3) << 4) | ((input[i + 2] & 3) << 2) | (input[i + 3]) & 3); output.write(val); val = 0; } if (remainder > 0) { int startShift = 6; for (int i = endUnroll; i < endOffset; i++) { val = (int) (val | (input[i] & 3) << startShift); startShift -= 2; } output.write(val); } }
private static void unrolledBitPack1(long[] input, int offset, int len, SliceOutput output) { final int numHops = 8; final int remainder = len % numHops; final int endOffset = offset + len; final int endUnroll = endOffset - remainder; int val = 0; for (int i = offset; i < endUnroll; i = i + numHops) { val = (int) (val | ((input[i] & 1) << 7) | ((input[i + 1] & 1) << 6) | ((input[i + 2] & 1) << 5) | ((input[i + 3] & 1) << 4) | ((input[i + 4] & 1) << 3) | ((input[i + 5] & 1) << 2) | ((input[i + 6] & 1) << 1) | (input[i + 7]) & 1); output.write(val); val = 0; } if (remainder > 0) { int startShift = 7; for (int i = endUnroll; i < endOffset; i++) { val = (int) (val | (input[i] & 1) << startShift); startShift -= 1; } output.write(val); } }
private void writeSlice(SliceOutput output, Block block, int position) { // first vint is scale writeVInt(output, type.getScale()); // second vint is length // todo get rid of BigInteger BigInteger decimal = Decimals.decodeUnscaledValue(block.getSlice(position, 0, BYTES_IN_LONG_DECIMAL)); byte[] decimalBytes = decimal.toByteArray(); writeVInt(output, decimalBytes.length); // write value (big endian) // NOTE: long decimals are stored in a slice in big endian encoding for (byte decimalByte : decimalBytes) { output.write(decimalByte); } } }
output.write(writeBuffer, 0, toWrite);
@SuppressWarnings("AssignmentToForLoopParameter") private static ColumnData unescape(ColumnData columnData, byte escapeByte) { Slice slice = columnData.getSlice(); // does slice contain escape byte if (slice.indexOfByte(escapeByte) < 0) { return columnData; } Slice newSlice = Slices.allocate(slice.length()); SliceOutput output = newSlice.getOutput(); int[] newOffsets = new int[columnData.rowCount() + 1]; for (int row = 0; row < columnData.rowCount(); row++) { int offset = columnData.getOffset(row); int length = columnData.getLength(row); for (int i = 0; i < length; i++) { byte value = slice.getByte(offset + i); if (value == escapeByte && i + 1 < length) { // read byte after escape i++; value = slice.getByte(offset + i); } output.write(value); } newOffsets[row + 1] = output.size(); } return new ColumnData(newOffsets, output.slice()); }
output.write(writeBuffer, 0, toWrite);
private static void unrolledBitPack4(long[] input, int offset, int len, SliceOutput output) { final int numHops = 2; final int remainder = len % numHops; final int endOffset = offset + len; final int endUnroll = endOffset - remainder; int val = 0; for (int i = offset; i < endUnroll; i = i + numHops) { val = (int) (val | ((input[i] & 15) << 4) | (input[i + 1]) & 15); output.write(val); val = 0; } if (remainder > 0) { int startShift = 4; for (int i = endUnroll; i < endOffset; i++) { val = (int) (val | (input[i] & 15) << startShift); startShift -= 4; } output.write(val); } }
private static void unrolledBitPack4(long[] input, int offset, int len, SliceOutput output) { final int numHops = 2; final int remainder = len % numHops; final int endOffset = offset + len; final int endUnroll = endOffset - remainder; int val = 0; for (int i = offset; i < endUnroll; i = i + numHops) { val = (int) (val | ((input[i] & 15) << 4) | (input[i + 1]) & 15); output.write(val); val = 0; } if (remainder > 0) { int startShift = 4; for (int i = endUnroll; i < endOffset; i++) { val = (int) (val | (input[i] & 15) << startShift); startShift -= 4; } output.write(val); } }
private static void unrolledBitPack4(long[] input, int offset, int len, SliceOutput output) { final int numHops = 2; final int remainder = len % numHops; final int endOffset = offset + len; final int endUnroll = endOffset - remainder; int val = 0; for (int i = offset; i < endUnroll; i = i + numHops) { val = (int) (val | ((input[i] & 15) << 4) | (input[i + 1]) & 15); output.write(val); val = 0; } if (remainder > 0) { int startShift = 4; for (int i = endUnroll; i < endOffset; i++) { val = (int) (val | (input[i] & 15) << startShift); startShift -= 4; } output.write(val); } }
@Test public void testEncodingBytes() throws Exception { byte[] data = new byte[18000]; ThreadLocalRandom.current().nextBytes(data); assertEncoding(sliceOutput -> sliceOutput.write(data, 0, 0), Arrays.copyOfRange(data, 0, 0)); assertEncoding(sliceOutput -> sliceOutput.write(data, 0, 3), Arrays.copyOfRange(data, 0, 3)); assertEncoding(sliceOutput -> sliceOutput.write(data, 0, 370), Arrays.copyOfRange(data, 0, 370)); assertEncoding(sliceOutput -> sliceOutput.write(data, 0, 4095), Arrays.copyOfRange(data, 0, 4095)); assertEncoding(sliceOutput -> sliceOutput.write(data, 0, 4096), Arrays.copyOfRange(data, 0, 4096)); assertEncoding(sliceOutput -> sliceOutput.write(data, 0, 12348), Arrays.copyOfRange(data, 0, 12348)); assertEncoding(sliceOutput -> sliceOutput.write(data, 0, 16384), Arrays.copyOfRange(data, 0, 16384)); assertEncoding(sliceOutput -> sliceOutput.write(data, 0, 18000), Arrays.copyOfRange(data, 0, 18000)); }
@Test public void testEncodingBytes() throws Exception { byte[] data = new byte[18000]; ThreadLocalRandom.current().nextBytes(data); assertEncoding(sliceOutput -> sliceOutput.write(data, 0, 0), Arrays.copyOfRange(data, 0, 0)); assertEncoding(sliceOutput -> sliceOutput.write(data, 0, 3), Arrays.copyOfRange(data, 0, 3)); assertEncoding(sliceOutput -> sliceOutput.write(data, 0, 370), Arrays.copyOfRange(data, 0, 370)); assertEncoding(sliceOutput -> sliceOutput.write(data, 0, 4095), Arrays.copyOfRange(data, 0, 4095)); assertEncoding(sliceOutput -> sliceOutput.write(data, 0, 4096), Arrays.copyOfRange(data, 0, 4096)); assertEncoding(sliceOutput -> sliceOutput.write(data, 0, 12348), Arrays.copyOfRange(data, 0, 12348)); assertEncoding(sliceOutput -> sliceOutput.write(data, 0, 16384), Arrays.copyOfRange(data, 0, 16384)); assertEncoding(sliceOutput -> sliceOutput.write(data, 0, 18000), Arrays.copyOfRange(data, 0, 18000)); }
private void writeSlice(SliceOutput output, Block block, int position) { // first vint is scale writeVInt(output, type.getScale()); // second vint is length // todo get rid of BigInteger BigInteger decimal = Decimals.decodeUnscaledValue(block.getSlice(position, 0, BYTES_IN_LONG_DECIMAL)); byte[] decimalBytes = decimal.toByteArray(); writeVInt(output, decimalBytes.length); // write value (big endian) // NOTE: long decimals are stored in a slice in big endian encoding for (byte decimalByte : decimalBytes) { output.write(decimalByte); } } }
private void writeSlice(SliceOutput output, Block block, int position) { // first vint is scale writeVInt(output, type.getScale()); // second vint is length // todo get rid of BigInteger BigInteger decimal = Decimals.decodeUnscaledValue(block.getSlice(position, 0, BYTES_IN_LONG_DECIMAL)); byte[] decimalBytes = decimal.toByteArray(); writeVInt(output, decimalBytes.length); // write value (big endian) // NOTE: long decimals are stored in a slice in big endian encoding for (byte decimalByte : decimalBytes) { output.write(decimalByte); } } }