this.packer.unpack8Values(buffer, buffer.position(), this.currentBuffer, valueIndex); valueIndex += 8;
private void pack() { packer.pack8Values(input, 0, packed, packedPosition); packedPosition += bitWidth; totalValues += inputSize; inputSize = 0; }
/** * Compatibility API * @param input the input bytes * @param inPos where to read from in input * @param output the output values * @param outPos where to write to in output */ @Deprecated public void unpack32Values(byte[] input, int inPos, int[] output, int outPos) { unpack32Values(ByteBuffer.wrap(input), inPos, output, outPos); } }
this.packer.unpack8Values(buffer, buffer.position(), this.currentBuffer, valueIndex); valueIndex += 8;
private void pack() { packer.pack8Values(input, 0, packed, packedPosition); packedPosition += bitWidth; totalValues += inputSize; inputSize = 0; }
/** * Compatibility API * @param input the input bytes * @param inPos where to read from in input * @param output the output values * @param outPos where to write to in output */ @Deprecated public void unpack32Values(byte[] input, int inPos, int[] output, int outPos) { unpack32Values(ByteBuffer.wrap(input), inPos, output, outPos); } }
int valueIndex = 0; for (int byteIndex = offset; valueIndex < this.currentCount; byteIndex += this.bitWidth) { this.packer.unpack8Values(in, byteIndex, this.currentBuffer, valueIndex); valueIndex += 8;
private void writeOrAppendBitPackedRun() throws IOException { if (bitPackedGroupCount >= 63) { // we've packed as many values as we can for this run, // end it and start a new one endPreviousBitPackedRun(); } if (bitPackedRunHeaderPointer == -1) { // this is a new bit-packed-run, allocate a byte for the header // and keep a "pointer" to it so that it can be mutated later baos.write(0); // write a sentinel value bitPackedRunHeaderPointer = baos.getCurrentIndex(); } packer.pack8Values(bufferedValues, 0, packBuffer, 0); baos.write(packBuffer); // empty the buffer, they've all been written numBufferedValues = 0; // clear the repeat count, as some repeated values // may have just been bit packed into this run repeatCount = 0; ++bitPackedGroupCount; }
/** * Compatibility API * @param input the input bytes * @param inPos where to read from in input * @param output the output values * @param outPos where to write to in output */ @Deprecated public void unpack8Values(final byte[] input, final int inPos, final int[] output, final int outPos) { unpack8Values(ByteBuffer.wrap(input), inPos, output, outPos); }
private void writeOrAppendBitPackedRun() throws IOException { if (bitPackedGroupCount >= 63) { // we've packed as many values as we can for this run, // end it and start a new one endPreviousBitPackedRun(); } if (bitPackedRunHeaderPointer == -1) { // this is a new bit-packed-run, allocate a byte for the header // and keep a "pointer" to it so that it can be mutated later baos.write(0); // write a sentinel value bitPackedRunHeaderPointer = baos.getCurrentIndex(); } packer.pack8Values(bufferedValues, 0, packBuffer, 0); baos.write(packBuffer); // empty the buffer, they've all been written numBufferedValues = 0; // clear the repeat count, as some repeated values // may have just been bit packed into this run repeatCount = 0; ++bitPackedGroupCount; }
/** * Compatibility API * @param input the input bytes * @param inPos where to read from in input * @param output the output values * @param outPos where to write to in output */ @Deprecated public void unpack8Values(final byte[] input, final int inPos, final int[] output, final int outPos) { unpack8Values(ByteBuffer.wrap(input), inPos, output, outPos); }
packer.pack8Values(deltaBlockBuffer, j, miniBlockByteBuffer, blockOffset); blockOffset += currentBitWidth;
private void readNext() throws IOException { Preconditions.checkArgument(in.available() > 0, "Reading past RLE/BitPacking stream."); final int header = BytesUtils.readUnsignedVarInt(in); mode = (header & 1) == 0 ? MODE.RLE : MODE.PACKED; switch (mode) { case RLE: currentCount = header >>> 1; LOG.debug("reading {} values RLE", currentCount); currentValue = BytesUtils.readIntLittleEndianPaddedOnBitWidth(in, bitWidth); break; case PACKED: int numGroups = header >>> 1; currentCount = numGroups * 8; LOG.debug("reading {} values BIT PACKED", currentCount); currentBuffer = new int[currentCount]; // TODO: reuse a buffer byte[] bytes = new byte[numGroups * bitWidth]; // At the end of the file RLE data though, there might not be that many bytes left. int bytesToRead = (int)Math.ceil(currentCount * bitWidth / 8.0); bytesToRead = Math.min(bytesToRead, in.available()); new DataInputStream(in).readFully(bytes, 0, bytesToRead); for (int valueIndex = 0, byteIndex = 0; valueIndex < currentCount; valueIndex += 8, byteIndex += bitWidth) { packer.unpack8Values(bytes, byteIndex, currentBuffer, valueIndex); } break; default: throw new ParquetDecodingException("not a valid mode " + mode); } } }
packer.pack8Values(deltaBlockBuffer, j, miniBlockByteBuffer, blockOffset); blockOffset += currentBitWidth;
private void readNext() throws IOException { Preconditions.checkArgument(in.available() > 0, "Reading past RLE/BitPacking stream."); final int header = BytesUtils.readUnsignedVarInt(in); mode = (header & 1) == 0 ? MODE.RLE : MODE.PACKED; switch (mode) { case RLE: currentCount = header >>> 1; LOG.debug("reading {} values RLE", currentCount); currentValue = BytesUtils.readIntLittleEndianPaddedOnBitWidth(in, bitWidth); break; case PACKED: int numGroups = header >>> 1; currentCount = numGroups * 8; LOG.debug("reading {} values BIT PACKED", currentCount); currentBuffer = new int[currentCount]; // TODO: reuse a buffer byte[] bytes = new byte[numGroups * bitWidth]; // At the end of the file RLE data though, there might not be that many bytes left. int bytesToRead = (int)Math.ceil(currentCount * bitWidth / 8.0); bytesToRead = Math.min(bytesToRead, in.available()); new DataInputStream(in).readFully(bytes, 0, bytesToRead); for (int valueIndex = 0, byteIndex = 0; valueIndex < currentCount; valueIndex += 8, byteIndex += bitWidth) { packer.unpack8Values(bytes, byteIndex, currentBuffer, valueIndex); } break; default: throw new ParquetDecodingException("not a valid mode " + mode); } } }
@Override public int readInteger() { ++ decodedPosition; if (decodedPosition == decoded.length) { try { if (in.available() < bitWidth) { // unpack8Values needs at least bitWidth bytes to read from, // We have to fill in 0 byte at the end of encoded bytes. byte[] tempEncode = new byte[bitWidth]; in.read(tempEncode, 0, in.available()); packer.unpack8Values(tempEncode, 0, decoded, 0); } else { ByteBuffer encoded = in.slice(bitWidth); packer.unpack8Values(encoded, encoded.position(), decoded, 0); } } catch (IOException e) { throw new ParquetDecodingException("Failed to read packed values", e); } decodedPosition = 0; } return decoded[decodedPosition]; }
@Override public int readInteger() { ++ decodedPosition; if (decodedPosition == decoded.length) { try { if (in.available() < bitWidth) { // unpack8Values needs at least bitWidth bytes to read from, // We have to fill in 0 byte at the end of encoded bytes. byte[] tempEncode = new byte[bitWidth]; in.read(tempEncode, 0, in.available()); packer.unpack8Values(tempEncode, 0, decoded, 0); } else { ByteBuffer encoded = in.slice(bitWidth); packer.unpack8Values(encoded, encoded.position(), decoded, 0); } } catch (IOException e) { throw new ParquetDecodingException("Failed to read packed values", e); } decodedPosition = 0; } return decoded[decodedPosition]; }
int valueIndex = 0; for (int byteIndex = offset; valueIndex < this.currentCount; byteIndex += this.bitWidth) { this.packer.unpack8Values(in, byteIndex, this.currentBuffer, valueIndex); valueIndex += 8;
int valueIndex = 0; for (int byteIndex = offset; valueIndex < this.currentCount; byteIndex += this.bitWidth) { this.packer.unpack8Values(in, byteIndex, this.currentBuffer, valueIndex); valueIndex += 8;