@Override public void initFromPage(int valueCount, ByteBufferInputStream in) throws IOException { this.in = in; if (fixedWidth) { // initialize for repetition and definition levels if (readLength) { int length = readIntLittleEndian(); this.in = in.sliceStream(length); } } else { // initialize for values if (in.available() > 0) { init(in.read()); } } if (bitWidth == 0) { // 0 bit width, treat this as an RLE run of valueCount number of 0's. this.mode = MODE.RLE; this.currentCount = valueCount; this.currentValue = 0; } else { this.currentCount = 0; } }
@Override public void initFromPage(int valueCount, ByteBufferInputStream in) throws IOException { this.in = in; if (fixedWidth) { // initialize for repetition and definition levels if (readLength) { int length = readIntLittleEndian(); this.in = in.sliceStream(length); } } else { // initialize for values if (in.available() > 0) { init(in.read()); } } if (bitWidth == 0) { // 0 bit width, treat this as an RLE run of valueCount number of 0's. this.mode = MODE.RLE; this.currentCount = valueCount; this.currentValue = 0; } else { this.currentCount = 0; } }
public int available() { return delegate.available(); }
/** * {@inheritDoc} * @see org.apache.parquet.column.values.ValuesReader#initFromPage(int, ByteBufferInputStream) */ @Override public void initFromPage(int valueCount, ByteBufferInputStream stream) throws IOException { LOG.debug("init from page at offset {} for length {}", stream.position(), stream.available()); this.in.initFromPage(valueCount, stream); }
/** * @return the slice of the byte buffer inside this stream * @deprecated Will be removed in 2.0.0; Use {@link #slice(int)} instead */ @Deprecated public ByteBuffer toByteBuffer() { try { return slice(available()); } catch (EOFException e) { throw new ShouldNeverHappenException(e); } }
@Override public void initFromPage(int valueCount, ByteBufferInputStream stream) throws IOException { int effectiveBitLength = valueCount * bitWidth; int length = BytesUtils.paddedByteCountFromBits(effectiveBitLength); // ceil LOG.debug("reading {} bytes for {} values of size {} bits.", length, valueCount, bitWidth); // work-around for null values. this will not happen for repetition or // definition levels (never null), but will happen when valueCount has not // been adjusted for null values in the data. length = Math.min(length, stream.available()); this.in = stream.sliceStream(length); this.decodedPosition = VALUES_AT_A_TIME - 1; }
@Override public void initFromPage(int valueCount, ByteBufferInputStream stream) throws IOException { LOG.debug("init from page at offset {} for length {}", stream.position(), stream.available()); this.in = stream.remainingStream(); } }
/** * {@inheritDoc} * @see org.apache.parquet.column.values.ValuesReader#initFromPage(int, ByteBufferInputStream) */ @Override public void initFromPage(int valueCount, ByteBufferInputStream stream) throws IOException { LOG.debug("init from page at offset {} for length {}", stream.position(), stream.available()); this.in.initFromPage(valueCount, stream); } }
@Override public void initFromPage(int valueCount, ByteBufferInputStream stream) throws IOException { LOG.debug("init from page at offset {} for length {}", stream.position(), stream.available()); this.in = stream.remainingStream(); } }
@Override public void initFromPage(int valueCount, ByteBufferInputStream stream) throws IOException { LOG.debug("init from page at offset {} for length {}", stream.position(), (stream.available() - stream.position())); this.in = stream.remainingStream(); } }
@Override public void initFromPage(int valueCount, ByteBufferInputStream stream) throws IOException { LOG.debug("init from page at offset {} for length {}", stream.position(), stream.available()); lengthReader.initFromPage(valueCount, stream); this.in = stream.remainingStream(); }
public BytesInput readAsBytesInput(int size) throws IOException { int available = stream.available(); if (size > available) { // this is to workaround a bug where the compressedLength // of the chunk is missing the size of the header of the dictionary // to allow reading older files (using dictionary) we need this. // usually 13 to 19 bytes are missing int missingBytes = size - available; LOG.info("completed the column chunk with {} bytes", missingBytes); List<ByteBuffer> buffers = new ArrayList<>(); buffers.addAll(stream.sliceBuffers(available)); ByteBuffer lastBuffer = ByteBuffer.allocate(missingBytes); f.readFully(lastBuffer); buffers.add(lastBuffer); return BytesInput.from(buffers); } return super.readAsBytesInput(size); }
@Override public void initFromPage(int valueCount, ByteBufferInputStream stream) throws IOException { LOG.debug("init from page at offset {} for length {}", stream.position(), stream.available()); this.in = new LittleEndianDataInputStream(stream.remainingStream()); }
@Override public void initFromPage(int valueCount, ByteBufferInputStream stream) throws IOException { LOG.debug("init from page at offset {} for length {}", stream.position(), (stream.available() - stream.position())); this.in = stream.remainingStream(); } }
@Override public void initFromPage(int valueCount, ByteBufferInputStream stream) throws IOException { LOG.debug("init from page at offset {} for length {}", stream.position(), stream.available()); this.in = new LittleEndianDataInputStream(stream.remainingStream()); }
@Override public void initFromPage(int valueCount, ByteBufferInputStream stream) throws IOException { int effectiveBitLength = valueCount * bitWidth; int length = BytesUtils.paddedByteCountFromBits(effectiveBitLength); // ceil LOG.debug("reading {} bytes for {} values of size {} bits.", length, valueCount, bitWidth); // work-around for null values. this will not happen for repetition or // definition levels (never null), but will happen when valueCount has not // been adjusted for null values in the data. length = Math.min(length, stream.available()); this.in = stream.sliceStream(length); this.decodedPosition = VALUES_AT_A_TIME - 1; updateNextOffset(length); }
@Override public void initFromPage(int valueCount, ByteBufferInputStream stream) throws IOException { LOG.debug("init from page at offset {} for length {}", stream.position(), stream.available()); lengthReader.initFromPage(valueCount, stream); this.in = stream.remainingStream(); }
public BytesInput readAsBytesInput(int size) throws IOException { int available = stream.available(); if (size > available) { // this is to workaround a bug where the compressedLength // of the chunk is missing the size of the header of the dictionary // to allow reading older files (using dictionary) we need this. // usually 13 to 19 bytes are missing int missingBytes = size - available; LOG.info("completed the column chunk with {} bytes", missingBytes); List<ByteBuffer> buffers = new ArrayList<>(); buffers.addAll(stream.sliceBuffers(available)); ByteBuffer lastBuffer = ByteBuffer.allocate(missingBytes); f.readFully(lastBuffer); buffers.add(lastBuffer); return BytesInput.from(buffers); } return super.readAsBytesInput(size); }
@Override public void initFromPage(int valueCount, ByteBufferInputStream stream) throws IOException { this.in = stream.remainingStream(); if (in.available() > 0) { LOG.debug("init from page at offset {} for length {}", stream.position(), stream.available()); int bitWidth = BytesUtils.readIntLittleEndianOnOneByte(in); LOG.debug("bit width {}", bitWidth); decoder = new RunLengthBitPackingHybridDecoder(bitWidth, in); } else { decoder = new RunLengthBitPackingHybridDecoder(1, in) { @Override public int readInt() throws IOException { throw new IOException("Attempt to read from empty page"); } }; } }
@Override public void initFromPage(int valueCount, ByteBufferInputStream stream) throws IOException { this.in = stream.remainingStream(); if (in.available() > 0) { LOG.debug("init from page at offset {} for length {}", stream.position(), stream.available()); int bitWidth = BytesUtils.readIntLittleEndianOnOneByte(in); LOG.debug("bit width {}", bitWidth); decoder = new RunLengthBitPackingHybridDecoder(bitWidth, in); } else { decoder = new RunLengthBitPackingHybridDecoder(1, in) { @Override public int readInt() throws IOException { throw new IOException("Attempt to read from empty page"); } }; } }