public BooleanInputStream(OrcInputStream byteStream) { this.byteStream = new ByteInputStream(byteStream); }
private void readByte() throws IOException { checkState(bitsInData == 0); data = byteStream.next(); bitsInData = 8; }
public byte next() throws IOException { if (offset == length) { readNextBlock(); } return buffer[offset++]; }
throw new OrcCorruptionException(streamDescriptor.getOrcDataSourceId(), "Value is not null but data stream is not present"); dataStream.skip(readOffset); throw new OrcCorruptionException(streamDescriptor.getOrcDataSourceId(), "Value is not null but data stream is not present"); dataStream.nextVector(type, nextBatchSize, builder); if (presentStream.nextBit()) { verify(dataStream != null); type.writeLong(builder, dataStream.next());
@Override public void seekToCheckpoint(BooleanStreamCheckpoint checkpoint) throws IOException { byteStream.seekToCheckpoint(checkpoint.getByteStreamCheckpoint()); bitsInData = 0; skip(checkpoint.getOffset()); }
@Override public void seekToCheckpoint(ByteStreamCheckpoint checkpoint) throws IOException { // if the checkpoint is within the current buffer, just adjust the pointer if (lastReadInputCheckpoint == checkpoint.getInputStreamCheckpoint() && checkpoint.getOffset() <= length) { offset = checkpoint.getOffset(); } else { // otherwise, discard the buffer and start over input.seekToCheckpoint(checkpoint.getInputStreamCheckpoint()); length = 0; offset = 0; skip(checkpoint.getOffset()); } }
throw new OrcCorruptionException(streamDescriptor.getOrcDataSourceId(), "Value is not null but data stream is not present"); dataStream.skip(readOffset); throw new OrcCorruptionException(streamDescriptor.getOrcDataSourceId(), "Value is not null but data stream is not present"); dataStream.nextVector(type, nextBatchSize, builder); if (presentStream.nextBit()) { verify(dataStream != null); type.writeLong(builder, dataStream.next());
@Override public void seekToCheckpoint(BooleanStreamCheckpoint checkpoint) throws IOException { byteStream.seekToCheckpoint(checkpoint.getByteStreamCheckpoint()); bitsInData = 0; skip(checkpoint.getOffset()); }
@Override public void seekToCheckpoint(ByteStreamCheckpoint checkpoint) throws IOException { // if the checkpoint is within the current buffer, just adjust the pointer if (lastReadInputCheckpoint == checkpoint.getInputStreamCheckpoint() && checkpoint.getOffset() <= length) { offset = checkpoint.getOffset(); } else { // otherwise, discard the buffer and start over input.seekToCheckpoint(checkpoint.getInputStreamCheckpoint()); length = 0; offset = 0; skip(checkpoint.getOffset()); } }
@Override protected Byte readValue(ByteInputStream valueStream) throws IOException { return valueStream.next(); } }
@Override protected ByteInputStream createValueStream(Slice slice) throws OrcCorruptionException { Optional<OrcDecompressor> orcDecompressor = createOrcDecompressor(ORC_DATA_SOURCE_ID, SNAPPY, COMPRESSION_BLOCK_SIZE); return new ByteInputStream(new OrcInputStream(ORC_DATA_SOURCE_ID, slice.getInput(), orcDecompressor, newSimpleAggregatedMemoryContext(), slice.getRetainedSize())); }
@Override public void skip(long items) throws IOException { while (items > 0) { if (offset == length) { readNextBlock(); } long consume = Math.min(items, length - offset); offset += consume; items -= consume; } }
public void nextVector(Type type, long items, BlockBuilder builder) throws IOException { for (int i = 0; i < items; i++) { type.writeLong(builder, next()); } } }
return new BooleanInputStream(inputStream); case BYTE: return new ByteInputStream(inputStream); case SHORT: case INT:
public byte next() throws IOException { if (offset == length) { readNextBlock(); } return buffer[offset++]; }
public int countBitsSet(int items) throws IOException { int count = 0; // count buffered data if (items > bitsInData && bitsInData > 0) { count += bitCount(data); items -= bitsInData; bitsInData = 0; } // count whole bytes while (items > 8) { count += bitCount(byteStream.next()); items -= 8; } // count remaining bits for (int i = 0; i < items; i++) { count += nextBit() ? 1 : 0; } return count; }
public BooleanInputStream(OrcInputStream byteStream) { this.byteStream = new ByteInputStream(byteStream); }
@Override public void skip(long items) throws IOException { while (items > 0) { if (offset == length) { readNextBlock(); } long consume = Math.min(items, length - offset); offset += consume; items -= consume; } }
private void readByte() throws IOException { checkState(bitsInData == 0); data = byteStream.next(); bitsInData = 8; }
@Override protected ByteInputStream createValueStream(Slice slice) throws OrcCorruptionException { Optional<OrcDecompressor> orcDecompressor = createOrcDecompressor(ORC_DATA_SOURCE_ID, SNAPPY, COMPRESSION_BLOCK_SIZE); return new ByteInputStream(new OrcInputStream(ORC_DATA_SOURCE_ID, slice.getInput(), orcDecompressor, newSimpleAggregatedMemoryContext(), slice.getRetainedSize())); }