readOffset = presentStream.countBitsSet(readOffset); inDictionaryStream.skip(readOffset); for (int i = 0; i < nextBatchSize; i++) { idsVector[i] = toIntExact(dataStream.next()); if (!inDictionaryStream.nextBit()) { int nullValues = presentStream.getUnsetBits(nextBatchSize); if (nullValues != nextBatchSize) { throw new OrcCorruptionException(streamDescriptor.getOrcDataSourceId(), "Value is not null but data stream is not present"); if (!presentStream.nextBit()) { if (inDictionaryStream != null && !inDictionaryStream.nextBit()) {
public int countBitsSet(int items) throws IOException { int count = 0; // count buffered data if (items > bitsInData && bitsInData > 0) { count += bitCount(data); items -= bitsInData; bitsInData = 0; } // count whole bytes while (items > 8) { count += bitCount(byteStream.next()); items -= 8; } // count remaining bits for (int i = 0; i < items; i++) { count += nextBit() ? 1 : 0; } return count; }
readOffset = presentStream.countBitsSet(readOffset); throw new OrcCorruptionException(streamDescriptor.getOrcDataSourceId(), "Value is not null but data stream is not present"); dataStream.skip(readOffset); throw new OrcCorruptionException(streamDescriptor.getOrcDataSourceId(), "Value is not null but data stream is not present"); dataStream.getSetBits(type, nextBatchSize, builder); if (presentStream.nextBit()) { verify(dataStream != null); type.writeBoolean(builder, dataStream.nextBit());
readOffset = presentStream.countBitsSet(readOffset); int valueReadOffset = inMapStreams.get(i).countBitsSet(readOffset); valueStreamReaders.get(i).prepareNextRead(valueReadOffset); for (int keyIndex = 0; keyIndex < inMapStreams.size(); keyIndex++) { inMapVectors[keyIndex] = new boolean[nextBatchSize]; totalMapEntries += inMapStreams.get(keyIndex).getSetBits(nextBatchSize, inMapVectors[keyIndex]); int nullValues = presentStream.getUnsetBits(nextBatchSize, nullVector); if (nullValues == nextBatchSize) { for (int i = 0; i < inMapStreams.size(); i++) { for (int i = 0; i < inMapStreams.size(); i++) { inMapVectors[i] = new boolean[nextBatchSize]; totalMapEntries += inMapStreams.get(i).getSetBits(nextBatchSize, inMapVectors[i], nullVector);
readOffset = presentStream.countBitsSet(readOffset); if (presentStream.nextBit()) { verify(dataStream != null); type.writeLong(builder, floatToRawIntBits(dataStream.next()));
/** * Return the number of unset bits */ public int getUnsetBits(int batchSize) throws IOException { int count = 0; for (int i = 0; i < batchSize; i++) { count += nextBit() ? 0 : 1; } return count; }
readOffset = presentStream.countBitsSet(readOffset); int nullValues = presentStream.getUnsetBits(nextBatchSize, nullVector); if (nullValues != nextBatchSize) { blocks = getBlocksForType(type, nextBatchSize - nullValues);
@Override protected BooleanInputStream createValueStream(Slice slice) throws OrcCorruptionException { Optional<OrcDecompressor> orcDecompressor = createOrcDecompressor(ORC_DATA_SOURCE_ID, SNAPPY, COMPRESSION_BLOCK_SIZE); return new BooleanInputStream(new OrcInputStream(ORC_DATA_SOURCE_ID, slice.getInput(), orcDecompressor, newSimpleAggregatedMemoryContext(), slice.getRetainedSize())); }
/** * Sets the vector element to true if the bit is not set. */ public int getUnsetBits(int batchSize, boolean[] vector) throws IOException { return getUnsetBits(batchSize, vector, 0); }
@Override public void seekToCheckpoint(BooleanStreamCheckpoint checkpoint) throws IOException { byteStream.seekToCheckpoint(checkpoint.getByteStreamCheckpoint()); bitsInData = 0; skip(checkpoint.getOffset()); }
public boolean nextBit() throws IOException { // read more data if necessary if (bitsInData == 0) { readByte(); } // read bit boolean result = (data & HIGH_BIT_MASK) != 0; // mark bit consumed data <<= 1; bitsInData--; return result; }
private void seekToOffset() throws IOException { if (readOffset > 0) { if (presentStream != null) { // skip ahead the present bit reader, but count the set bits // and use this as the skip size for the data reader readOffset = presentStream.countBitsSet(readOffset); } if (readOffset > 0) { if (decimalStream == null) { throw new OrcCorruptionException(streamDescriptor.getOrcDataSourceId(), "Value is not null but decimal stream is not present"); } if (scaleStream == null) { throw new OrcCorruptionException(streamDescriptor.getOrcDataSourceId(), "Value is not null but scale stream is not present"); } decimalStream.skip(readOffset); scaleStream.skip(readOffset); } } }
readOffset = presentStream.countBitsSet(readOffset); if (presentStream.nextBit()) { verify(dataStream != null); type.writeLong(builder, dataStream.next());
/** * Sets the vector element to true if the bit is set. */ public int getSetBits(int batchSize, boolean[] vector) throws IOException { int count = 0; for (int i = 0; i < batchSize; i++) { vector[i] = nextBit(); count += vector[i] ? 1 : 0; } return count; }
readOffset = presentStream.countBitsSet(readOffset); int nullValues = presentStream.getUnsetBits(nextBatchSize, nullVector); if (nullValues != nextBatchSize) { if (lengthStream == null) {
readOffset = presentStream.countBitsSet(readOffset); int valueReadOffset = inMapStreams.get(i).countBitsSet(readOffset); valueStreamReaders.get(i).prepareNextRead(valueReadOffset); for (int keyIndex = 0; keyIndex < inMapStreams.size(); keyIndex++) { inMapVectors[keyIndex] = new boolean[nextBatchSize]; totalMapEntries += inMapStreams.get(keyIndex).getSetBits(nextBatchSize, inMapVectors[keyIndex]); int nullValues = presentStream.getUnsetBits(nextBatchSize, nullVector); if (nullValues == nextBatchSize) { for (int i = 0; i < inMapStreams.size(); i++) { for (int i = 0; i < inMapStreams.size(); i++) { inMapVectors[i] = new boolean[nextBatchSize]; totalMapEntries += inMapStreams.get(i).getSetBits(nextBatchSize, inMapVectors[i], nullVector);
return new BooleanInputStream(inputStream); switch (type) { case BOOLEAN: return new BooleanInputStream(inputStream); case BYTE: return new ByteInputStream(inputStream); return new BooleanInputStream(inputStream);
/** * Sets the vector element to true if the bit is not set. */ public int getUnsetBits(int batchSize, boolean[] vector) throws IOException { return getUnsetBits(batchSize, vector, 0); }
@Override public void seekToCheckpoint(BooleanStreamCheckpoint checkpoint) throws IOException { byteStream.seekToCheckpoint(checkpoint.getByteStreamCheckpoint()); bitsInData = 0; skip(checkpoint.getOffset()); }
public boolean nextBit() throws IOException { // read more data if necessary if (bitsInData == 0) { readByte(); } // read bit boolean result = (data & HIGH_BIT_MASK) != 0; // mark bit consumed data <<= 1; bitsInData--; return result; }