private static ValueInputStream<?> createLongStream( OrcInputStream inputStream, ColumnEncodingKind encoding, OrcTypeKind type, boolean signed, boolean usesVInt) { if (encoding == DIRECT_V2 || encoding == DICTIONARY_V2) { return new LongInputStreamV2(inputStream, signed, false); } else if (encoding == DIRECT || encoding == DICTIONARY) { return new LongInputStreamV1(inputStream, signed); } else if (encoding == DWRF_DIRECT) { return new LongInputStreamDwrf(inputStream, type, signed, usesVInt); } else { throw new IllegalArgumentException("Unsupported encoding for long stream: " + encoding); } } }
@Override protected Long readValue(LongInputStreamV1 valueStream) throws IOException { return valueStream.next(); } }
@Override // This comes from the Apache Hive ORC code public long next() throws IOException { long result; if (used == numLiterals) { readValues(); } if (repeat) { result = literals[0] + (used++) * delta; } else { result = literals[used++]; } return result; }
@Override public void seekToCheckpoint(LongStreamCheckpoint checkpoint) throws IOException { super.seekToCheckpoint(checkpoint); RowGroupDictionaryLengthStreamCheckpoint rowGroupDictionaryLengthStreamCheckpoint = (RowGroupDictionaryLengthStreamCheckpoint) checkpoint; entryCount = rowGroupDictionaryLengthStreamCheckpoint.getRowGroupDictionarySize(); } }
@Override public void seekToCheckpoint(LongStreamCheckpoint checkpoint) throws IOException { LongStreamV1Checkpoint v1Checkpoint = (LongStreamV1Checkpoint) checkpoint; // if the checkpoint is within the current buffer, just adjust the pointer if (lastReadInputCheckpoint == v1Checkpoint.getInputStreamCheckpoint() && v1Checkpoint.getOffset() <= numLiterals) { used = v1Checkpoint.getOffset(); } else { // otherwise, discard the buffer and start over input.seekToCheckpoint(v1Checkpoint.getInputStreamCheckpoint()); numLiterals = 0; used = 0; skip(v1Checkpoint.getOffset()); } }
@Override public void seekToCheckpoint(LongStreamCheckpoint checkpoint) throws IOException { super.seekToCheckpoint(checkpoint); RowGroupDictionaryLengthStreamCheckpoint rowGroupDictionaryLengthStreamCheckpoint = (RowGroupDictionaryLengthStreamCheckpoint) checkpoint; entryCount = rowGroupDictionaryLengthStreamCheckpoint.getRowGroupDictionarySize(); } }
@Override public void seekToCheckpoint(LongStreamCheckpoint checkpoint) throws IOException { LongStreamV1Checkpoint v1Checkpoint = (LongStreamV1Checkpoint) checkpoint; // if the checkpoint is within the current buffer, just adjust the pointer if (lastReadInputCheckpoint == v1Checkpoint.getInputStreamCheckpoint() && v1Checkpoint.getOffset() <= numLiterals) { used = v1Checkpoint.getOffset(); } else { // otherwise, discard the buffer and start over input.seekToCheckpoint(v1Checkpoint.getInputStreamCheckpoint()); numLiterals = 0; used = 0; skip(v1Checkpoint.getOffset()); } }
@Override protected LongInputStreamV1 createValueStream(Slice slice) throws OrcCorruptionException { Optional<OrcDecompressor> orcDecompressor = createOrcDecompressor(ORC_DATA_SOURCE_ID, SNAPPY, COMPRESSION_BLOCK_SIZE); OrcInputStream input = new OrcInputStream(ORC_DATA_SOURCE_ID, slice.getInput(), orcDecompressor, newSimpleAggregatedMemoryContext(), slice.getRetainedSize()); return new LongInputStreamV1(input, true); }
@Override public void skip(long items) throws IOException { while (items > 0) { if (used == numLiterals) { readValues(); } long consume = Math.min(items, numLiterals - used); used += consume; items -= consume; } } }
@Override protected Long readValue(LongInputStreamV1 valueStream) throws IOException { return valueStream.next(); } }
private static ValueInputStream<?> createLongStream( OrcInputStream inputStream, ColumnEncodingKind encoding, OrcTypeKind type, boolean signed, boolean usesVInt) { if (encoding == DIRECT_V2 || encoding == DICTIONARY_V2) { return new LongInputStreamV2(inputStream, signed, false); } else if (encoding == DIRECT || encoding == DICTIONARY) { return new LongInputStreamV1(inputStream, signed); } else if (encoding == DWRF_DIRECT) { return new LongInputStreamDwrf(inputStream, type, signed, usesVInt); } else { throw new IllegalArgumentException("Unsupported encoding for long stream: " + encoding); } } }
@Override // This comes from the Apache Hive ORC code public long next() throws IOException { long result; if (used == numLiterals) { readValues(); } if (repeat) { result = literals[0] + (used++) * delta; } else { result = literals[used++]; } return result; }
@Override protected LongInputStreamV1 createValueStream(Slice slice) throws OrcCorruptionException { Optional<OrcDecompressor> orcDecompressor = createOrcDecompressor(ORC_DATA_SOURCE_ID, SNAPPY, COMPRESSION_BLOCK_SIZE); OrcInputStream input = new OrcInputStream(ORC_DATA_SOURCE_ID, slice.getInput(), orcDecompressor, newSimpleAggregatedMemoryContext(), slice.getRetainedSize()); return new LongInputStreamV1(input, true); }
@Override public void skip(long items) throws IOException { while (items > 0) { if (used == numLiterals) { readValues(); } long consume = Math.min(items, numLiterals - used); used += consume; items -= consume; } } }