public BooleanOutputStream(CompressionKind compression, int bufferSize) { this(new ByteOutputStream(compression, bufferSize)); }
private void flushData() { byteOutputStream.writeByte((byte) data); data = 0; bitsInData = 0; }
@Override public void close() { closed = true; dataStream.close(); presentStream.close(); }
@Override public long getRetainedBytes() { // NOTE: we do not include checkpoints because they should be small and it would be annoying to calculate the size return INSTANCE_SIZE + byteOutputStream.getRetainedBytes(); }
@Override public void reset() { data = 0; bitsInData = 0; closed = false; byteOutputStream.reset(); checkpointBitOffsets.clear(); }
@Override public void recordCheckpoint() { checkState(!closed); byteOutputStream.recordCheckpoint(); checkpointBitOffsets.add(bitsInData); }
flushSequence(); flushSequence(); runCount = 0; flushSequence();
@Override public long getBufferedBytes() { return byteOutputStream.getBufferedBytes(); }
@Override public StreamDataOutput getStreamDataOutput(int column) { checkState(closed); return byteOutputStream.getStreamDataOutput(column); }
@Override public List<BooleanStreamCheckpoint> getCheckpoints() { checkState(closed); ImmutableList.Builder<BooleanStreamCheckpoint> booleanStreamCheckpoint = ImmutableList.builder(); List<ByteStreamCheckpoint> byteStreamCheckpoints = byteOutputStream.getCheckpoints(); for (int groupId = 0; groupId < checkpointBitOffsets.size(); groupId++) { int checkpointBitOffset = checkpointBitOffsets.get(groupId); ByteStreamCheckpoint byteStreamCheckpoint = byteStreamCheckpoints.get(groupId); booleanStreamCheckpoint.add(new BooleanStreamCheckpoint(checkpointBitOffset, byteStreamCheckpoint)); } return booleanStreamCheckpoint.build(); }
@Override public long getRetainedBytes() { long retainedBytes = INSTANCE_SIZE + dataStream.getRetainedBytes() + presentStream.getRetainedBytes(); for (ColumnStatistics statistics : rowGroupColumnStatistics) { retainedBytes += statistics.getRetainedSizeInBytes(); } return retainedBytes; }
@Override public void reset() { closed = false; dataStream.reset(); presentStream.reset(); rowGroupColumnStatistics.clear(); nonNullValueCount = 0; } }
@Override public void beginRowGroup() { presentStream.recordCheckpoint(); dataStream.recordCheckpoint(); }
@Override public void close() { closed = true; flushSequence(); buffer.close(); }
@Override public long getBufferedBytes() { return dataStream.getBufferedBytes() + presentStream.getBufferedBytes(); }
@Override public List<StreamDataOutput> getDataStreams() { checkState(closed); ImmutableList.Builder<StreamDataOutput> outputDataStreams = ImmutableList.builder(); presentStream.getStreamDataOutput(column).ifPresent(outputDataStreams::add); outputDataStreams.add(dataStream.getStreamDataOutput(column)); return outputDataStreams.build(); }
@Override public List<StreamDataOutput> getIndexStreams(CompressedMetadataWriter metadataWriter) throws IOException { checkState(closed); ImmutableList.Builder<RowGroupIndex> rowGroupIndexes = ImmutableList.builder(); List<ByteStreamCheckpoint> dataCheckpoints = dataStream.getCheckpoints(); Optional<List<BooleanStreamCheckpoint>> presentCheckpoints = presentStream.getCheckpoints(); for (int i = 0; i < rowGroupColumnStatistics.size(); i++) { int groupId = i; ColumnStatistics columnStatistics = rowGroupColumnStatistics.get(groupId); ByteStreamCheckpoint dataCheckpoint = dataCheckpoints.get(groupId); Optional<BooleanStreamCheckpoint> presentCheckpoint = presentCheckpoints.map(checkpoints -> checkpoints.get(groupId)); List<Integer> positions = createByteColumnPositionList(compressed, dataCheckpoint, presentCheckpoint); rowGroupIndexes.add(new RowGroupIndex(positions, columnStatistics)); } Slice slice = metadataWriter.writeRowIndexes(rowGroupIndexes.build()); Stream stream = new Stream(column, StreamKind.ROW_INDEX, slice.length(), false); return ImmutableList.of(new StreamDataOutput(slice, stream)); }
public BooleanOutputStream(OrcOutputBuffer buffer) { this(new ByteOutputStream(buffer)); }
@Override protected void writeValue(ByteOutputStream outputStream, Byte value) { outputStream.writeByte(value); }
@Override public long getRetainedBytes() { // NOTE: we do not include checkpoints because they should be small and it would be annoying to calculate the size return INSTANCE_SIZE + byteOutputStream.getRetainedBytes(); }