@Override protected DecimalOutputStream createValueOutputStream() { return new DecimalOutputStream(SNAPPY, COMPRESSION_BLOCK_SIZE); }
@Override protected void writeValue(DecimalOutputStream outputStream, Slice value) { outputStream.writeUnscaledValue(value); }
@Override public void close() { closed = true; dataStream.close(); scaleStream.close(); presentStream.close(); }
@Override public long getRetainedBytes() { long retainedBytes = INSTANCE_SIZE + dataStream.getRetainedBytes() + scaleStream.getRetainedBytes() + presentStream.getRetainedBytes(); for (ColumnStatistics statistics : rowGroupColumnStatistics) { retainedBytes += statistics.getRetainedSizeInBytes(); } return retainedBytes; }
@Override public void reset() { closed = false; dataStream.reset(); scaleStream.reset(); presentStream.reset(); rowGroupColumnStatistics.clear(); shortDecimalStatisticsBuilder = new ShortDecimalStatisticsBuilder(this.type.getScale()); longDecimalStatisticsBuilder = new LongDecimalStatisticsBuilder(); } }
@Override public void beginRowGroup() { presentStream.recordCheckpoint(); dataStream.recordCheckpoint(); scaleStream.recordCheckpoint(); }
@Override public List<StreamDataOutput> getDataStreams() { checkState(closed); ImmutableList.Builder<StreamDataOutput> outputDataStreams = ImmutableList.builder(); presentStream.getStreamDataOutput(column).ifPresent(outputDataStreams::add); outputDataStreams.add(dataStream.getStreamDataOutput(column)); outputDataStreams.add(scaleStream.getStreamDataOutput(column)); return outputDataStreams.build(); }
@Override public long getBufferedBytes() { return dataStream.getBufferedBytes() + scaleStream.getBufferedBytes() + presentStream.getBufferedBytes(); }
@Override public List<StreamDataOutput> getIndexStreams(CompressedMetadataWriter metadataWriter) throws IOException { checkState(closed); ImmutableList.Builder<RowGroupIndex> rowGroupIndexes = ImmutableList.builder(); List<DecimalStreamCheckpoint> dataCheckpoints = dataStream.getCheckpoints(); List<LongStreamCheckpoint> scaleCheckpoints = scaleStream.getCheckpoints(); Optional<List<BooleanStreamCheckpoint>> presentCheckpoints = presentStream.getCheckpoints(); for (int i = 0; i < rowGroupColumnStatistics.size(); i++) { int groupId = i; ColumnStatistics columnStatistics = rowGroupColumnStatistics.get(groupId); DecimalStreamCheckpoint dataCheckpoint = dataCheckpoints.get(groupId); LongStreamCheckpoint scaleCheckpoint = scaleCheckpoints.get(groupId); Optional<BooleanStreamCheckpoint> presentCheckpoint = presentCheckpoints.map(checkpoints -> checkpoints.get(groupId)); List<Integer> positions = createDecimalColumnPositionList(compressed, dataCheckpoint, scaleCheckpoint, presentCheckpoint); rowGroupIndexes.add(new RowGroupIndex(positions, columnStatistics)); } Slice slice = metadataWriter.writeRowIndexes(rowGroupIndexes.build()); Stream stream = new Stream(column, StreamKind.ROW_INDEX, slice.length(), false); return ImmutableList.of(new StreamDataOutput(slice, stream)); }
@Override public long getRetainedBytes() { long retainedBytes = INSTANCE_SIZE + dataStream.getRetainedBytes() + scaleStream.getRetainedBytes() + presentStream.getRetainedBytes(); for (ColumnStatistics statistics : rowGroupColumnStatistics) { retainedBytes += statistics.getRetainedSizeInBytes(); } return retainedBytes; }
@Override public void reset() { closed = false; dataStream.reset(); scaleStream.reset(); presentStream.reset(); rowGroupColumnStatistics.clear(); shortDecimalStatisticsBuilder = new ShortDecimalStatisticsBuilder(this.type.getScale()); longDecimalStatisticsBuilder = new LongDecimalStatisticsBuilder(); } }
@Override public void beginRowGroup() { presentStream.recordCheckpoint(); dataStream.recordCheckpoint(); scaleStream.recordCheckpoint(); }
@Override public List<StreamDataOutput> getDataStreams() { checkState(closed); ImmutableList.Builder<StreamDataOutput> outputDataStreams = ImmutableList.builder(); presentStream.getStreamDataOutput(column).ifPresent(outputDataStreams::add); outputDataStreams.add(dataStream.getStreamDataOutput(column)); outputDataStreams.add(scaleStream.getStreamDataOutput(column)); return outputDataStreams.build(); }
@Override public long getBufferedBytes() { return dataStream.getBufferedBytes() + scaleStream.getBufferedBytes() + presentStream.getBufferedBytes(); }
@Override public List<StreamDataOutput> getIndexStreams(CompressedMetadataWriter metadataWriter) throws IOException { checkState(closed); ImmutableList.Builder<RowGroupIndex> rowGroupIndexes = ImmutableList.builder(); List<DecimalStreamCheckpoint> dataCheckpoints = dataStream.getCheckpoints(); List<LongStreamCheckpoint> scaleCheckpoints = scaleStream.getCheckpoints(); Optional<List<BooleanStreamCheckpoint>> presentCheckpoints = presentStream.getCheckpoints(); for (int i = 0; i < rowGroupColumnStatistics.size(); i++) { int groupId = i; ColumnStatistics columnStatistics = rowGroupColumnStatistics.get(groupId); DecimalStreamCheckpoint dataCheckpoint = dataCheckpoints.get(groupId); LongStreamCheckpoint scaleCheckpoint = scaleCheckpoints.get(groupId); Optional<BooleanStreamCheckpoint> presentCheckpoint = presentCheckpoints.map(checkpoints -> checkpoints.get(groupId)); List<Integer> positions = createDecimalColumnPositionList(compressed, dataCheckpoint, scaleCheckpoint, presentCheckpoint); rowGroupIndexes.add(new RowGroupIndex(positions, columnStatistics)); } Slice slice = metadataWriter.writeRowIndexes(rowGroupIndexes.build()); Stream stream = new Stream(column, StreamKind.ROW_INDEX, slice.length(), false); return ImmutableList.of(new StreamDataOutput(slice, stream)); }
@Override protected void writeValue(DecimalOutputStream outputStream, Long value) { outputStream.writeUnscaledValue(Decimals.encodeUnscaledValue(value)); }
@Override protected DecimalOutputStream createValueOutputStream() { return new DecimalOutputStream(SNAPPY, COMPRESSION_BLOCK_SIZE); }
@Override public void close() { closed = true; dataStream.close(); scaleStream.close(); presentStream.close(); }
if (!block.isNull(position)) { long value = type.getLong(block, position); dataStream.writeUnscaledValue(value); shortDecimalStatisticsBuilder.addValue(value); if (!block.isNull(position)) { Slice value = type.getSlice(block, position); dataStream.writeUnscaledValue(value);
public DecimalColumnWriter(int column, Type type, CompressionKind compression, int bufferSize, OrcEncoding orcEncoding) { checkArgument(column >= 0, "column is negative"); checkArgument(orcEncoding != DWRF, "DWRF does not support %s type", type); this.column = column; this.type = (DecimalType) requireNonNull(type, "type is null"); this.compressed = requireNonNull(compression, "compression is null") != NONE; this.columnEncoding = new ColumnEncoding(DIRECT_V2, 0); this.dataStream = new DecimalOutputStream(compression, bufferSize); this.scaleStream = new LongOutputStreamV2(compression, bufferSize, true, SECONDARY); this.presentStream = new PresentOutputStream(compression, bufferSize); if (this.type.isShort()) { shortDecimalStatisticsBuilder = new ShortDecimalStatisticsBuilder(this.type.getScale()); } else { longDecimalStatisticsBuilder = new LongDecimalStatisticsBuilder(); } }