public StructColumnWriter(int column, CompressionKind compression, int bufferSize, List<ColumnWriter> structFields) { checkArgument(column >= 0, "column is negative"); this.column = column; this.compressed = requireNonNull(compression, "compression is null") != NONE; this.structFields = ImmutableList.copyOf(requireNonNull(structFields, "structFields is null")); this.presentStream = new PresentOutputStream(compression, bufferSize); }
@Override public void close() { closed = true; structFields.forEach(ColumnWriter::close); presentStream.close(); }
public void writeBoolean(boolean value) { checkArgument(!closed); if (!value && booleanOutputStream == null) { createBooleanOutputStream(); } if (booleanOutputStream != null) { booleanOutputStream.writeBoolean(value); } currentGroupCount++; }
@Override public long getRetainedBytes() { long retainedBytes = INSTANCE_SIZE + dataStream.getRetainedBytes() + presentStream.getRetainedBytes(); for (ColumnStatistics statistics : rowGroupColumnStatistics) { retainedBytes += statistics.getRetainedSizeInBytes(); } return retainedBytes; }
@Override public void beginRowGroup() { presentStream.recordCheckpoint(); structFields.forEach(ColumnWriter::beginRowGroup); }
@Override public List<StreamDataOutput> getDataStreams() { checkState(closed); ImmutableList.Builder<StreamDataOutput> outputDataStreams = ImmutableList.builder(); presentStream.getStreamDataOutput(column).ifPresent(outputDataStreams::add); outputDataStreams.add(dataStream.getStreamDataOutput(column)); return outputDataStreams.build(); }
@Override public void reset() { closed = false; presentStream.reset(); structFields.forEach(ColumnWriter::reset); rowGroupColumnStatistics.clear(); nonNullValueCount = 0; } }
@Override public long getBufferedBytes() { return dataStream.getBufferedBytes() + presentStream.getBufferedBytes(); }
@Override public List<StreamDataOutput> getIndexStreams(CompressedMetadataWriter metadataWriter) throws IOException { checkState(closed); ImmutableList.Builder<RowGroupIndex> rowGroupIndexes = ImmutableList.builder(); List<LongStreamCheckpoint> dataCheckpoints = dataStream.getCheckpoints(); Optional<List<BooleanStreamCheckpoint>> presentCheckpoints = presentStream.getCheckpoints(); for (int i = 0; i < rowGroupColumnStatistics.size(); i++) { int groupId = i; ColumnStatistics columnStatistics = rowGroupColumnStatistics.get(groupId); LongStreamCheckpoint dataCheckpoint = dataCheckpoints.get(groupId); Optional<BooleanStreamCheckpoint> presentCheckpoint = presentCheckpoints.map(checkpoints -> checkpoints.get(groupId)); List<Integer> positions = createLongColumnPositionList(compressed, dataCheckpoint, presentCheckpoint); rowGroupIndexes.add(new RowGroupIndex(positions, columnStatistics)); } Slice slice = metadataWriter.writeRowIndexes(rowGroupIndexes.build()); Stream stream = new Stream(column, StreamKind.ROW_INDEX, slice.length(), false); return ImmutableList.of(new StreamDataOutput(slice, stream)); }
private void writeColumnarRow(ColumnarRow columnarRow) { // record nulls for (int position = 0; position < columnarRow.getPositionCount(); position++) { boolean present = !columnarRow.isNull(position); presentStream.writeBoolean(present); if (present) { nonNullValueCount++; } } // write field values for (int i = 0; i < structFields.size(); i++) { ColumnWriter columnWriter = structFields.get(i); Block fieldBlock = columnarRow.getField(i); if (fieldBlock.getPositionCount() > 0) { columnWriter.writeBlock(fieldBlock); } } }
@Override public long getRetainedBytes() { long retainedBytes = INSTANCE_SIZE + presentStream.getRetainedBytes(); for (ColumnWriter structField : structFields) { retainedBytes += structField.getRetainedBytes(); } for (ColumnStatistics statistics : rowGroupColumnStatistics) { retainedBytes += statistics.getRetainedSizeInBytes(); } return retainedBytes; }
@Override public void beginRowGroup() { presentStream.recordCheckpoint(); dataStream.recordCheckpoint(); }
@Override public List<StreamDataOutput> getDataStreams() { checkState(closed); ImmutableList.Builder<StreamDataOutput> outputDataStreams = ImmutableList.builder(); presentStream.getStreamDataOutput(column).ifPresent(outputDataStreams::add); outputDataStreams.add(dataStream.getStreamDataOutput(column)); return outputDataStreams.build(); }
@Override public void reset() { closed = false; dataStream.reset(); presentStream.reset(); rowGroupColumnStatistics.clear(); nonNullValueCount = 0; } }
@Override public long getBufferedBytes() { return dataStream.getBufferedBytes() + presentStream.getBufferedBytes(); }
@Override public List<StreamDataOutput> getIndexStreams(CompressedMetadataWriter metadataWriter) throws IOException { checkState(closed); ImmutableList.Builder<RowGroupIndex> rowGroupIndexes = ImmutableList.builder(); Optional<List<BooleanStreamCheckpoint>> presentCheckpoints = presentStream.getCheckpoints(); for (int i = 0; i < rowGroupColumnStatistics.size(); i++) { int groupId = i; ColumnStatistics columnStatistics = rowGroupColumnStatistics.get(groupId); Optional<BooleanStreamCheckpoint> presentCheckpoint = presentCheckpoints.map(checkpoints -> checkpoints.get(groupId)); List<Integer> positions = createStructColumnPositionList(compressed, presentCheckpoint); rowGroupIndexes.add(new RowGroupIndex(positions, columnStatistics)); } Slice slice = metadataWriter.writeRowIndexes(rowGroupIndexes.build()); Stream stream = new Stream(column, StreamKind.ROW_INDEX, slice.length(), false); ImmutableList.Builder<StreamDataOutput> indexStreams = ImmutableList.builder(); indexStreams.add(new StreamDataOutput(slice, stream)); for (ColumnWriter structField : structFields) { indexStreams.addAll(structField.getIndexStreams(metadataWriter)); } return indexStreams.build(); }
@Override public void writeBlock(Block block) { checkState(!closed); checkArgument(block.getPositionCount() > 0, "Block is empty"); // record nulls for (int position = 0; position < block.getPositionCount(); position++) { presentStream.writeBoolean(!block.isNull(position)); } // record values for (int position = 0; position < block.getPositionCount(); position++) { if (!block.isNull(position)) { dataStream.writeByte((byte) type.getLong(block, position)); nonNullValueCount++; } } }
@Override public long getRetainedBytes() { long retainedBytes = INSTANCE_SIZE + dataStream.getRetainedBytes() + presentStream.getRetainedBytes(); for (ColumnStatistics statistics : rowGroupColumnStatistics) { retainedBytes += statistics.getRetainedSizeInBytes(); } return retainedBytes; }
public FloatColumnWriter(int column, Type type, CompressionKind compression, int bufferSize) { checkArgument(column >= 0, "column is negative"); this.column = column; this.type = requireNonNull(type, "type is null"); this.compressed = requireNonNull(compression, "compression is null") != NONE; this.dataStream = new FloatOutputStream(compression, bufferSize); this.presentStream = new PresentOutputStream(compression, bufferSize); }
@Override public void beginRowGroup() { presentStream.recordCheckpoint(); dataStream.recordCheckpoint(); }