@Override public MetadataWriter createMetadataWriter() { return new OrcMetadataWriter(); } },
private static OrcProto.ColumnEncoding toColumnEncoding(ColumnEncoding columnEncodings) { checkArgument( !columnEncodings.getAdditionalSequenceEncodings().isPresent(), "Writing columns with non-zero sequence IDs is not supported in ORC: " + columnEncodings); return OrcProto.ColumnEncoding.newBuilder() .setKind(toColumnEncoding(columnEncodings.getColumnEncodingKind())) .setDictionarySize(columnEncodings.getDictionarySize()) .build(); }
private static RowIndexEntry toRowGroupIndex(RowGroupIndex rowGroupIndex) { return OrcProto.RowIndexEntry.newBuilder() .addAllPositions(rowGroupIndex.getPositions().stream() .map(Integer::longValue) .collect(toList())) .setStatistics(toColumnStatistics(rowGroupIndex.getColumnStatistics())) .build(); }
@Override public int writePostscript(SliceOutput output, int footerLength, int metadataLength, CompressionKind compression, int compressionBlockSize) throws IOException { OrcProto.PostScript postScriptProtobuf = OrcProto.PostScript.newBuilder() .addAllVersion(ORC_METADATA_VERSION) .setFooterLength(footerLength) .setMetadataLength(metadataLength) .setCompression(toCompression(compression)) .setCompressionBlockSize(compressionBlockSize) .setWriterVersion(ORC_WRITER_VERSION) .build(); return writeProtobufObject(output, postScriptProtobuf); }
@Override public int writeRowIndexes(SliceOutput output, List<RowGroupIndex> rowGroupIndexes) throws IOException { OrcProto.RowIndex rowIndexProtobuf = OrcProto.RowIndex.newBuilder() .addAllEntry(rowGroupIndexes.stream() .map(OrcMetadataWriter::toRowGroupIndex) .collect(toList())) .build(); return writeProtobufObject(output, rowIndexProtobuf); }
private static OrcProto.Stream toStream(Stream stream) { return OrcProto.Stream.newBuilder() .setColumn(stream.getColumn()) .setKind(toStreamKind(stream.getStreamKind())) .setLength(stream.getLength()) .build(); }
private static Type toType(OrcType type) { Builder builder = Type.newBuilder() .setKind(toTypeKind(type.getOrcTypeKind())) .addAllSubtypes(type.getFieldTypeIndexes()) .addAllFieldNames(type.getFieldNames()); if (type.getLength().isPresent()) { builder.setMaximumLength(type.getLength().get()); } if (type.getPrecision().isPresent()) { builder.setPrecision(type.getPrecision().get()); } if (type.getScale().isPresent()) { builder.setScale(type.getScale().get()); } return builder.build(); }
@Override public int writeMetadata(SliceOutput output, Metadata metadata) throws IOException { OrcProto.Metadata metadataProtobuf = OrcProto.Metadata.newBuilder() .addAllStripeStats(metadata.getStripeStatsList().stream() .map(OrcMetadataWriter::toStripeStatistics) .collect(toList())) .build(); return writeProtobufObject(output, metadataProtobuf); }
@Override public int writePostscript(SliceOutput output, int footerLength, int metadataLength, CompressionKind compression, int compressionBlockSize) throws IOException { OrcProto.PostScript postScriptProtobuf = OrcProto.PostScript.newBuilder() .addAllVersion(ORC_METADATA_VERSION) .setFooterLength(footerLength) .setMetadataLength(metadataLength) .setCompression(toCompression(compression)) .setCompressionBlockSize(compressionBlockSize) .setWriterVersion(ORC_WRITER_VERSION) .build(); return writeProtobufObject(output, postScriptProtobuf); }
private static OrcProto.Stream toStream(Stream stream) { return OrcProto.Stream.newBuilder() .setColumn(stream.getColumn()) .setKind(toStreamKind(stream.getStreamKind())) .setLength(stream.getLength()) .build(); }
private static Type toType(OrcType type) { Builder builder = Type.newBuilder() .setKind(toTypeKind(type.getOrcTypeKind())) .addAllSubtypes(type.getFieldTypeIndexes()) .addAllFieldNames(type.getFieldNames()); if (type.getLength().isPresent()) { builder.setMaximumLength(type.getLength().get()); } if (type.getPrecision().isPresent()) { builder.setPrecision(type.getPrecision().get()); } if (type.getScale().isPresent()) { builder.setScale(type.getScale().get()); } return builder.build(); }
@Override public int writeStripeFooter(SliceOutput output, StripeFooter footer) throws IOException { OrcProto.StripeFooter footerProtobuf = OrcProto.StripeFooter.newBuilder() .addAllStreams(footer.getStreams().stream() .map(OrcMetadataWriter::toStream) .collect(toList())) .addAllColumns(footer.getColumnEncodings().stream() .map(OrcMetadataWriter::toColumnEncoding) .collect(toList())) .build(); return writeProtobufObject(output, footerProtobuf); }
private static OrcProto.ColumnEncoding toColumnEncoding(ColumnEncoding columnEncodings) { checkArgument( !columnEncodings.getAdditionalSequenceEncodings().isPresent(), "Writing columns with non-zero sequence IDs is not supported in ORC: " + columnEncodings); return OrcProto.ColumnEncoding.newBuilder() .setKind(toColumnEncoding(columnEncodings.getColumnEncodingKind())) .setDictionarySize(columnEncodings.getDictionarySize()) .build(); }
private static RowIndexEntry toRowGroupIndex(RowGroupIndex rowGroupIndex) { return OrcProto.RowIndexEntry.newBuilder() .addAllPositions(rowGroupIndex.getPositions().stream() .map(Integer::longValue) .collect(toList())) .setStatistics(toColumnStatistics(rowGroupIndex.getColumnStatistics())) .build(); }
@Override public MetadataWriter createMetadataWriter() { return new OrcMetadataWriter(); } },
@Override public int writeFooter(SliceOutput output, Footer footer) throws IOException { OrcProto.Footer footerProtobuf = OrcProto.Footer.newBuilder() .setWriter(PRESTO_WRITER_ID) .setNumberOfRows(footer.getNumberOfRows()) .setRowIndexStride(footer.getRowsInRowGroup()) .addAllStripes(footer.getStripes().stream() .map(OrcMetadataWriter::toStripeInformation) .collect(toList())) .addAllTypes(footer.getTypes().stream() .map(OrcMetadataWriter::toType) .collect(toList())) .addAllStatistics(footer.getFileStats().stream() .map(OrcMetadataWriter::toColumnStatistics) .collect(toList())) .addAllMetadata(footer.getUserMetadata().entrySet().stream() .map(OrcMetadataWriter::toUserMetadata) .collect(toList())) .build(); return writeProtobufObject(output, footerProtobuf); }
@Override public int writeRowIndexes(SliceOutput output, List<RowGroupIndex> rowGroupIndexes) throws IOException { OrcProto.RowIndex rowIndexProtobuf = OrcProto.RowIndex.newBuilder() .addAllEntry(rowGroupIndexes.stream() .map(OrcMetadataWriter::toRowGroupIndex) .collect(toList())) .build(); return writeProtobufObject(output, rowIndexProtobuf); }
@Override public int writeMetadata(SliceOutput output, Metadata metadata) throws IOException { OrcProto.Metadata metadataProtobuf = OrcProto.Metadata.newBuilder() .addAllStripeStats(metadata.getStripeStatsList().stream() .map(OrcMetadataWriter::toStripeStatistics) .collect(toList())) .build(); return writeProtobufObject(output, metadataProtobuf); }
@Override public int writeStripeFooter(SliceOutput output, StripeFooter footer) throws IOException { OrcProto.StripeFooter footerProtobuf = OrcProto.StripeFooter.newBuilder() .addAllStreams(footer.getStreams().stream() .map(OrcMetadataWriter::toStream) .collect(toList())) .addAllColumns(footer.getColumnEncodings().stream() .map(OrcMetadataWriter::toColumnEncoding) .collect(toList())) .build(); return writeProtobufObject(output, footerProtobuf); }
@Override public int writeFooter(SliceOutput output, Footer footer) throws IOException { OrcProto.Footer footerProtobuf = OrcProto.Footer.newBuilder() .setWriter(PRESTO_WRITER_ID) .setNumberOfRows(footer.getNumberOfRows()) .setRowIndexStride(footer.getRowsInRowGroup()) .addAllStripes(footer.getStripes().stream() .map(OrcMetadataWriter::toStripeInformation) .collect(toList())) .addAllTypes(footer.getTypes().stream() .map(OrcMetadataWriter::toType) .collect(toList())) .addAllStatistics(footer.getFileStats().stream() .map(OrcMetadataWriter::toColumnStatistics) .collect(toList())) .addAllMetadata(footer.getUserMetadata().entrySet().stream() .map(OrcMetadataWriter::toUserMetadata) .collect(toList())) .build(); return writeProtobufObject(output, footerProtobuf); }