@Override public int writeFooter(SliceOutput output, Footer footer) throws IOException { OrcProto.Footer footerProtobuf = OrcProto.Footer.newBuilder() .setWriter(PRESTO_WRITER_ID) .setNumberOfRows(footer.getNumberOfRows()) .setRowIndexStride(footer.getRowsInRowGroup()) .addAllStripes(footer.getStripes().stream() .map(OrcMetadataWriter::toStripeInformation) .collect(toList())) .addAllTypes(footer.getTypes().stream() .map(OrcMetadataWriter::toType) .collect(toList())) .addAllStatistics(footer.getFileStats().stream() .map(OrcMetadataWriter::toColumnStatistics) .collect(toList())) .addAllMetadata(footer.getUserMetadata().entrySet().stream() .map(OrcMetadataWriter::toUserMetadata) .collect(toList())) .build(); return writeProtobufObject(output, footerProtobuf); }
@Override public int writeFooter(SliceOutput output, Footer footer) throws IOException { DwrfProto.Footer footerProtobuf = DwrfProto.Footer.newBuilder() .setNumberOfRows(footer.getNumberOfRows()) .setRowIndexStride(footer.getRowsInRowGroup()) .addAllStripes(footer.getStripes().stream() .map(DwrfMetadataWriter::toStripeInformation) .collect(toImmutableList())) .addAllTypes(footer.getTypes().stream() .map(DwrfMetadataWriter::toType) .collect(toImmutableList())) .addAllStatistics(footer.getFileStats().stream() .map(DwrfMetadataWriter::toColumnStatistics) .collect(toImmutableList())) .addAllMetadata(footer.getUserMetadata().entrySet().stream() .map(DwrfMetadataWriter::toUserMetadata) .collect(toImmutableList())) .addAllMetadata(STATIC_METADATA.entrySet().stream() .map(DwrfMetadataWriter::toUserMetadata) .collect(toImmutableList())) .build(); return writeProtobufObject(output, footerProtobuf); }
tinyStripeThreshold, maxBlockSize, footer.getUserMetadata(), systemMemoryUsage, writeValidation,
validateWrite(validation -> validation.getRowGroupMaxRowCount() == footer.getRowsInRowGroup(), "Unexpected rows in group"); if (writeValidation.isPresent()) { writeValidation.get().validateMetadata(orcDataSource.getId(), footer.getUserMetadata()); writeValidation.get().validateFileStatistics(orcDataSource.getId(), footer.getFileStats()); writeValidation.get().validateStripeStatistics(orcDataSource.getId(), footer.getStripes(), metadata.getStripeStatsList());
@Test public void testReadUserMetadata() throws Exception { try (TempFile tempFile = new TempFile()) { Map<String, String> metadata = ImmutableMap.of( "a", "ala", "b", "ma", "c", "kota"); createFileWithOnlyUserMetadata(tempFile.getFile(), metadata); OrcDataSource orcDataSource = new FileOrcDataSource(tempFile.getFile(), new DataSize(1, MEGABYTE), new DataSize(1, MEGABYTE), new DataSize(1, MEGABYTE), true); OrcReader orcReader = new OrcReader(orcDataSource, ORC, new DataSize(1, MEGABYTE), new DataSize(1, MEGABYTE), new DataSize(1, MEGABYTE), new DataSize(1, MEGABYTE)); Footer footer = orcReader.getFooter(); Map<String, String> readMetadata = Maps.transformValues(footer.getUserMetadata(), Slice::toStringAscii); assertEquals(readMetadata, metadata); } }
private static Optional<OrcFileMetadata> getOrcFileMetadata(OrcReader reader) { return Optional.ofNullable(reader.getFooter().getUserMetadata().get(OrcFileMetadata.KEY)) .map(slice -> METADATA_CODEC.fromJson(slice.getBytes())); }
@Override public int writeFooter(SliceOutput output, Footer footer) throws IOException { OrcProto.Footer footerProtobuf = OrcProto.Footer.newBuilder() .setWriter(PRESTO_WRITER_ID) .setNumberOfRows(footer.getNumberOfRows()) .setRowIndexStride(footer.getRowsInRowGroup()) .addAllStripes(footer.getStripes().stream() .map(OrcMetadataWriter::toStripeInformation) .collect(toList())) .addAllTypes(footer.getTypes().stream() .map(OrcMetadataWriter::toType) .collect(toList())) .addAllStatistics(footer.getFileStats().stream() .map(OrcMetadataWriter::toColumnStatistics) .collect(toList())) .addAllMetadata(footer.getUserMetadata().entrySet().stream() .map(OrcMetadataWriter::toUserMetadata) .collect(toList())) .build(); return writeProtobufObject(output, footerProtobuf); }
@Override public int writeFooter(SliceOutput output, Footer footer) throws IOException { DwrfProto.Footer footerProtobuf = DwrfProto.Footer.newBuilder() .setNumberOfRows(footer.getNumberOfRows()) .setRowIndexStride(footer.getRowsInRowGroup()) .addAllStripes(footer.getStripes().stream() .map(DwrfMetadataWriter::toStripeInformation) .collect(toImmutableList())) .addAllTypes(footer.getTypes().stream() .map(DwrfMetadataWriter::toType) .collect(toImmutableList())) .addAllStatistics(footer.getFileStats().stream() .map(DwrfMetadataWriter::toColumnStatistics) .collect(toImmutableList())) .addAllMetadata(footer.getUserMetadata().entrySet().stream() .map(DwrfMetadataWriter::toUserMetadata) .collect(toImmutableList())) .addAllMetadata(STATIC_METADATA.entrySet().stream() .map(DwrfMetadataWriter::toUserMetadata) .collect(toImmutableList())) .build(); return writeProtobufObject(output, footerProtobuf); }
tinyStripeThreshold, maxBlockSize, footer.getUserMetadata(), systemMemoryUsage, writeValidation,
validateWrite(validation -> validation.getRowGroupMaxRowCount() == footer.getRowsInRowGroup(), "Unexpected rows in group"); if (writeValidation.isPresent()) { writeValidation.get().validateMetadata(orcDataSource.getId(), footer.getUserMetadata()); writeValidation.get().validateFileStatistics(orcDataSource.getId(), footer.getFileStats()); writeValidation.get().validateStripeStatistics(orcDataSource.getId(), footer.getStripes(), metadata.getStripeStatsList());
@Test public void testReadUserMetadata() throws Exception { try (TempFile tempFile = new TempFile()) { Map<String, String> metadata = ImmutableMap.of( "a", "ala", "b", "ma", "c", "kota"); createFileWithOnlyUserMetadata(tempFile.getFile(), metadata); OrcDataSource orcDataSource = new FileOrcDataSource(tempFile.getFile(), new DataSize(1, MEGABYTE), new DataSize(1, MEGABYTE), new DataSize(1, MEGABYTE), true); OrcReader orcReader = new OrcReader(orcDataSource, ORC, new DataSize(1, MEGABYTE), new DataSize(1, MEGABYTE), new DataSize(1, MEGABYTE), new DataSize(1, MEGABYTE)); Footer footer = orcReader.getFooter(); Map<String, String> readMetadata = Maps.transformValues(footer.getUserMetadata(), Slice::toStringAscii); assertEquals(readMetadata, metadata); } }