@Override public int writeStripeFooter(SliceOutput output, StripeFooter footer) throws IOException { OrcProto.StripeFooter footerProtobuf = OrcProto.StripeFooter.newBuilder() .addAllStreams(footer.getStreams().stream() .map(OrcMetadataWriter::toStream) .collect(toList())) .addAllColumns(footer.getColumnEncodings().stream() .map(OrcMetadataWriter::toColumnEncoding) .collect(toList())) .build(); return writeProtobufObject(output, footerProtobuf); }
StripeFooter stripeFooter = new StripeFooter(allStreams, toDenseList(columnEncodings, orcTypes.size())); Slice footer = metadataWriter.writeStripeFooter(stripeFooter); outputData.add(createDataOutput(footer));
for (Stream stream : stripeFooter.getStreams()) { if (isIndexStream(stream)) { assertFalse(dataStreamStarted);
for (Stream stream : stripeFooter.getStreams()) { if (isIndexStream(stream)) { assertFalse(dataStreamStarted);
@Override public int writeStripeFooter(SliceOutput output, StripeFooter footer) throws IOException { DwrfProto.StripeFooter footerProtobuf = DwrfProto.StripeFooter.newBuilder() .addAllStreams(footer.getStreams().stream() .map(DwrfMetadataWriter::toStream) .collect(toImmutableList())) .addAllColumns(footer.getColumnEncodings().stream() .map(DwrfMetadataWriter::toColumnEncoding) .collect(toImmutableList())) .build(); return writeProtobufObject(output, footerProtobuf); }
@Override public StripeFooter readStripeFooter(List<OrcType> types, InputStream inputStream) throws IOException { CodedInputStream input = CodedInputStream.newInstance(inputStream); OrcProto.StripeFooter stripeFooter = OrcProto.StripeFooter.parseFrom(input); return new StripeFooter(toStream(stripeFooter.getStreamsList()), toColumnEncoding(stripeFooter.getColumnsList())); }
List<ColumnEncoding> columnEncodings = stripeFooter.getColumnEncodings(); for (Stream stream : stripeFooter.getStreams()) { if (includedOrcColumns.contains(stream.getColumn())) { streams.put(new StreamId(stream), stream); if ((stripe.getNumberOfRows() > rowsInRowGroup) || hasRowGroupDictionary) { Map<StreamId, DiskRange> diskRanges = getDiskRanges(stripeFooter.getStreams()); diskRanges = Maps.filterKeys(diskRanges, Predicates.in(streams.keySet())); for (Entry<StreamId, DiskRange> entry : getDiskRanges(stripeFooter.getStreams()).entrySet()) { StreamId streamId = entry.getKey(); if (streams.keySet().contains(streamId)) {
@Override public StripeFooter readStripeFooter(List<OrcType> types, InputStream inputStream) throws IOException { CodedInputStream input = CodedInputStream.newInstance(inputStream); DwrfProto.StripeFooter stripeFooter = DwrfProto.StripeFooter.parseFrom(input); return new StripeFooter(toStream(stripeFooter.getStreamsList()), toColumnEncoding(types, stripeFooter.getColumnsList())); }
@Override public int writeStripeFooter(SliceOutput output, StripeFooter footer) throws IOException { OrcProto.StripeFooter footerProtobuf = OrcProto.StripeFooter.newBuilder() .addAllStreams(footer.getStreams().stream() .map(OrcMetadataWriter::toStream) .collect(toList())) .addAllColumns(footer.getColumnEncodings().stream() .map(OrcMetadataWriter::toColumnEncoding) .collect(toList())) .build(); return writeProtobufObject(output, footerProtobuf); }
@Override public StripeFooter readStripeFooter(List<OrcType> types, InputStream inputStream) throws IOException { CodedInputStream input = CodedInputStream.newInstance(inputStream); DwrfProto.StripeFooter stripeFooter = DwrfProto.StripeFooter.parseFrom(input); return new StripeFooter(toStream(stripeFooter.getStreamsList()), toColumnEncoding(types, stripeFooter.getColumnsList())); }
@Override public int writeStripeFooter(SliceOutput output, StripeFooter footer) throws IOException { DwrfProto.StripeFooter footerProtobuf = DwrfProto.StripeFooter.newBuilder() .addAllStreams(footer.getStreams().stream() .map(DwrfMetadataWriter::toStream) .collect(toImmutableList())) .addAllColumns(footer.getColumnEncodings().stream() .map(DwrfMetadataWriter::toColumnEncoding) .collect(toImmutableList())) .build(); return writeProtobufObject(output, footerProtobuf); }
@Override public StripeFooter readStripeFooter(List<OrcType> types, InputStream inputStream) throws IOException { CodedInputStream input = CodedInputStream.newInstance(inputStream); OrcProto.StripeFooter stripeFooter = OrcProto.StripeFooter.parseFrom(input); return new StripeFooter(toStream(stripeFooter.getStreamsList()), toColumnEncoding(stripeFooter.getColumnsList())); }
List<ColumnEncoding> columnEncodings = stripeFooter.getColumnEncodings(); for (Stream stream : stripeFooter.getStreams()) { if (includedOrcColumns.contains(stream.getColumn())) { streams.put(new StreamId(stream), stream); Map<StreamId, DiskRange> diskRanges = getDiskRanges(stripeFooter.getStreams()); diskRanges = Maps.filterKeys(diskRanges, Predicates.in(streams.keySet())); for (Entry<StreamId, DiskRange> entry : getDiskRanges(stripeFooter.getStreams()).entrySet()) { StreamId streamId = entry.getKey(); if (streamId.getStreamKind() != ROW_INDEX && streams.keySet().contains(streamId)) {
StripeFooter stripeFooter = new StripeFooter(allStreams, toDenseList(columnEncodings, orcTypes.size())); Slice footer = metadataWriter.writeStripeFooter(stripeFooter); outputData.add(createDataOutput(footer));
List<ColumnEncoding> columnEncodings = stripeFooter.getColumnEncodings(); for (Stream stream : stripeFooter.getStreams()) { if (includedOrcColumns.contains(stream.getColumn())) { streams.put(new StreamId(stream), stream); if ((stripe.getNumberOfRows() > rowsInRowGroup) || hasRowGroupDictionary) { Map<StreamId, DiskRange> diskRanges = getDiskRanges(stripeFooter.getStreams()); diskRanges = Maps.filterKeys(diskRanges, Predicates.in(streams.keySet())); for (Entry<StreamId, DiskRange> entry : getDiskRanges(stripeFooter.getStreams()).entrySet()) { StreamId streamId = entry.getKey(); if (streams.keySet().contains(streamId)) {
@Override public StripeFooter readStripeFooter(List<OrcType> types, InputStream inputStream) throws IOException { CodedInputStream input = CodedInputStream.newInstance(inputStream); OrcProto.StripeFooter stripeFooter = OrcProto.StripeFooter.parseFrom(input); return new StripeFooter(toStream(stripeFooter.getStreamsList()), toColumnEncoding(stripeFooter.getColumnsList())); }
@Override public StripeFooter readStripeFooter(List<OrcType> types, InputStream inputStream) throws IOException { CodedInputStream input = CodedInputStream.newInstance(inputStream); OrcProto.StripeFooter stripeFooter = OrcProto.StripeFooter.parseFrom(input); return new StripeFooter(toStream(stripeFooter.getStreamsList()), toColumnEncoding(types, stripeFooter.getColumnsList())); }