/** * Reads the metadata for all stripes in the file. */ private ArrayList<OrcStripeMetadata> readStripesMetadata( boolean[] includes, boolean[] sargColumns) throws IOException { ArrayList<OrcStripeMetadata> result = new ArrayList<OrcStripeMetadata>(stripeRgs.length); boolean hasFileId = this.fileKey != null; OrcBatchKey stripeKey = hasFileId ? new OrcBatchKey(fileKey, 0, 0) : null; for (int stripeIxMod = 0; stripeIxMod < stripeRgs.length; ++stripeIxMod) { int stripeIx = stripeIxMod + stripeIxFrom; stripeKey.stripeIx = stripeIx; StripeInformation si = fileMetadata.getStripes().get(stripeIx); OrcProto.StripeFooter footer = getStripeFooterFromCacheOrDisk(si, stripeKey); OrcStripeMetadata osm = createOrcStripeMetadataObject( stripeIx, si, footer, includes, sargColumns); ensureDataReader(); OrcIndex index = osm.getIndex(); stripeReader.readIndexStreams(index, si, footer.getStreamsList(), includes, sargColumns); result.add(osm); consumer.setStripeMetadata(osm); } return result; }
ensureDataReader(); stripeReader.readIndexStreams(stripeMetadata.getIndex(), si, footer.getStreamsList(), fileIncludes, sargColumns); consumer.setStripeMetadata(stripeMetadata);
private void readPartialDataStreams(StripeInformation stripe) throws IOException { List<OrcProto.Stream> streamList = stripeFooter.getStreamsList(); DiskRangeList toRead = planReadPartialDataStreams(streamList, included, true); if (LOG.isDebugEnabled()) { LOG.debug("chunks = " + RecordReaderUtils.stringifyDiskRanges(toRead)); } bufferChunks = dataReader.readFileData(toRead, stripe.getOffset(), false); if (LOG.isDebugEnabled()) { LOG.debug("merge = " + RecordReaderUtils.stringifyDiskRanges(bufferChunks)); } createStreams(streamList, bufferChunks, included, codec, bufferSize, streams); }
ensureDataReader(); stripeReader.readIndexStreams(stripeMetadata.getIndex(), si, footer.getStreamsList(), fileIncludes, sargColumns); consumer.setStripeMetadata(stripeMetadata);
inputStream.readFully(offset + indexLength + dataLength, buffer); OrcProto.StripeFooter footer = OrcProto.StripeFooter.parseFrom(buffer); List<OrcProto.Stream> streams = footer.getStreamsList(); for (OrcProto.Stream stream : streams)
long offset = 0; DiskRangeList range = ranges; for(OrcProto.Stream stream: footer.getStreamsList()) {
OrcProto.Stream.Kind[] bloomFilterKinds) { DiskRangeList.CreateHelper result = new DiskRangeList.CreateHelper(); List<OrcProto.Stream> streams = footer.getStreamsList(); for(OrcProto.Stream stream: footer.getStreamsList()) { if (stream.hasKind() && stream.hasColumn()) { int column = stream.getColumn();
/** * Reads the metadata for all stripes in the file. */ private ArrayList<OrcStripeMetadata> readStripesMetadata( boolean[] includes, boolean[] sargColumns) throws IOException { ArrayList<OrcStripeMetadata> result = new ArrayList<OrcStripeMetadata>(stripeRgs.length); boolean hasFileId = this.fileKey != null; OrcBatchKey stripeKey = hasFileId ? new OrcBatchKey(fileKey, 0, 0) : null; for (int stripeIxMod = 0; stripeIxMod < stripeRgs.length; ++stripeIxMod) { int stripeIx = stripeIxMod + stripeIxFrom; stripeKey.stripeIx = stripeIx; StripeInformation si = fileMetadata.getStripes().get(stripeIx); OrcProto.StripeFooter footer = getStripeFooterFromCacheOrDisk(si, stripeKey); OrcStripeMetadata osm = createOrcStripeMetadataObject( stripeIx, si, footer, includes, sargColumns); ensureDataReader(); OrcIndex index = osm.getIndex(); stripeReader.readIndexStreams(index, si, footer.getStreamsList(), includes, sargColumns); result.add(osm); consumer.setStripeMetadata(osm); } return result; }
private void readPartialDataStreams(StripeInformation stripe) throws IOException { List<OrcProto.Stream> streamList = stripeFooter.getStreamsList(); DiskRangeList toRead = planReadPartialDataStreams(streamList, indexes, fileIncluded, includedRowGroups, dataReader.getCompressionCodec() != null, stripeFooter.getColumnsList(), types, bufferSize, true); if (LOG.isDebugEnabled()) { LOG.debug("chunks = " + RecordReaderUtils.stringifyDiskRanges(toRead)); } bufferChunks = dataReader.readFileData(toRead, stripe.getOffset(), false); if (LOG.isDebugEnabled()) { LOG.debug("merge = " + RecordReaderUtils.stringifyDiskRanges(bufferChunks)); } createStreams(streamList, bufferChunks, fileIncluded, dataReader.getCompressionCodec(), bufferSize, streams); }
public OrcStripeMetadata(OrcBatchKey stripeKey, OrcProto.StripeFooter footer, OrcIndex orcIndex, StripeInformation stripe) throws IOException { this.stripeKey = stripeKey; streams = footer.getStreamsList(); encodings = footer.getColumnsList(); writerTimezone = footer.getWriterTimezone(); rowCount = stripe.getNumberOfRows(); rowIndex = orcIndex; }
private void readAllDataStreams(StripeInformation stripe) throws IOException { long start = stripe.getIndexLength(); long end = start + stripe.getDataLength(); // explicitly trigger 1 big read DiskRangeList toRead = new DiskRangeList(start, end); bufferChunks = dataReader.readFileData(toRead, stripe.getOffset(), false); List<OrcProto.Stream> streamDescriptions = stripeFooter.getStreamsList(); createStreams(streamDescriptions, bufferChunks, included, codec, bufferSize, streams); }
private void readAllDataStreams(StripeInformation stripe) throws IOException { long start = stripe.getIndexLength(); long end = start + stripe.getDataLength(); // explicitly trigger 1 big read DiskRangeList toRead = new DiskRangeList(start, end); bufferChunks = dataReader.readFileData(toRead, stripe.getOffset(), false); List<OrcProto.Stream> streamDescriptions = stripeFooter.getStreamsList(); createStreams(streamDescriptions, bufferChunks, included, codec, bufferSize, streams); }
private void readAllDataStreams(StripeInformation stripe) throws IOException { long start = stripe.getIndexLength(); long end = start + stripe.getDataLength(); // explicitly trigger 1 big read DiskRangeList toRead = new DiskRangeList(start, end); bufferChunks = dataReader.readFileData(toRead, stripe.getOffset(), false); List<OrcProto.Stream> streamDescriptions = stripeFooter.getStreamsList(); createStreams(streamDescriptions, bufferChunks, null, dataReader.getCompressionCodec(), bufferSize, streams); }
private void readPartialDataStreams(StripeInformation stripe) throws IOException { List<OrcProto.Stream> streamList = stripeFooter.getStreamsList(); DiskRangeList toRead = planReadPartialDataStreams(streamList, included, true); if (LOG.isDebugEnabled()) { LOG.debug("chunks = " + RecordReaderUtils.stringifyDiskRanges(toRead)); } bufferChunks = dataReader.readFileData(toRead, stripe.getOffset(), false); if (LOG.isDebugEnabled()) { LOG.debug("merge = " + RecordReaderUtils.stringifyDiskRanges(bufferChunks)); } createStreams(streamList, bufferChunks, included, codec, bufferSize, streams); }
public OrcStripeMetadata(OrcBatchKey stripeKey, OrcProto.StripeFooter footer, OrcIndex orcIndex, StripeInformation stripe) throws IOException { this.stripeKey = stripeKey; streams = footer.getStreamsList(); encodings = footer.getColumnsList(); writerTimezone = footer.getWriterTimezone(); rowCount = stripe.getNumberOfRows(); rowIndex = orcIndex; }