ensureDataReader(); stripeReader.readIndexStreams(stripeMetadata.getIndex(), si, footer.getStreamsList(), fileIncludes, sargColumns); consumer.setStripeMetadata(stripeMetadata);
@Override public OrcProto.StripeFooter readStripeFooter(StripeInformation stripe) throws IOException { OrcProto.StripeFooter.Builder fb = OrcProto.StripeFooter.newBuilder(); if (!isEmpty) { fb.addStreams(OrcProto.Stream.newBuilder().setColumn(0).setLength(20).setKind(OrcProto.Stream.Kind.LENGTH)) .addStreams(OrcProto.Stream.newBuilder().setColumn(0).setLength(40).setKind(OrcProto.Stream.Kind.DATA)) .addColumns(OrcProto.ColumnEncoding.newBuilder().setDictionarySize(10).setKind(OrcProto.ColumnEncoding.Kind.DIRECT_V2)); } OrcProto.StripeFooter footer = fb.build(); if (doStreamStep) { ByteArrayOutputStream baos = new ByteArrayOutputStream(); CodedOutputStream cos = CodedOutputStream.newInstance(baos); footer.writeTo(cos); cos.flush(); footer = OrcProto.StripeFooter.newBuilder().mergeFrom(baos.toByteArray()).build(); } return footer; }
ensureDataReader(); stripeReader.readIndexStreams(stripeMetadata.getIndex(), si, footer.getStreamsList(), fileIncludes, sargColumns); consumer.setStripeMetadata(stripeMetadata);
(int) ((rowsInStripe + rowIndexStride - 1) / rowIndexStride); OrcProto.StripeFooter.Builder builder = OrcProto.StripeFooter.newBuilder(); treeWriter.writeStripe(builder, requiredIndexEntries); long indexSize = 0; final long currentStripeSize = indexSize + dataSize + footer.getSerializedSize(); final long available = blockSize - (start % blockSize); final long overflow = currentStripeSize - adjustedStripeSize; footer.writeTo(protobufWriter); protobufWriter.flush(); writer.flush();
(int) ((rowsInStripe + rowIndexStride - 1) / rowIndexStride); OrcProto.StripeFooter.Builder builder = OrcProto.StripeFooter.newBuilder(); treeWriter.writeStripe(builder, requiredIndexEntries); long indexSize = 0; final long currentStripeSize = indexSize + dataSize + footer.getSerializedSize(); final long available = blockSize - (start % blockSize); final long overflow = currentStripeSize - adjustedStripeSize; footer.writeTo(protobufWriter); protobufWriter.flush(); writer.flush();
OrcProto.Stream.Kind[] bloomFilterKinds) { DiskRangeList.CreateHelper result = new DiskRangeList.CreateHelper(); List<OrcProto.Stream> streams = footer.getStreamsList(); for(OrcProto.Stream stream: footer.getStreamsList()) { if (stream.hasKind() && stream.hasColumn()) { int column = stream.getColumn();
byte[] buffer = new byte[(int) footerLength]; inputStream.readFully(offset + indexLength + dataLength, buffer); OrcProto.StripeFooter footer = OrcProto.StripeFooter.parseFrom(buffer); List<OrcProto.Stream> streams = footer.getStreamsList(); for (OrcProto.Stream stream : streams)
long offset = 0; DiskRangeList range = ranges; for(OrcProto.Stream stream: footer.getStreamsList()) {
padStripe(indexSize + dataSize + footer.getSerializedSize());
(int) ((rowsInStripe + rowIndexStride - 1) / rowIndexStride); OrcProto.StripeFooter.Builder builder = OrcProto.StripeFooter.newBuilder(); if (writeTimeZone) { if (useUTCTimeZone) {
/** * Reads the metadata for all stripes in the file. */ private ArrayList<OrcStripeMetadata> readStripesMetadata( boolean[] includes, boolean[] sargColumns) throws IOException { ArrayList<OrcStripeMetadata> result = new ArrayList<OrcStripeMetadata>(stripeRgs.length); boolean hasFileId = this.fileKey != null; OrcBatchKey stripeKey = hasFileId ? new OrcBatchKey(fileKey, 0, 0) : null; for (int stripeIxMod = 0; stripeIxMod < stripeRgs.length; ++stripeIxMod) { int stripeIx = stripeIxMod + stripeIxFrom; stripeKey.stripeIx = stripeIx; StripeInformation si = fileMetadata.getStripes().get(stripeIx); OrcProto.StripeFooter footer = getStripeFooterFromCacheOrDisk(si, stripeKey); OrcStripeMetadata osm = createOrcStripeMetadataObject( stripeIx, si, footer, includes, sargColumns); ensureDataReader(); OrcIndex index = osm.getIndex(); stripeReader.readIndexStreams(index, si, footer.getStreamsList(), includes, sargColumns); result.add(osm); consumer.setStripeMetadata(osm); } return result; }
@Override void startStripe(Map<org.apache.orc.impl.StreamName, InStream> streams, OrcProto.StripeFooter stripeFooter ) throws IOException { // For each stripe, checks the encoding and initializes the appropriate // reader switch (stripeFooter.getColumnsList().get(columnId).getKind()) { case DIRECT: case DIRECT_V2: reader = new StringDirectTreeReader(columnId); break; case DICTIONARY: case DICTIONARY_V2: reader = new StringDictionaryTreeReader(columnId); break; default: throw new IllegalArgumentException("Unsupported encoding " + stripeFooter.getColumnsList().get(columnId).getKind()); } reader.startStripe(streams, stripeFooter); }
@Override void startStripe(Map<org.apache.orc.impl.StreamName, InStream> streams, OrcProto.StripeFooter stripeFooter ) throws IOException { super.startStripe(streams, stripeFooter); // read the dictionary blob org.apache.orc.impl.StreamName name = new org.apache.orc.impl.StreamName(columnId, OrcProto.Stream.Kind.DICTIONARY_DATA); InStream in = streams.get(name); readDictionaryStream(in); // read the lengths name = new org.apache.orc.impl.StreamName(columnId, OrcProto.Stream.Kind.LENGTH); in = streams.get(name); readDictionaryLengthStream(in, stripeFooter.getColumnsList().get(columnId)); // set up the row reader name = new org.apache.orc.impl.StreamName(columnId, OrcProto.Stream.Kind.DATA); reader = createIntegerReader(stripeFooter.getColumnsList().get(columnId).getKind(), streams.get(name), false, false); }
@Override void startStripe(Map<StreamName, InStream> streams, OrcProto.StripeFooter stripeFooter ) throws IOException { // For each stripe, checks the encoding and initializes the appropriate // reader switch (stripeFooter.getColumnsList().get(columnId).getKind()) { case DIRECT: case DIRECT_V2: reader = new StringDirectTreeReader(columnId); break; case DICTIONARY: case DICTIONARY_V2: reader = new StringDictionaryTreeReader(columnId); break; default: throw new IllegalArgumentException("Unsupported encoding " + stripeFooter.getColumnsList().get(columnId).getKind()); } reader.startStripe(streams, stripeFooter); }
@Override void startStripe(Map<org.apache.orc.impl.StreamName, InStream> streams, OrcProto.StripeFooter stripeFooter ) throws IOException { // For each stripe, checks the encoding and initializes the appropriate // reader switch (stripeFooter.getColumnsList().get(columnId).getKind()) { case DIRECT: case DIRECT_V2: reader = new StringDirectTreeReader(columnId); break; case DICTIONARY: case DICTIONARY_V2: reader = new StringDictionaryTreeReader(columnId); break; default: throw new IllegalArgumentException("Unsupported encoding " + stripeFooter.getColumnsList().get(columnId).getKind()); } reader.startStripe(streams, stripeFooter); }
@Override void startStripe(Map<StreamName, InStream> streams, OrcProto.StripeFooter stripeFooter ) throws IOException { // For each stripe, checks the encoding and initializes the appropriate // reader switch (stripeFooter.getColumnsList().get(columnId).getKind()) { case DIRECT: case DIRECT_V2: reader = new StringDirectTreeReader(columnId); break; case DICTIONARY: case DICTIONARY_V2: reader = new StringDictionaryTreeReader(columnId, context); break; default: throw new IllegalArgumentException("Unsupported encoding " + stripeFooter.getColumnsList().get(columnId).getKind()); } reader.startStripe(streams, stripeFooter); }
@Override void startStripe(Map<StreamName, InStream> streams, OrcProto.StripeFooter stripeFooter ) throws IOException { // For each stripe, checks the encoding and initializes the appropriate // reader switch (stripeFooter.getColumnsList().get(columnId).getKind()) { case DIRECT: case DIRECT_V2: reader = new StringDirectTreeReader(columnId); break; case DICTIONARY: case DICTIONARY_V2: reader = new StringDictionaryTreeReader(columnId); break; default: throw new IllegalArgumentException("Unsupported encoding " + stripeFooter.getColumnsList().get(columnId).getKind()); } reader.startStripe(streams, stripeFooter); }
@Override void startStripe(Map<org.apache.orc.impl.StreamName, InStream> streams, OrcProto.StripeFooter stripeFooter ) throws IOException { super.startStripe(streams, stripeFooter); // read the dictionary blob org.apache.orc.impl.StreamName name = new org.apache.orc.impl.StreamName(columnId, OrcProto.Stream.Kind.DICTIONARY_DATA); InStream in = streams.get(name); readDictionaryStream(in); // read the lengths name = new org.apache.orc.impl.StreamName(columnId, OrcProto.Stream.Kind.LENGTH); in = streams.get(name); readDictionaryLengthStream(in, stripeFooter.getColumnsList().get(columnId)); // set up the row reader name = new org.apache.orc.impl.StreamName(columnId, OrcProto.Stream.Kind.DATA); reader = createIntegerReader(stripeFooter.getColumnsList().get(columnId).getKind(), streams.get(name), false, false); }
@Override void startStripe(Map<StreamName, InStream> streams, OrcProto.StripeFooter stripeFooter ) throws IOException { super.startStripe(streams, stripeFooter); // read the dictionary blob StreamName name = new StreamName(columnId, OrcProto.Stream.Kind.DICTIONARY_DATA); InStream in = streams.get(name); readDictionaryStream(in); // read the lengths name = new StreamName(columnId, OrcProto.Stream.Kind.LENGTH); in = streams.get(name); readDictionaryLengthStream(in, stripeFooter.getColumnsList().get(columnId)); // set up the row reader name = new StreamName(columnId, OrcProto.Stream.Kind.DATA); reader = createIntegerReader(stripeFooter.getColumnsList().get(columnId).getKind(), streams.get(name), false, context); }
@Override void startStripe(Map<StreamName, InStream> streams, OrcProto.StripeFooter stripeFooter ) throws IOException { super.startStripe(streams, stripeFooter); lengths = createIntegerReader(stripeFooter.getColumnsList().get(columnId).getKind(), streams.get(new StreamName(columnId, OrcProto.Stream.Kind.LENGTH)), false, context); if (keyReader != null) { keyReader.startStripe(streams, stripeFooter); } if (valueReader != null) { valueReader.startStripe(streams, stripeFooter); } }