private int writeFooter(long bodyLength) throws IOException { getStream(); OrcProto.Footer.Builder builder = OrcProto.Footer.newBuilder(); builder.setContentLength(bodyLength); builder.setHeaderLength(headerLength); builder.setNumberOfRows(rowCount); builder.setRowIndexStride(rowIndexStride); // populate raw data size rawDataSize = computeRawDataSize(); // serialize the types writeTypes(builder, schema); // add the stripe information for(OrcProto.StripeInformation stripe: stripes) { builder.addStripes(stripe); } // add the column statistics writeFileStatistics(builder, treeWriter); // add all of the user metadata for(Map.Entry<String, ByteString> entry: userMetadata.entrySet()) { builder.addMetadata(OrcProto.UserMetadataItem.newBuilder() .setName(entry.getKey()).setValue(entry.getValue())); } long startPosn = rawWriter.getPos(); OrcProto.Footer footer = builder.build(); footer.writeTo(protobufWriter); protobufWriter.flush(); writer.flush(); return (int) (rawWriter.getPos() - startPosn); }
DataOutputBuffer buffer = new DataOutputBuffer(); for(long stripeLength: stripeLengths) { footer.addStripes(OrcProto.StripeInformation.newBuilder() .setOffset(offset) .setIndexLength(0) footer.addTypes(OrcProto.Type.newBuilder() .setKind(OrcProto.Type.Kind.STRUCT) .addFieldNames("col1") .addSubtypes(1)); footer.addTypes(OrcProto.Type.newBuilder() .setKind(OrcProto.Type.Kind.STRING)); footer.setNumberOfRows(1000 * stripeLengths.length) .setHeaderLength(headerLen) .setContentLength(offset - headerLen); footer.addStatistics(OrcProto.ColumnStatistics.newBuilder() .setNumberOfValues(1000 * stripeLengths.length).build()); footer.addStatistics(OrcProto.ColumnStatistics.newBuilder() .setNumberOfValues(1000 * stripeLengths.length) .setStringStatistics( .build() ).build()); footer.build().writeTo(buffer); int footerEnd = buffer.getLength(); OrcProto.PostScript ps =
public Builder mergeFrom(org.apache.orc.OrcProto.Footer other) { if (other == org.apache.orc.OrcProto.Footer.getDefaultInstance()) return this; if (other.hasHeaderLength()) { setHeaderLength(other.getHeaderLength()); setContentLength(other.getContentLength()); bitField0_ = (bitField0_ & ~0x00000004); } else { ensureStripesIsMutable(); stripes_.addAll(other.stripes_); onChanged(); stripesBuilder_ = com.google.protobuf.GeneratedMessage.alwaysUseFieldBuilders ? getStripesFieldBuilder() : null; } else { stripesBuilder_.addAllMessages(other.stripes_); bitField0_ = (bitField0_ & ~0x00000008); } else { ensureTypesIsMutable(); types_.addAll(other.types_); onChanged(); typesBuilder_ = com.google.protobuf.GeneratedMessage.alwaysUseFieldBuilders ? getTypesFieldBuilder() : null;
schema.getCategory()); builder.addTypes(type); if (children != null) { for(TypeDescription child: children) {
schema.getCategory()); builder.addTypes(type); if (children != null) { for(TypeDescription child: children) {
/** * Build a virtual OrcTail for empty files. * @return a new OrcTail */ OrcTail buildEmptyTail() { OrcProto.PostScript.Builder postscript = OrcProto.PostScript.newBuilder(); OrcFile.Version version = OrcFile.Version.CURRENT; postscript.setMagic(OrcFile.MAGIC) .setCompression(OrcProto.CompressionKind.NONE) .setFooterLength(0) .addVersion(version.getMajor()) .addVersion(version.getMinor()) .setMetadataLength(0) .setWriterVersion(OrcFile.CURRENT_WRITER.getId()); // Use a struct with no fields OrcProto.Type.Builder struct = OrcProto.Type.newBuilder(); struct.setKind(OrcProto.Type.Kind.STRUCT); OrcProto.Footer.Builder footer = OrcProto.Footer.newBuilder(); footer.setHeaderLength(0) .setContentLength(0) .addTypes(struct) .setNumberOfRows(0) .setRowIndexStride(0); OrcProto.FileTail.Builder result = OrcProto.FileTail.newBuilder(); result.setFooter(footer); result.setPostscript(postscript); result.setFileLength(0); result.setPostscriptLength(0); return new OrcTail(result.build(), null); }
private int writeFooter(long bodyLength) throws IOException { getStream(); OrcProto.Footer.Builder builder = OrcProto.Footer.newBuilder(); builder.setContentLength(bodyLength); builder.setHeaderLength(headerLength); builder.setNumberOfRows(rowCount); builder.setRowIndexStride(rowIndexStride); // populate raw data size rawDataSize = computeRawDataSize(); // serialize the types writeTypes(builder, schema); // add the stripe information for(OrcProto.StripeInformation stripe: stripes) { builder.addStripes(stripe); } // add the column statistics writeFileStatistics(builder, treeWriter); // add all of the user metadata for(Map.Entry<String, ByteString> entry: userMetadata.entrySet()) { builder.addMetadata(OrcProto.UserMetadataItem.newBuilder() .setName(entry.getKey()).setValue(entry.getValue())); } long startPosn = rawWriter.getPos(); OrcProto.Footer footer = builder.build(); footer.writeTo(protobufWriter); protobufWriter.flush(); writer.flush(); return (int) (rawWriter.getPos() - startPosn); }
private long writeFooter() throws IOException { writeMetadata(); OrcProto.Footer.Builder builder = OrcProto.Footer.newBuilder(); builder.setNumberOfRows(rowCount); builder.setRowIndexStride(rowIndexStride); rawDataSize = computeRawDataSize(); // serialize the types writeTypes(builder, schema); // add the stripe information for(OrcProto.StripeInformation stripe: stripes) { builder.addStripes(stripe); } // add the column statistics writeFileStatistics(builder, treeWriter); // add all of the user metadata for(Map.Entry<String, ByteString> entry: userMetadata.entrySet()) { builder.addMetadata(OrcProto.UserMetadataItem.newBuilder() .setName(entry.getKey()).setValue(entry.getValue())); } builder.setWriter(OrcFile.WriterImplementation.ORC_JAVA.getId()); physicalWriter.writeFileFooter(builder); return writePostScript(); }
@Override public void writeFileStatistics(OrcProto.Footer.Builder footer) { footer.addStatistics(fileStatistics.serialize()); }
/** * <code>optional .orc.proto.Footer footer = 2;</code> */ public Builder setFooter( org.apache.orc.OrcProto.Footer.Builder builderForValue) { if (footerBuilder_ == null) { footer_ = builderForValue.build(); onChanged(); } else { footerBuilder_.setMessage(builderForValue.build()); } bitField0_ |= 0x00000002; return this; } /**
@Override public void writeFileFooter(OrcProto.Footer.Builder builder) throws IOException { OrcProto.Footer footer = builder.build(); validateIncludes(footer); }
private void writeFileStatistics(OrcProto.Footer.Builder builder, TreeWriter writer) throws IOException { builder.addStatistics(writer.fileStatistics.serialize()); for(TreeWriter child: writer.getChildrenWriters()) { writeFileStatistics(builder, child); } }
@Override public void writeFileFooter(OrcProto.Footer.Builder builder) throws IOException { long bodyLength = rawWriter.getPos() - metadataLength; builder.setContentLength(bodyLength); builder.setHeaderLength(headerLength); long startPosn = rawWriter.getPos(); OrcProto.Footer footer = builder.build(); footer.writeTo(protobufWriter); protobufWriter.flush(); writer.flush(); this.footerLength = (int) (rawWriter.getPos() - startPosn); }
private void writeFileStatistics(OrcProto.Footer.Builder builder, TreeWriter writer) throws IOException { builder.addStatistics(writer.fileStatistics.serialize()); for(TreeWriter child: writer.getChildrenWriters()) { writeFileStatistics(builder, child); } }
public OrcProto.FileTail getMinimalFileTail() { OrcProto.FileTail.Builder fileTailBuilder = OrcProto.FileTail.newBuilder(fileTail); OrcProto.Footer.Builder footerBuilder = OrcProto.Footer.newBuilder(fileTail.getFooter()); footerBuilder.clearStatistics(); fileTailBuilder.setFooter(footerBuilder.build()); OrcProto.FileTail result = fileTailBuilder.build(); return result; } }
@Override public void writeFileFooter(OrcProto.Footer.Builder builder) throws IOException { OrcProto.Footer footer = builder.build(); validateIncludes(footer); }