protected PageHeader readPageHeader() throws IOException { return Util.readPageHeader(this); }
@Override public FileMetaData visit(SkipMetadataFilter filter) throws IOException { return readFileMetaData(from, true); }
private static void write(TBase<?, ?> tbase, OutputStream to) throws IOException { try { tbase.write(protocol(to)); } catch (TException e) { throw new IOException("can not write " + tbase, e); } } }
/** * reads the meta data from the stream * @param from the stream to read the metadata from * @param skipRowGroups whether row groups should be skipped * @return the resulting metadata * @throws IOException */ public static FileMetaData readFileMetaData(InputStream from, boolean skipRowGroups) throws IOException { FileMetaData md = new FileMetaData(); if (skipRowGroups) { readFileMetaData(from, new DefaultFileMetaDataConsumer(md), skipRowGroups); } else { read(from, md); } return md; }
static final class OffsetMetadataFilter extends MetadataFilter { private final Set<Long> offsets; public OffsetMetadataFilter(Set<Long> offsets) { this.offsets = offsets; } public boolean contains(long offset) { return offsets.contains(offset); } @Override <T, E extends Throwable> T accept(MetadataFilterVisitor<T, E> visitor) throws E { return visitor.visit(this); } }
public static FileMetaData readFileMetaData(InputStream from) throws IOException { return read(from, new FileMetaData()); } /**
public static void writeFileMetaData(org.apache.parquet.format.FileMetaData fileMetadata, OutputStream to) throws IOException { write(fileMetadata, to); }
private static void serializeFooter(ParquetMetadata footer, PositionOutputStream out) throws IOException { long footerIndex = out.getPos(); org.apache.parquet.format.FileMetaData parquetMetadata = metadataConverter.toParquetMetadata(CURRENT_VERSION, footer); writeFileMetaData(parquetMetadata, out); LOG.debug("{}: footer length = {}" , out.getPos(), (out.getPos() - footerIndex)); BytesUtils.writeIntLittleEndian(out, (int) (out.getPos() - footerIndex)); out.write(MAGIC); }
/** * reads the meta data from the stream * @param from the stream to read the metadata from * @param skipRowGroups whether row groups should be skipped * @return the resulting metadata * @throws IOException if any I/O error occurs during the reading */ public static FileMetaData readFileMetaData(InputStream from, boolean skipRowGroups) throws IOException { FileMetaData md = new FileMetaData(); if (skipRowGroups) { readFileMetaData(from, new DefaultFileMetaDataConsumer(md), skipRowGroups); } else { read(from, md); } return md; }
static final class OffsetMetadataFilter extends MetadataFilter { private final Set<Long> offsets; public OffsetMetadataFilter(Set<Long> offsets) { this.offsets = offsets; } public boolean contains(long offset) { return offsets.contains(offset); } @Override <T, E extends Throwable> T accept(MetadataFilterVisitor<T, E> visitor) throws E { return visitor.visit(this); } }
public static FileMetaData readFileMetaData(InputStream from) throws IOException { return read(from, new FileMetaData()); } /**
public static void writeOffsetIndex(OffsetIndex offsetIndex, OutputStream to) throws IOException { write(offsetIndex, to); }
private static void serializeFooter(ParquetMetadata footer, PositionOutputStream out) throws IOException { long footerIndex = out.getPos(); org.apache.parquet.format.FileMetaData parquetMetadata = metadataConverter.toParquetMetadata(CURRENT_VERSION, footer); writeFileMetaData(parquetMetadata, out); LOG.debug("{}: footer length = {}" , out.getPos(), (out.getPos() - footerIndex)); BytesUtils.writeIntLittleEndian(out, (int) (out.getPos() - footerIndex)); out.write(MAGIC); }
public PageHeader readPageHeader() throws IOException{ return Util.readPageHeader(input); }
@Override public FileMetaData visit(NoFilter filter) throws IOException { return readFileMetaData(from); }
private static void write(TBase<?, ?> tbase, OutputStream to) throws IOException { try { tbase.write(protocol(to)); } catch (TException e) { throw new IOException("can not write " + tbase, e); } } }
public void writeDataPageV2Header( int uncompressedSize, int compressedSize, int valueCount, int nullCount, int rowCount, org.apache.parquet.column.Encoding dataEncoding, int rlByteLength, int dlByteLength, OutputStream to) throws IOException { writePageHeader( newDataPageV2Header( uncompressedSize, compressedSize, valueCount, nullCount, rowCount, dataEncoding, rlByteLength, dlByteLength), to); }
public static ColumnIndex readColumnIndex(InputStream from) throws IOException { return read(from, new ColumnIndex()); }
public static void writeFileMetaData(org.apache.parquet.format.FileMetaData fileMetadata, OutputStream to) throws IOException { write(fileMetadata, to); }
protected PageHeader readPageHeader() throws IOException { return Util.readPageHeader(this); }