@Deprecated protected Builder(Path path) { this.readSupport = null; this.file = null; this.path = checkNotNull(path, "path"); this.conf = new Configuration(); this.optionsBuilder = HadoopReadOptions.builder(conf); }
@Override public ParquetReadOptions build() { return new HadoopReadOptions( useSignedStringMinMax, useStatsFilter, useDictionaryFilter, useRecordFilter, recordFilter, metadataFilter, codecFactory, allocator, maxAllocationSize, properties, conf); } }
public void initialize(ParquetFileReader reader, ParquetReadOptions options) { // copy custom configuration to the Configuration passed to the ReadSupport Configuration conf = new Configuration(); if (options instanceof HadoopReadOptions) { conf = ((HadoopReadOptions) options).getConf(); } for (String property : options.getPropertyNames()) { conf.set(property, options.getProperty(property)); } // initialize a ReadContext for this file this.reader = reader; FileMetaData parquetFileMetadata = reader.getFooter().getFileMetaData(); this.fileSchema = parquetFileMetadata.getSchema(); Map<String, String> fileMetadata = parquetFileMetadata.getKeyValueMetaData(); ReadSupport.ReadContext readContext = readSupport.init(new InitContext(conf, toSetMultiMap(fileMetadata), fileSchema)); this.columnIOFactory = new ColumnIOFactory(parquetFileMetadata.getCreatedBy()); this.requestedSchema = readContext.getRequestedSchema(); this.columnCount = requestedSchema.getPaths().size(); this.recordConverter = readSupport.prepareForRead(conf, fileMetadata, fileSchema, readContext); this.strictTypeChecking = options.isEnabled(STRICT_TYPE_CHECKING, true); this.total = reader.getRecordCount(); this.unmaterializableRecordCounter = new UnmaterializableRecordCounter(options, total); this.filterRecords = options.useRecordFilter(); reader.setRequestedSchema(requestedSchema); LOG.info("RecordReader initialized will read a total of {} records.", total); }
public void initialize(ParquetFileReader reader, ParquetReadOptions options) { // copy custom configuration to the Configuration passed to the ReadSupport Configuration conf = new Configuration(); if (options instanceof HadoopReadOptions) { conf = ((HadoopReadOptions) options).getConf(); } for (String property : options.getPropertyNames()) { conf.set(property, options.getProperty(property)); } // initialize a ReadContext for this file this.reader = reader; FileMetaData parquetFileMetadata = reader.getFooter().getFileMetaData(); this.fileSchema = parquetFileMetadata.getSchema(); Map<String, String> fileMetadata = parquetFileMetadata.getKeyValueMetaData(); ReadSupport.ReadContext readContext = readSupport.init(new InitContext(conf, toSetMultiMap(fileMetadata), fileSchema)); this.columnIOFactory = new ColumnIOFactory(parquetFileMetadata.getCreatedBy()); this.requestedSchema = readContext.getRequestedSchema(); this.columnCount = requestedSchema.getPaths().size(); this.recordConverter = readSupport.prepareForRead(conf, fileMetadata, fileSchema, readContext); this.strictTypeChecking = options.isEnabled(STRICT_TYPE_CHECKING, true); this.total = reader.getFilteredRecordCount(); this.unmaterializableRecordCounter = new UnmaterializableRecordCounter(options, total); this.filterRecords = options.useRecordFilter(); reader.setRequestedSchema(requestedSchema); LOG.info("RecordReader initialized will read a total of {} records.", total); }
@Deprecated protected Builder(Path path) { this.readSupport = null; this.file = null; this.path = checkNotNull(path, "path"); this.conf = new Configuration(); this.optionsBuilder = HadoopReadOptions.builder(conf); }
@Override public ParquetReadOptions build() { return new HadoopReadOptions( useSignedStringMinMax, useStatsFilter, useDictionaryFilter, useRecordFilter, useColumnIndexFilter, recordFilter, metadataFilter, codecFactory, allocator, maxAllocationSize, properties, conf); } }
@Deprecated private Builder(ReadSupport<T> readSupport, Path path) { this.readSupport = checkNotNull(readSupport, "readSupport"); this.file = null; this.path = checkNotNull(path, "path"); this.conf = new Configuration(); this.optionsBuilder = HadoopReadOptions.builder(conf); }
@Deprecated private Builder(ReadSupport<T> readSupport, Path path) { this.readSupport = checkNotNull(readSupport, "readSupport"); this.file = null; this.path = checkNotNull(path, "path"); this.conf = new Configuration(); this.optionsBuilder = HadoopReadOptions.builder(conf); }
protected Builder(InputFile file) { this.readSupport = null; this.file = checkNotNull(file, "file"); this.path = null; if (file instanceof HadoopInputFile) { this.conf = ((HadoopInputFile) file).getConfiguration(); } else { this.conf = new Configuration(); } optionsBuilder = HadoopReadOptions.builder(conf); }
protected Builder(InputFile file) { this.readSupport = null; this.file = checkNotNull(file, "file"); this.path = null; if (file instanceof HadoopInputFile) { this.conf = ((HadoopInputFile) file).getConfiguration(); } else { this.conf = new Configuration(); } optionsBuilder = HadoopReadOptions.builder(conf); }
public Builder<T> withConf(Configuration conf) { this.conf = checkNotNull(conf, "conf"); // previous versions didn't use the builder, so may set filter before conf. this maintains // compatibility for filter. other options are reset by a new conf. this.optionsBuilder = HadoopReadOptions.builder(conf); if (filter != null) { optionsBuilder.withRecordFilter(filter); } return this; }
public Builder<T> withConf(Configuration conf) { this.conf = checkNotNull(conf, "conf"); // previous versions didn't use the builder, so may set filter before conf. this maintains // compatibility for filter. other options are reset by a new conf. this.optionsBuilder = HadoopReadOptions.builder(conf); if (filter != null) { optionsBuilder.withRecordFilter(filter); } return this; }
/** * @param conf a configuration * @param file a file path to open * @return a parquet file reader * @throws IOException if there is an error while opening the file * @deprecated will be removed in 2.0.0; use {@link #open(InputFile)} */ @Deprecated public static ParquetFileReader open(Configuration conf, Path file) throws IOException { return new ParquetFileReader(HadoopInputFile.fromPath(file, conf), HadoopReadOptions.builder(conf).build()); }
/** * @param conf the Hadoop Configuration * @param file Path to a parquet file * @param filter a {@link MetadataFilter} for selecting row groups * @throws IOException if the file can not be opened * @deprecated will be removed in 2.0.0. */ @Deprecated public ParquetFileReader(Configuration conf, Path file, MetadataFilter filter) throws IOException { this(HadoopInputFile.fromPath(file, conf), HadoopReadOptions.builder(conf).withMetadataFilter(filter).build()); }
/** * @param conf a configuration * @param file a file path to open * @return a parquet file reader * @throws IOException if there is an error while opening the file * @deprecated will be removed in 2.0.0; use {@link #open(InputFile)} */ @Deprecated public static ParquetFileReader open(Configuration conf, Path file) throws IOException { return new ParquetFileReader(HadoopInputFile.fromPath(file, conf), HadoopReadOptions.builder(conf).build()); }
/** * @param conf the Hadoop Configuration * @param file Path to a parquet file * @param filter a {@link MetadataFilter} for selecting row groups * @throws IOException if the file can not be opened * @deprecated will be removed in 2.0.0. */ @Deprecated public ParquetFileReader(Configuration conf, Path file, MetadataFilter filter) throws IOException { this(HadoopInputFile.fromPath(file, conf), HadoopReadOptions.builder(conf).withMetadataFilter(filter).build()); }
private ParquetReader(Configuration conf, Path file, ReadSupport<T> readSupport, FilterCompat.Filter filter) throws IOException { this(Collections.singletonList((InputFile) HadoopInputFile.fromPath(file, conf)), HadoopReadOptions.builder(conf) .withRecordFilter(checkNotNull(filter, "filter")) .build(), readSupport); }
private ParquetReader(Configuration conf, Path file, ReadSupport<T> readSupport, FilterCompat.Filter filter) throws IOException { this(Collections.singletonList((InputFile) HadoopInputFile.fromPath(file, conf)), HadoopReadOptions.builder(conf) .withRecordFilter(checkNotNull(filter, "filter")) .build(), readSupport); }
/** * @param conf a configuration * @param file a file path to open * @param filter a metadata filter * @return a parquet file reader * @throws IOException if there is an error while opening the file * @deprecated will be removed in 2.0.0; use {@link #open(InputFile,ParquetReadOptions)} */ @Deprecated public static ParquetFileReader open(Configuration conf, Path file, MetadataFilter filter) throws IOException { return open(HadoopInputFile.fromPath(file, conf), HadoopReadOptions.builder(conf).withMetadataFilter(filter).build()); }
/** * @param conf a configuration * @param file a file path to open * @param filter a metadata filter * @return a parquet file reader * @throws IOException if there is an error while opening the file * @deprecated will be removed in 2.0.0; use {@link #open(InputFile,ParquetReadOptions)} */ @Deprecated public static ParquetFileReader open(Configuration conf, Path file, MetadataFilter filter) throws IOException { return open(HadoopInputFile.fromPath(file, conf), HadoopReadOptions.builder(conf).withMetadataFilter(filter).build()); }