/** * Set the subset of columns to read (projection pushdown). Specified as an Avro * schema, the requested projection is converted into a Parquet schema for Parquet * column projection. * <p> * This is useful if the full schema is large and you only want to read a few * columns, since it saves time by not reading unused columns. * <p> * If a requested projection is set, then the Avro schema used for reading * must be compatible with the projection. For instance, if a column is not included * in the projection then it must either not be included or be optional in the read * schema. Use {@link #setAvroReadSchema(org.apache.hadoop.mapreduce.Job, * org.apache.avro.Schema)} to set a read schema, if needed. * @param job a job * @param requestedProjection the requested projection schema * @see #setAvroReadSchema(org.apache.hadoop.mapreduce.Job, org.apache.avro.Schema) * @see org.apache.parquet.avro.AvroParquetOutputFormat#setSchema(org.apache.hadoop.mapreduce.Job, org.apache.avro.Schema) */ public static void setRequestedProjection(Job job, Schema requestedProjection) { AvroReadSupport.setRequestedProjection(ContextUtil.getConfiguration(job), requestedProjection); }
Schema readSchema = HoodieAvroUtils.getRecordKeySchema(); AvroReadSupport.setAvroReadSchema(conf, readSchema); AvroReadSupport.setRequestedProjection(conf, readSchema); Set<String> rowKeys = new HashSet<>(); try (ParquetReader reader = AvroParquetReader.builder(filePath).withConf(conf).build()) {
private ParquetReader<GenericRecord> initReader() throws IOException { Configuration configuration = getFs().getConf(); if (this.schema != null) { AvroReadSupport.setAvroReadSchema(configuration, this.schema); } if (this.projection != null) { AvroReadSupport.setRequestedProjection(configuration, this.projection); } ParquetReader reader = AvroParquetReader.<GenericRecord>builder(getFilePath()) .withConf(configuration).build(); return reader; }
AvroReadSupport.setRequestedProjection(conf, projectionSchema);
try { Schema chukwaAvroSchema = ChukwaAvroSchema.getSchema(); AvroReadSupport.setRequestedProjection(conf, chukwaAvroSchema); reader = new AvroParquetReader<GenericRecord>(conf, new Path(dataSinkFile));
AvroReadSupport.setRequestedProjection(configuration, AvroSchemaUtil.convert(expectedSchema, projection.getName())); org.apache.avro.Schema avroReadSchema = AvroSchemaUtil.buildAvroProjection(