static Input input(String filename) throws IOException { if (filename.startsWith("hdfs://")) { return new HadoopInput(new Path(filename), new Configuration()); } else { return new InputFile(new File(filename)); } }
/** {@inheritDoc} */ @Override public void initialize(InputSplit inputSplit, TaskAttemptContext context) throws IOException, InterruptedException { final FileSplit file = (FileSplit)inputSplit; context.setStatus(file.toString()); final AvroColumnReader.Params params = new AvroColumnReader.Params(new HadoopInput(file.getPath(), context.getConfiguration())); params.setModel(ReflectData.get()); if (AvroJob.getInputKeySchema(context.getConfiguration()) != null) { params.setSchema(AvroJob.getInputKeySchema(context.getConfiguration())); } reader = new AvroColumnReader<>(params); rows = reader.getRowCount(); }
@Override public RecordReader<AvroWrapper<T>, NullWritable> getRecordReader(InputSplit split, final JobConf job, Reporter reporter) throws IOException { final FileSplit file = (FileSplit)split; reporter.setStatus(file.toString()); final AvroColumnReader.Params params = new AvroColumnReader.Params(new HadoopInput(file.getPath(), job)); params.setModel(ReflectData.get()); if (job.get(AvroJob.INPUT_SCHEMA) != null) params.setSchema(AvroJob.getInputSchema(job)); return new RecordReader<AvroWrapper<T>, NullWritable>() { private AvroColumnReader<T> reader = new AvroColumnReader<>(params); private float rows = reader.getRowCount(); private long row; public AvroWrapper<T> createKey() { return new AvroWrapper<>(null); } public NullWritable createValue() { return NullWritable.get(); } public boolean next(AvroWrapper<T> wrapper, NullWritable ignore) throws IOException { if (!reader.hasNext()) return false; wrapper.datum(reader.next()); row++; return true; } public float getProgress() throws IOException { return row / rows; } public long getPos() throws IOException { return row; } public void close() throws IOException { reader.close(); } }; }
public TrevniScanner(Configuration conf, Schema schema, TableMeta meta, FileFragment fragment) throws IOException { super(conf, schema, meta, fragment); reader = new ColumnFileReader(new HadoopInput(fragment.getPath(), conf)); }
static Input input(String filename) throws IOException { if (filename.startsWith("hdfs://")) { return new HadoopInput(new Path(filename), new Configuration()); } else { return new InputFile(new File(filename)); } }
static Input input(String filename) throws IOException { if (filename.startsWith("hdfs://")) { return new HadoopInput(new Path(filename), new Configuration()); } else { return new InputFile(new File(filename)); } }
new HadoopInput(filePath, job.getConfiguration())); AvroColumnReader<GenericData.Record> reader = new AvroColumnReader<GenericData.Record>(params);
new AvroColumnReader.Params(new HadoopInput(file.getPath(), job)); params.setModel(ReflectData.get()); if (job.get(AvroJob.INPUT_SCHEMA) != null)