/** * Verifies that a non-null record reader can be created, and the key/value types are * as expected. */ @Test public void testCreateRecordReader() throws IOException, InterruptedException { // Set up the job configuration. Job job = new Job(); AvroJob.setInputKeySchema(job, Schema.create(Schema.Type.STRING)); Configuration conf = job.getConfiguration(); FileSplit inputSplit = createMock(FileSplit.class); TaskAttemptContext context = createMock(TaskAttemptContext.class); expect(context.getConfiguration()).andReturn(conf).anyTimes(); replay(inputSplit); replay(context); AvroKeyInputFormat inputFormat = new AvroKeyInputFormat(); @SuppressWarnings("unchecked") RecordReader<AvroKey<Object>, NullWritable> recordReader = inputFormat.createRecordReader( inputSplit, context); assertNotNull(inputFormat); recordReader.close(); verify(inputSplit); verify(context); } }
@Override FileInputFormat<AvroKey<E>, NullWritable> getInputFormat() { return new AvroKeyInputFormat<E>(); } }
private void initialiseInput(final Job job, final MapReduce operation) throws IOException { if (null == avroSchemaFilePath) { throw new IllegalArgumentException("Avro schema file path has not been set"); } final Schema schema = new Parser().parse(new File(avroSchemaFilePath)); AvroJob.setInputKeySchema(job, schema); job.setInputFormatClass(AvroKeyInputFormat.class); for (final Map.Entry<String, String> entry : operation.getInputMapperPairs().entrySet()) { if (entry.getValue().contains(job.getConfiguration().get(MAPPER_GENERATOR))) { AvroKeyInputFormat.addInputPath(job, new Path(entry.getKey())); } } }
@Override protected RecordReader<NullWritable, StructuredRecord.Builder> createRecordReader( FileSplit split, TaskAttemptContext context, @Nullable String pathField, @Nullable Schema schema) throws IOException, InterruptedException { RecordReader<AvroKey<GenericRecord>, NullWritable> delegate = (new AvroKeyInputFormat<GenericRecord>()) .createRecordReader(split, context); return new AvroRecordReader(delegate, schema, pathField); }