/** {@inheritDoc} */ @Override public RecordReader<AvroKey<T>, NullWritable> createRecordReader( InputSplit split, TaskAttemptContext context) throws IOException, InterruptedException { Schema readerSchema = AvroJob.getInputKeySchema(context.getConfiguration()); if (null == readerSchema) { LOG.warn("Reader schema was not set. Use AvroJob.setInputKeySchema() if desired."); LOG.info("Using a reader schema equal to the writer schema."); } return new AvroKeyRecordReader<>(readerSchema); } }
private static Schema getSchema(CombineFileSplit split, TaskAttemptContext cx, Integer idx) throws IOException { Schema schema = AvroJob.getInputKeySchema(cx.getConfiguration()); if (schema != null) { return schema; } Path path = split.getPath(idx); FileSystem fs = path.getFileSystem(cx.getConfiguration()); return AvroUtils.getSchemaFromDataFile(path, fs); }
/** {@inheritDoc} */ @Override public RecordReader<AvroKey<K>, AvroValue<V>> createRecordReader( InputSplit split, TaskAttemptContext context) throws IOException, InterruptedException { Schema keyReaderSchema = AvroJob.getInputKeySchema(context.getConfiguration()); if (null == keyReaderSchema) { LOG.warn("Key reader schema was not set. Use AvroJob.setInputKeySchema() if desired."); LOG.info("Using a key reader schema equal to the writer schema."); } Schema valueReaderSchema = AvroJob.getInputValueSchema(context.getConfiguration()); if (null == valueReaderSchema) { LOG.warn("Value reader schema was not set. Use AvroJob.setInputValueSchema() if desired."); LOG.info("Using a value reader schema equal to the writer schema."); } return new AvroKeyValueRecordReader<>(keyReaderSchema, valueReaderSchema); } }
/** {@inheritDoc} */ @Override public void initialize(InputSplit inputSplit, TaskAttemptContext context) throws IOException, InterruptedException { final FileSplit file = (FileSplit)inputSplit; context.setStatus(file.toString()); final AvroColumnReader.Params params = new AvroColumnReader.Params(new HadoopInput(file.getPath(), context.getConfiguration())); params.setModel(ReflectData.get()); if (AvroJob.getInputKeySchema(context.getConfiguration()) != null) { params.setSchema(AvroJob.getInputKeySchema(context.getConfiguration())); } reader = new AvroColumnReader<>(params); rows = reader.getRowCount(); }
public CombinedAvroKeyRecordReader(CombineFileSplit inputSplit, TaskAttemptContext context, Integer idx) { super(AvroJob.getInputKeySchema(context.getConfiguration())); this.inputSplit = inputSplit; this.idx = idx; }
@Override public RecordReader<Text, Text> createRecordReader( InputSplit split, TaskAttemptContext context) throws IOException, InterruptedException { Schema readerSchema = AvroJob.getInputKeySchema(context.getConfiguration()); return new AvroBatchRecordReader<T>(readerSchema); }
@SuppressWarnings("unchecked") @Override public RecordReader<AvroKey<T>, NullWritable> createRecordReader(InputSplit inputSplit, TaskAttemptContext context) throws IOException { Schema readerSchema = AvroJob.getInputKeySchema(context.getConfiguration()); if (null == readerSchema) { LOG.warn("Reader schema was not set. Use AvroJob.setInputKeySchema() if desired."); LOG.info("Using a reader schema equal to the writer schema."); } Object c = CombinedAvroKeyRecordReader.class; return new CombineFileRecordReader<AvroKey<T>, NullWritable>((CombineFileSplit) inputSplit, context, (Class<? extends RecordReader<AvroKey<T>, NullWritable>>)c); }
/** {@inheritDoc} */ @Override public RecordReader<AvroKey<T>, NullWritable> createRecordReader( InputSplit split, TaskAttemptContext context) throws IOException, InterruptedException { Schema readerSchema = AvroJob.getInputKeySchema(context.getConfiguration()); if (null == readerSchema) { LOG.warn("Reader schema was not set. Use AvroJob.setInputKeySchema() if desired."); LOG.info("Using a reader schema equal to the writer schema."); } return new AvroKeyRecordReader<T>(readerSchema); } }
private static Schema getSchema(CombineFileSplit split, TaskAttemptContext cx, Integer idx) throws IOException { Schema schema = AvroJob.getInputKeySchema(cx.getConfiguration()); if (schema != null) { return schema; } Path path = split.getPath(idx); FileSystem fs = path.getFileSystem(cx.getConfiguration()); return AvroUtils.getSchemaFromDataFile(path, fs); }
private static Schema getSchema(CombineFileSplit split, TaskAttemptContext cx, Integer idx) throws IOException { Schema schema = AvroJob.getInputKeySchema(cx.getConfiguration()); if (schema != null) { return schema; } Path path = split.getPath(idx); FileSystem fs = path.getFileSystem(cx.getConfiguration()); return AvroUtils.getSchemaFromDataFile(path, fs); }
public DelegatingAvroRecordReader(CombineFileSplit split, TaskAttemptContext context, Integer splitIndex) { Schema readerSchema = AvroJob.getInputKeySchema(context.getConfiguration()); if (null == readerSchema) { logger().warn("Reader schema was not set. Use AvroJob.setInputKeySchema() if desired."); logger().info("Using a reader schema equal to the writer schema."); } this.splitIndex = splitIndex; delegate = new AvroKeyRecordReaderSkipBad<>(readerSchema); }
/** {@inheritDoc} */ @Override public RecordReader<AvroKey<K>, AvroValue<V>> createRecordReader( InputSplit split, TaskAttemptContext context) throws IOException, InterruptedException { Schema keyReaderSchema = AvroJob.getInputKeySchema(context.getConfiguration()); if (null == keyReaderSchema) { LOG.warn("Key reader schema was not set. Use AvroJob.setInputKeySchema() if desired."); LOG.info("Using a key reader schema equal to the writer schema."); } Schema valueReaderSchema = AvroJob.getInputValueSchema(context.getConfiguration()); if (null == valueReaderSchema) { LOG.warn("Value reader schema was not set. Use AvroJob.setInputValueSchema() if desired."); LOG.info("Using a value reader schema equal to the writer schema."); } return new AvroKeyValueRecordReader<K, V>(keyReaderSchema, valueReaderSchema); } }
/** * {@inheritDoc} */ @Override @SuppressWarnings("unchecked") public RecordReader<AvroKey<K>, AvroValue<V>> createRecordReader( InputSplit split, TaskAttemptContext context) throws IOException { final Schema keyReaderSchema = AvroJob.getInputKeySchema(HadoopCompat.getConfiguration(context)); if (null == keyReaderSchema) { LOG.warn("Key reader schema was not set. Use AvroJob.setInputKeySchema() if desired."); LOG.info("Using a key reader schema equal to the writer schema."); } final Schema valueReaderSchema = AvroJob.getInputValueSchema(HadoopCompat.getConfiguration(context)); if (null == valueReaderSchema) { LOG.warn("Value reader schema was not set. Use AvroJob.setInputValueSchema() if desired."); LOG.info("Using a value reader schema equal to the writer schema."); } return new CommonCombineFileRecordReader<K, V>(new CommonCombineFileRecordReader.RecordReaderEngineerer<K, V>() { @Override public RecordReader createRecordReader() { return new AvroKeyValueRecordReader<K, V>(keyReaderSchema, valueReaderSchema); } }); } }