@Override public void open(FileInputSplit split) throws IOException { // open input split using FileInputFormat super.open(split); // adjust the stream positions for boundary splits int recordOffset = (int) (this.splitStart % this.recordLength); if(recordOffset != 0) { // move start to next boundary super.stream.seek(this.splitStart + recordOffset); } this.streamPos = this.splitStart + recordOffset; this.streamEnd = this.splitStart + this.splitLength; this.streamEnd += this.streamEnd % this.recordLength; // adjust readBufferSize this.readBufferSize += this.recordLength - (this.readBufferSize % this.recordLength); if (this.readBuffer == null || this.readBuffer.length != this.readBufferSize) { this.readBuffer = new byte[this.readBufferSize]; } this.readBufferLimit = 0; this.readBufferPos = 0; this.exhausted = false; fillReadBuffer(); }
@Override public void open(FileInputSplit split) throws IOException { super.open(split); DatumReader<GenericRecord> datumReader = new GenericDatumReader<GenericRecord>(); SeekableInput in = new FSDataInputStreamWrapper(stream, (int) split.getLength()); LOG.info("Opening split " + split); dataFileReader = DataFileReader.openReader(in, datumReader); dataFileReader.sync(split.getStart()); }
@Override public void open(FileInputSplit split) throws IOException { super.open(split); this.wrapper = InstantiationUtil.instantiate(avroWrapperTypeClass, AvroBaseValue.class); DatumReader<E> datumReader; if (org.apache.avro.specific.SpecificRecordBase.class.isAssignableFrom(avroValueType)) { datumReader = new SpecificDatumReader<E>(avroValueType); } else { datumReader = new ReflectDatumReader<E>(avroValueType); } LOG.info("Opening split " + split); SeekableInput in = new FSDataInputStreamWrapper(stream, (int) split.getLength()); dataFileReader = DataFileReader.openReader(in, datumReader); dataFileReader.sync(split.getStart()); reuseAvroValue = null; }