@Test public void emptyFile() throws IOException { AvroGenericRecordReader reader = new AvroGenericRecordReader(jobConf, emptyFileSplit, reporter); //next() should always return false Assert.assertEquals(false, reader.next(null, null)); //getPos() should always return 0 Assert.assertEquals(0, reader.getPos()); //close() should just do nothing reader.close(); } }
@Override public float getProgress() throws IOException { return stop == start ? 0.0f : Math.min(1.0f, (getPos() - start) / (float)(stop - start)); }
@Override public RecordReader<NullWritable, AvroGenericRecordWritable> getRecordReader(InputSplit inputSplit, JobConf jc, Reporter reporter) throws IOException { return new AvroGenericRecordReader(jc, (FileSplit) inputSplit, reporter); }
public AvroGenericRecordReader(JobConf job, FileSplit split, Reporter reporter) throws IOException { this.jobConf = job; Schema latest; try { latest = getSchema(job, split); } catch (AvroSerdeException e) { throw new IOException(e); } GenericDatumReader<GenericRecord> gdr = new GenericDatumReader<GenericRecord>(); if(latest != null) { gdr.setExpected(latest); } if (split.getLength() == 0) { this.isEmptyInput = true; this.start = 0; this.reader = null; } else { this.isEmptyInput = false; this.reader = new DataFileReader<GenericRecord>(new FsInput(split.getPath(), job), gdr); this.reader.sync(split.getStart()); this.start = reader.tell(); } this.stop = split.getStart() + split.getLength(); this.recordReaderID = new UID(); }
if(pathIsInPartition(split.getPath(), partitionPath)) { if(LOG.isInfoEnabled()) { LOG.info("Matching partition " + partitionPath +
@Override public float getProgress() throws IOException { return stop == start ? 0.0f : Math.min(1.0f, (getPos() - start) / (float)(stop - start)); }
@Override public RecordReader<NullWritable, AvroGenericRecordWritable> getRecordReader(InputSplit inputSplit, JobConf jc, Reporter reporter) throws IOException { return new AvroGenericRecordReader(jc, (FileSplit) inputSplit, reporter); }
public AvroGenericRecordReader(JobConf job, FileSplit split, Reporter reporter) throws IOException { this.jobConf = job; Schema latest; try { latest = getSchema(job, split); } catch (AvroSerdeException e) { throw new IOException(e); } GenericDatumReader<GenericRecord> gdr = new GenericDatumReader<GenericRecord>(); if(latest != null) { gdr.setExpected(latest); } if (split.getLength() == 0) { this.isEmptyInput = true; this.start = 0; this.reader = null; } else { this.isEmptyInput = false; this.reader = new DataFileReader<GenericRecord>(new FsInput(split.getPath(), job), gdr); this.reader.sync(split.getStart()); this.start = reader.tell(); } this.stop = split.getStart() + split.getLength(); this.recordReaderID = new UID(); }
if(pathIsInPartition(split.getPath(), partitionPath)) { if(LOG.isInfoEnabled()) { LOG.info("Matching partition " + partitionPath +
@Override public float getProgress() throws IOException { return stop == start ? 0.0f : Math.min(1.0f, (getPos() - start) / (float)(stop - start)); }
@Override public RecordReader<NullWritable, AvroGenericRecordWritable> getRecordReader(InputSplit inputSplit, JobConf jc, Reporter reporter) throws IOException { return new AvroGenericRecordReader(jc, (FileSplit) inputSplit, reporter); }
public AvroGenericRecordReader(JobConf job, FileSplit split, Reporter reporter) throws IOException { this.jobConf = job; Schema latest; try { latest = getSchema(job, split); } catch (AvroSerdeException e) { throw new IOException(e); } GenericDatumReader<GenericRecord> gdr = new GenericDatumReader<GenericRecord>(); if(latest != null) gdr.setExpected(latest); this.reader = new DataFileReader<GenericRecord>(new FsInput(split.getPath(), job), gdr); this.reader.sync(split.getStart()); this.start = reader.tell(); this.stop = split.getStart() + split.getLength(); }
if(pathIsInPartition(split.getPath(), partitionPath)) { if(LOG.isInfoEnabled()) { LOG.info("Matching partition " + partitionPath +
@Override public float getProgress() throws IOException { return stop == start ? 0.0f : Math.min(1.0f, (getPos() - start) / (float)(stop - start)); }
@Override public RecordReader<NullWritable, AvroGenericRecordWritable> getRecordReader(InputSplit inputSplit, JobConf jc, Reporter reporter) throws IOException { return new AvroGenericRecordReader(jc, (FileSplit) inputSplit, reporter); }
public AvroGenericRecordReader(JobConf job, FileSplit split, Reporter reporter) throws IOException { this.jobConf = job; Schema latest; try { latest = getSchema(job, split); } catch (AvroSerdeException e) { throw new IOException(e); } GenericDatumReader<GenericRecord> gdr = new GenericDatumReader<GenericRecord>(); if(latest != null) { gdr.setExpected(latest); } this.reader = new DataFileReader<GenericRecord>(new FsInput(split.getPath(), job), gdr); this.reader.sync(split.getStart()); this.start = reader.tell(); this.stop = split.getStart() + split.getLength(); this.recordReaderID = new UID(); }
if(pathIsInPartition(split.getPath(), partitionPath)) { if(LOG.isInfoEnabled()) { LOG.info("Matching partition " + partitionPath +