@Override public void initialize(InputSplit unusedSplit, TaskAttemptContext cx) throws IOException, InterruptedException { super.initialize( new FileSplit(this.split.getPath(this.idx), this.split.getOffset(this.idx), this.split.getLength(this.idx), null), cx); }
public CustomRecordReader(CombineFileSplit split, TaskAttemptContext context, Integer index) throws IOException{ path = split.getPath(index); fs = path.getFileSystem(context.getConfiguration()); startOffset = split.getOffset(index); endOffset = startOffset + split.getLength(index); fileIn = fs.open(path); reader = new LineReader(fileIn); pos = startOffset; }
/** Returns the start offset of the i<sup>th</sup> Path */ public long getOffset(int i) { return isMapRedSet() ? mapredSplit.getOffset(i) : mapreduceSplit.getOffset(i); }
@SuppressWarnings("unchecked") @Override public void initialize(InputSplit curSplit, TaskAttemptContext curContext) throws IOException, InterruptedException { this.split = (CombineFileSplit) curSplit; this.context = curContext; if (null == rr) { rr = ReflectionUtils.newInstance(SequenceFileRecordReader.class, context.getConfiguration()); } FileSplit fileSplit = new FileSplit(this.split.getPath(index), this.split.getOffset(index), this.split.getLength(index), this.split.getLocations()); this.rr.initialize(fileSplit, this.context); }
@SuppressWarnings("unchecked") @Override public void initialize(InputSplit curSplit, TaskAttemptContext curContext) throws IOException, InterruptedException { this.split = (CombineFileSplit) curSplit; this.context = curContext; if (null == rr) { rr = ReflectionUtils.newInstance(SequenceFileRecordReader.class, context.getConfiguration()); } FileSplit fileSplit = new FileSplit(this.split.getPath(index), this.split.getOffset(index), this.split.getLength(index), this.split.getLocations()); this.rr.initialize(fileSplit, this.context); }
@Override public void initialize(InputSplit split, TaskAttemptContext context) throws IOException, InterruptedException { this.context = context; this.split = (CombineFileSplit) split; conf = HadoopCompat.getConfiguration(context); for (int i = 0; i < this.split.getPaths().length; i++) { totalBytes += this.split.getLength(i); if (LOG.isDebugEnabled()) { LOG.debug(String.format("Got split %s offest %d len %d", this.split.getPath(i), this.split.getOffset(i), this.split.getLength(i))); } } nextReader(); }
@Override public void initialize(InputSplit unusedSplit, TaskAttemptContext cx) throws IOException, InterruptedException { super.initialize( new FileSplit(this.split.getPath(this.idx), this.split.getOffset(this.idx), this.split.getLength(this.idx), null), cx); }
@Override public void initialize(InputSplit split, TaskAttemptContext context) throws IOException, InterruptedException { CombineFileSplit combineSplit = (CombineFileSplit)split; Path path = combineSplit.getPath(splitIndex); this.fileName = path.getName(); FileSplit fileSplit = new FileSplit( path, combineSplit.getOffset(splitIndex), combineSplit.getLength(splitIndex), combineSplit.getLocations()); delegate.initialize(fileSplit, context); }
@Override public void initialize(InputSplit unusedSplit, TaskAttemptContext cx) throws IOException, InterruptedException { super.initialize( new FileSplit(this.split.getPath(this.idx), this.split.getOffset(this.idx), this.split.getLength(this.idx), null), cx); }
@Override public void initialize(InputSplit arg0, TaskAttemptContext arg1) throws IOException, InterruptedException { innerReader.initialize(new FileSplit(split.getPath(idx), split.getOffset(idx), split.getLength(idx), null), arg1); }
protected CombineFileRecordReaderWrapper(FileInputFormat<K,V> inputFormat, CombineFileSplit split, TaskAttemptContext context, Integer idx) throws IOException, InterruptedException { fileSplit = new FileSplit(split.getPath(idx), split.getOffset(idx), split.getLength(idx), split.getLocations()); delegate = inputFormat.createRecordReader(fileSplit, context); }
private InputSplit getDelegateSplit() throws IOException { if (combineFileSplit != null) { return new FileSplit(combineFileSplit.getPath(idx - 1), combineFileSplit.getOffset(idx - 1), combineFileSplit.getLength(idx - 1), combineFileSplit.getLocations()); } else { return crunchSplit.get(); } }
@Override public void initialize(InputSplit inputSplit, TaskAttemptContext context) throws IOException, InterruptedException { this.inputSplit = (CombineFileSplit)inputSplit; FileSplit fileSplit = new FileSplit(this.inputSplit.getPath(idx), this.inputSplit.getOffset(idx), this.inputSplit.getLength(idx), this.inputSplit.getLocations()); super.initialize(fileSplit, context); } }
protected CombineFileRecordReaderWrapper(FileInputFormat<K,V> inputFormat, CombineFileSplit split, TaskAttemptContext context, Integer idx) throws IOException, InterruptedException { fileSplit = new FileSplit(split.getPath(idx), split.getOffset(idx), split.getLength(idx), split.getLocations()); delegate = inputFormat.createRecordReader(fileSplit, context); }
@Override public void initialize(InputSplit inputSplit, TaskAttemptContext context) throws IOException, InterruptedException { CombineFileSplit combineSplit = (CombineFileSplit)inputSplit; FileSplit split = new FileSplit( combineSplit.getPath(splitIndex), combineSplit.getOffset(splitIndex), combineSplit.getLength(splitIndex), combineSplit.getLocations()); // Initialize with the single FileSplit for the current index delegate.initialize(split, context); }
protected CombineFileRecordReaderWrapper(FileInputFormat<K,V> inputFormat, CombineFileSplit split, TaskAttemptContext context, Integer idx) throws IOException, InterruptedException { fileSplit = new FileSplit(split.getPath(idx), split.getOffset(idx), split.getLength(idx), split.getLocations()); delegate = inputFormat.createRecordReader(fileSplit, context); }
protected CombineFileRecordReaderWrapper(FileInputFormat<K,V> inputFormat, CombineFileSplit split, TaskAttemptContext context, Integer idx) throws IOException, InterruptedException { fileSplit = new FileSplit(split.getPath(idx), split.getOffset(idx), split.getLength(idx), split.getLocations()); delegate = inputFormat.createRecordReader(fileSplit, context); }
@Override public void initialize(InputSplit split, TaskAttemptContext context) throws IOException, InterruptedException { if (delegate != null) { delegate.close(); } if (split instanceof CombineFileSplit) { CombineFileSplit combineSplit = (CombineFileSplit) split; FileSplit fileSplit = new FileSplit(combineSplit.getPath(idx), combineSplit.getOffset(idx), combineSplit.getLength(idx), combineSplit.getLocations()); delegate = getInputFormat().createRecordReader(fileSplit, context); delegate.initialize(fileSplit, context); } else { throw new DatasetOperationException( "Split is not a CombineFileSplit: %s:%s", split.getClass().getCanonicalName(), split); } }
public HDFSSplitIterator(FileSystem fs, Path[] paths, long[] offsets, long[] lengths, long startTime, long endTime) throws IOException { this.fs = fs; this.split = new CombineFileSplit(paths, offsets, lengths, null); while(currentHopIndex < split.getNumPaths() && !fs.exists(split.getPath(currentHopIndex))){ logger.warn(LocalizedMessage.create(LocalizedStrings.HOPLOG_CLEANED_UP_BY_JANITOR, split.getPath(currentHopIndex))); currentHopIndex++; } if(currentHopIndex == split.getNumPaths()){ this.hoplog = null; iterator = null; } else { this.hoplog = getHoplog(fs,split.getPath(currentHopIndex)); iterator = hoplog.getReader().scan(split.getOffset(currentHopIndex), split.getLength(currentHopIndex)); } this.startTime = startTime; this.endTime = endTime; }
public OrcRecordReader(CombineFileSplit split, TaskAttemptContext context, Integer index) throws IOException { this.path = split.getPath(index); this.offset = split.getOffset(index); this.end = offset + split.getLength(index); final Reader reader = OrcFile.createReader(path, OrcFile.readerOptions(context.getConfiguration())); final Reader.Options options = new Reader.Options(); options.range(offset, split.getLength(index)); in = reader.rows(options); schema = reader.getSchema(); this.batch = schema.createRowBatch(); rowInBatch = 0; this.row = (OrcStruct) OrcStruct.createValue(schema); }