@Override public void initialize(InputSplit unusedSplit, TaskAttemptContext cx) throws IOException, InterruptedException { super.initialize( new FileSplit(this.split.getPath(this.idx), this.split.getOffset(this.idx), this.split.getLength(this.idx), null), cx); }
public CustomRecordReader(CombineFileSplit split, TaskAttemptContext context, Integer index) throws IOException{ path = split.getPath(index); fs = path.getFileSystem(context.getConfiguration()); startOffset = split.getOffset(index); endOffset = startOffset + split.getLength(index); fileIn = fs.open(path); reader = new LineReader(fileIn); pos = startOffset; }
private float getOverallLength() { if (combineFileSplit == null) { return 1.0f; } else { return (float) combineFileSplit.getLength(); } }
private long getCurLength() { if (combineFileSplit == null) { return 1L; } else { return combineFileSplit.getLength(idx - 1); } }
/** * return progress based on the amount of data processed so far. */ public float getProgress() throws IOException, InterruptedException { long subprogress = 0; // bytes processed in current split if (null != curReader) { // idx is always one past the current subsplit's true index. subprogress = (long)(curReader.getProgress() * split.getLength(idx - 1)); } return Math.min(1.0f, (progress + subprogress)/(float)(split.getLength())); }
/** * return progress based on the amount of data processed so far. */ public float getProgress() throws IOException, InterruptedException { long subprogress = 0; // bytes processed in current split if (null != curReader) { // idx is always one past the current subsplit's true index. subprogress = (long)(curReader.getProgress() * split.getLength(idx - 1)); } return Math.min(1.0f, (progress + subprogress)/(float)(split.getLength())); }
/** * return progress based on the amount of data processed so far. */ public float getProgress() throws IOException, InterruptedException { long subprogress = 0; // bytes processed in current split if (null != curReader) { // idx is always one past the current subsplit's true index. subprogress = (long)(curReader.getProgress() * split.getLength(idx - 1)); } return Math.min(1.0f, (progress + subprogress)/(float)(split.getLength())); }
/** * return progress based on the amount of data processed so far. */ public float getProgress() throws IOException, InterruptedException { long subprogress = 0; // bytes processed in current split if (null != curReader) { // idx is always one past the current subsplit's true index. subprogress = (long)(curReader.getProgress() * split.getLength(idx - 1)); } return Math.min(1.0f, (progress + subprogress)/(float)(split.getLength())); }
/** * return progress based on the amount of data processed so far. */ public float getProgress() throws IOException, InterruptedException { long subprogress = 0; // bytes processed in current split if (null != curReader) { // idx is always one past the current subsplit's true index. subprogress = (long)(curReader.getProgress() * split.getLength(idx - 1)); } return Math.min(1.0f, (progress + subprogress)/(float)(split.getLength())); }
/** return progress based on the amount of data processed so far. */ public float getProgress() throws IOException, InterruptedException { long subprogress = 0; // bytes processed in current split if (null != curReader) { // idx is always one past the current subsplit's true index. subprogress = (long)(curReader.getProgress() * split.getLength(idx - 1)); } return Math.min(1.0f, (progress + subprogress)/(float)(split.getLength())); }
public CombinePcapRecordReader(CombineFileSplit split, TaskAttemptContext context, Integer index) throws IOException { Path path = split.getPath(index); long start = 0L; long length = split.getLength(index); recordReader = PcapInputFormat.initPcapRecordReader(path, start, length, context); }
@Override public void initialize(InputSplit unusedSplit, TaskAttemptContext cx) throws IOException, InterruptedException { super.initialize( new FileSplit(this.split.getPath(this.idx), this.split.getOffset(this.idx), this.split.getLength(this.idx), null), cx); }
@Override public void initialize(InputSplit split, TaskAttemptContext context) throws IOException, InterruptedException { CombineFileSplit fSplit = (CombineFileSplit) split; super.initialize(new FileSplit(fSplit.getPath(index), 0, fSplit.getLength(index), fSplit.getLocations()), context); } }
protected CombineFileRecordReaderWrapper(FileInputFormat<K,V> inputFormat, CombineFileSplit split, TaskAttemptContext context, Integer idx) throws IOException, InterruptedException { fileSplit = new FileSplit(split.getPath(idx), split.getOffset(idx), split.getLength(idx), split.getLocations()); delegate = inputFormat.createRecordReader(fileSplit, context); }
private InputSplit getDelegateSplit() throws IOException { if (combineFileSplit != null) { return new FileSplit(combineFileSplit.getPath(idx - 1), combineFileSplit.getOffset(idx - 1), combineFileSplit.getLength(idx - 1), combineFileSplit.getLocations()); } else { return crunchSplit.get(); } }
protected CombineFileRecordReaderWrapper(FileInputFormat<K,V> inputFormat, CombineFileSplit split, TaskAttemptContext context, Integer idx) throws IOException, InterruptedException { fileSplit = new FileSplit(split.getPath(idx), split.getOffset(idx), split.getLength(idx), split.getLocations()); delegate = inputFormat.createRecordReader(fileSplit, context); }
@Override public void initialize(InputSplit inputSplit, TaskAttemptContext context) throws IOException, InterruptedException { CombineFileSplit combineSplit = (CombineFileSplit)inputSplit; FileSplit split = new FileSplit( combineSplit.getPath(splitIndex), combineSplit.getOffset(splitIndex), combineSplit.getLength(splitIndex), combineSplit.getLocations()); // Initialize with the single FileSplit for the current index delegate.initialize(split, context); }
@Override public void initialize(InputSplit inputSplit, TaskAttemptContext context) throws IOException, InterruptedException { this.inputSplit = (CombineFileSplit)inputSplit; FileSplit fileSplit = new FileSplit(this.inputSplit.getPath(idx), this.inputSplit.getOffset(idx), this.inputSplit.getLength(idx), this.inputSplit.getLocations()); super.initialize(fileSplit, context); } }
protected CombineFileRecordReaderWrapper(FileInputFormat<K,V> inputFormat, CombineFileSplit split, TaskAttemptContext context, Integer idx) throws IOException, InterruptedException { fileSplit = new FileSplit(split.getPath(idx), split.getOffset(idx), split.getLength(idx), split.getLocations()); delegate = inputFormat.createRecordReader(fileSplit, context); }
protected CombineFileRecordReaderWrapper(FileInputFormat<K,V> inputFormat, CombineFileSplit split, TaskAttemptContext context, Integer idx) throws IOException, InterruptedException { fileSplit = new FileSplit(split.getPath(idx), split.getOffset(idx), split.getLength(idx), split.getLocations()); delegate = inputFormat.createRecordReader(fileSplit, context); }