public void seek(long p) throws IOException { stream.seek(p); pos = p; }
@Override public int read(final ByteBuffer dst, final long position) throws IOException { final int length = Math.min(this.buf.length, dst.remaining()); final FSDataInputStream inputStream = getInputStream(); if (position != this.nextExpectedReadPosition) { System.out.println("Next expected position is " + this.nextExpectedReadPosition + ", seeking to " + position); inputStream.seek(position); this.nextExpectedReadPosition = position; } final int bytesRead = inputStream.read(this.buf, 0, length); if (bytesRead == -1) { return -1; } dst.put(this.buf, 0, length); this.nextExpectedReadPosition += bytesRead; return bytesRead; }
@Override public int read(final ByteBuffer dst, final long position) throws IOException { final int length = Math.min(this.buf.length, dst.remaining()); final FSDataInputStream inputStream = getInputStream(); if (position != this.nextExpectedReadPosition) { System.out.println("Next expected position is " + this.nextExpectedReadPosition + ", seeking to " + position); inputStream.seek(position); this.nextExpectedReadPosition = position; } final int bytesRead = inputStream.read(this.buf, 0, length); if (bytesRead == -1) { return -1; } dst.put(this.buf, 0, length); this.nextExpectedReadPosition += bytesRead; return bytesRead; }
public LineReader(final FSDataInputStream strm, final long start, final long length, final int buffersize) throws IOException { this.stream = strm; this.readBuffer = new byte[buffersize]; this.wrapBuffer = new byte[256]; this.lengthLeft = length; this.readPos = 0; this.overLimit = false; if (start != 0) { strm.seek(start); readLine(); } else { fillBuffer(); } }
public LineReader(final FSDataInputStream strm, final long start, final long length, final int buffersize) throws IOException { this.stream = strm; this.readBuffer = new byte[buffersize]; this.wrapBuffer = new byte[256]; this.lengthLeft = length; this.readPos = 0; this.overLimit = false; if (start != 0) { strm.seek(start); readLine(); } else { fillBuffer(); } }
@Override public void open(FileInputSplit split) throws IOException { // open input split using FileInputFormat super.open(split); // adjust the stream positions for boundary splits int recordOffset = (int) (this.splitStart % this.recordLength); if(recordOffset != 0) { // move start to next boundary super.stream.seek(this.splitStart + recordOffset); } this.streamPos = this.splitStart + recordOffset; this.streamEnd = this.splitStart + this.splitLength; this.streamEnd += this.streamEnd % this.recordLength; // adjust readBufferSize this.readBufferSize += this.recordLength - (this.readBufferSize % this.recordLength); if (this.readBuffer == null || this.readBuffer.length != this.readBufferSize) { this.readBuffer = new byte[this.readBufferSize]; } this.readBufferLimit = 0; this.readBufferPos = 0; this.exhausted = false; fillReadBuffer(); }
@Override public void open(FileInputSplit split) throws IOException { super.open(split); final long blockSize = this.blockSize == NATIVE_BLOCK_SIZE ? this.filePath.getFileSystem().getDefaultBlockSize() : this.blockSize; this.blockInfo = this.createBlockInfo(); if (this.splitLength > this.blockInfo.getInfoSize()) { // TODO: seek not supported by compressed streams. Will throw exception this.stream.seek(this.splitStart + this.splitLength - this.blockInfo.getInfoSize()); DataInputStream infoStream = new DataInputStream(this.stream); this.blockInfo.read(infoStream); } this.stream.seek(this.splitStart + this.blockInfo.getFirstRecordStart()); this.blockBasedInput = new BlockBasedInput(this.stream, (int) blockSize); this.dataInputStream = new DataInputStream(this.blockBasedInput); this.readRecords = 0; }
@Override public void open(FileInputSplit split) throws IOException { super.open(split); final long blockSize = this.blockSize == NATIVE_BLOCK_SIZE ? this.filePath.getFileSystem().getDefaultBlockSize() : this.blockSize; this.blockInfo = this.createBlockInfo(); if (this.splitLength > this.blockInfo.getInfoSize()) { // TODO: seek not supported by compressed streams. Will throw exception this.stream.seek(this.splitStart + this.splitLength - this.blockInfo.getInfoSize()); DataInputStream infoStream = new DataInputStream(this.stream); this.blockInfo.read(infoStream); } this.stream.seek(this.splitStart + this.blockInfo.getFirstRecordStart()); this.blockBasedInput = new BlockBasedInput(this.stream, (int) blockSize); this.dataInputStream = new DataInputStream(this.blockBasedInput); this.readRecords = 0; }
this.stream.seek(this.splitStart); readLine();
@Override public void open(final FileInputSplit split) throws IOException { super.open(split); final long blockSize = this.blockSize == NATIVE_BLOCK_SIZE ? this.filePath.getFileSystem().getDefaultBlockSize() : this.blockSize; this.blockInfo = this.createBlockInfo(); if (this.splitLength > this.blockInfo.getInfoSize()) { this.stream.seek(this.splitStart + this.splitLength - this.blockInfo.getInfoSize()); final DataInputStream infoStream = new DataInputStream(this.stream); this.blockInfo.read(infoStream); } this.stream.seek(this.splitStart + this.blockInfo.getFirstRecordStart()); this.blockBasedInput = new BlockBasedInput(this.stream, (int) blockSize); this.dataInputStream = new DataInputViewStream(this.blockBasedInput); this.readRecords = 0; }
this.stream.seek(this.splitStart); readLine();
this.stream.seek(this.splitStart);
this.stream.seek(this.splitStart);
/** * Fill in the statistics. The last modification time and the total input size are prefilled. * * @param files * The files that are associated with this block input format. * @param stats * The pre-filled statistics. */ protected SequentialStatistics createStatistics(final List<FileStatus> files, final FileBaseStatistics stats) throws IOException { if (files.isEmpty()) return null; final BlockInfo blockInfo = this.createBlockInfo(); long totalCount = 0; for (final FileStatus file : files) { // invalid file if (file.getLen() < blockInfo.getInfoSize()) continue; final FSDataInputStream fdis = file.getPath().getFileSystem().open(file.getPath(), blockInfo.getInfoSize()); fdis.seek(file.getLen() - blockInfo.getInfoSize()); final DataInputStream input = new DataInputStream(fdis); blockInfo.read(input); totalCount += blockInfo.getAccumulatedRecordCount(); } final float avgWidth = totalCount == 0 ? 0 : (float) stats.getTotalInputSize() / totalCount; return new SequentialStatistics(stats.getLastModificationTime(), stats.getTotalInputSize(), avgWidth, totalCount); }
/** * Fill in the statistics. The last modification time and the total input size are prefilled. * * @param files * The files that are associated with this block input format. * @param stats * The pre-filled statistics. */ protected SequentialStatistics createStatistics(List<FileStatus> files, FileBaseStatistics stats) throws IOException { if (files.isEmpty()) { return null; } BlockInfo blockInfo = this.createBlockInfo(); long totalCount = 0; for (FileStatus file : files) { // invalid file if (file.getLen() < blockInfo.getInfoSize()) { continue; } FSDataInputStream fdis = file.getPath().getFileSystem().open(file.getPath(), blockInfo.getInfoSize()); fdis.seek(file.getLen() - blockInfo.getInfoSize()); DataInputStream input = new DataInputStream(fdis); blockInfo.read(input); totalCount += blockInfo.getAccumulatedRecordCount(); } final float avgWidth = totalCount == 0 ? 0 : ((float) stats.getTotalInputSize() / totalCount); return new SequentialStatistics(stats.getLastModificationTime(), stats.getTotalInputSize(), avgWidth, totalCount); }
/** * Fill in the statistics. The last modification time and the total input size are prefilled. * * @param files * The files that are associated with this block input format. * @param stats * The pre-filled statistics. */ protected SequentialStatistics createStatistics(List<FileStatus> files, FileBaseStatistics stats) throws IOException { if (files.isEmpty()) { return null; } BlockInfo blockInfo = this.createBlockInfo(); long totalCount = 0; for (FileStatus file : files) { // invalid file if (file.getLen() < blockInfo.getInfoSize()) { continue; } FSDataInputStream fdis = file.getPath().getFileSystem().open(file.getPath(), blockInfo.getInfoSize()); fdis.seek(file.getLen() - blockInfo.getInfoSize()); DataInputStream input = new DataInputStream(fdis); blockInfo.read(input); totalCount += blockInfo.getAccumulatedRecordCount(); } final float avgWidth = totalCount == 0 ? 0 : ((float) stats.getTotalInputSize() / totalCount); return new SequentialStatistics(stats.getLastModificationTime(), stats.getTotalInputSize(), avgWidth, totalCount); }