@Override public void seek(long desired) throws IOException { inputStream.seek(desired); }
@Override public void seek(long p) throws IOException { stream.seek(p); }
@Override public void seek(long desired) throws IOException { try { originalStream.seek(desired); } catch (IOException e) { handleIOException(e); } }
@Override public void seek(long desired) throws IOException { original.seek(desired); }
@Override public void flatMap(Tuple3<String, Long, Long> value, Collector<String> out) throws Exception { FSDataInputStream stream = FileSystem.get(new URI(value.f0)).open(new Path(value.f0)); stream.seek(value.f1); BufferedReader reader = new BufferedReader(new InputStreamReader(stream)); String line; try { while ((line = reader.readLine()) != null && (value.f2 == -1L || stream.getPos() <= value.f2)) { out.collect(line); } } finally { reader.close(); } } }
@Override public KeyGroupStatePartitionStreamProvider next() { if (!hasNext()) { throw new NoSuchElementException("Iterator exhausted"); } Tuple2<Integer, Long> keyGroupOffset = currentOffsetsIterator.next(); try { if (null == currentStream) { openCurrentStream(); } currentStream.seek(keyGroupOffset.f1); return new KeyGroupStatePartitionStreamProvider(currentStream, keyGroupOffset.f0); } catch (IOException ioex) { return new KeyGroupStatePartitionStreamProvider(ioex, keyGroupOffset.f0); } } }
@Override public StatePartitionStreamProvider next() { if (!hasNext()) { throw new NoSuchElementException("Iterator exhausted"); } long offset = offsets[offPos++]; try { if (null == currentStream) { openCurrentStream(); } currentStream.seek(offset); return new StatePartitionStreamProvider(currentStream); } catch (IOException ioex) { return new StatePartitionStreamProvider(ioex); } } }
private BlockInfo createAndReadBlockInfo() throws IOException { BlockInfo blockInfo = new BlockInfo(); if (this.splitLength > blockInfo.getInfoSize()) { // At first we go and read the block info containing the recordCount, the accumulatedRecordCount // and the firstRecordStart offset in the current block. This is written at the end of the block and // is of fixed size, currently 3 * Long.SIZE. // TODO: seek not supported by compressed streams. Will throw exception this.stream.seek(this.splitStart + this.splitLength - blockInfo.getInfoSize()); blockInfo.read(new DataInputViewStreamWrapper(this.stream)); } return blockInfo; }
this.stream.seek(this.splitStart);
/** * Opens the given input split. This method opens the input stream to the specified file, allocates read buffers * and positions the stream at the correct position, making sure that any partial record at the beginning is skipped. * * @param split The input split to open. * * @see org.apache.flink.api.common.io.FileInputFormat#open(org.apache.flink.core.fs.FileInputSplit) */ @Override public void open(FileInputSplit split) throws IOException { super.open(split); initBuffers(); this.offset = splitStart; if (this.splitStart != 0) { this.stream.seek(offset); readLine(); // if the first partial record already pushes the stream over // the limit of our split, then no record starts within this split if (this.overLimit) { this.end = true; } } else { fillBuffer(0); } }
@Override public void open(FileInputSplit split) throws IOException { super.open(split); this.blockInfo = this.createAndReadBlockInfo(); // We set the size of the BlockBasedInput to splitLength as each split contains one block. // After reading the block info, we seek in the file to the correct position. this.readRecords = 0; this.stream.seek(this.splitStart + this.blockInfo.getFirstRecordStart()); this.blockBasedInput = new BlockBasedInput(this.stream, (int) blockInfo.getFirstRecordStart(), this.splitLength); this.dataInputStream = new DataInputViewStreamWrapper(blockBasedInput); }
@PublicEvolving @Override public void reopen(FileInputSplit split, Tuple2<Long, Long> state) throws IOException { Preconditions.checkNotNull(split, "reopen() cannot be called on a null split."); Preconditions.checkNotNull(state, "reopen() cannot be called with a null initial state."); try { this.open(split); } finally { this.blockInfo = this.createAndReadBlockInfo(); long blockPos = state.f0; this.readRecords = state.f1; this.stream.seek(this.splitStart + blockPos); this.blockBasedInput = new BlockBasedInput(this.stream, (int) blockPos, this.splitLength); this.dataInputStream = new DataInputViewStreamWrapper(blockBasedInput); } } }
currentStateHandleInStream.seek(offset); try (InputStream compressedKgIn = keygroupStreamCompressionDecorator.decorateWithCompression(currentStateHandleInStream)) { DataInputViewStreamWrapper compressedKgInputView = new DataInputViewStreamWrapper(compressedKgIn);
fdis.seek(file.getLen() - blockInfo.getInfoSize());
initBuffers(); this.stream.seek(this.offset); if (split.getLength() == -1) {
@Override public void seek(long desired) throws IOException { try { originalStream.seek(desired); } catch (IOException e) { handleIOException(e); } }
@Override public void reopen(FileInputSplit split, Long state) throws IOException { try { this.open(split); } finally { this.stream.seek(state); } }
private BlockInfo createAndReadBlockInfo() throws IOException { BlockInfo blockInfo = new BlockInfo(); if (this.splitLength > blockInfo.getInfoSize()) { // At first we go and read the block info containing the recordCount, the accumulatedRecordCount // and the firstRecordStart offset in the current block. This is written at the end of the block and // is of fixed size, currently 3 * Long.SIZE. // TODO: seek not supported by compressed streams. Will throw exception this.stream.seek(this.splitStart + this.splitLength - blockInfo.getInfoSize()); blockInfo.read(new DataInputViewStreamWrapper(this.stream)); } return blockInfo; }
@Override public void open(FileInputSplit split) throws IOException { super.open(split); this.blockInfo = this.createAndReadBlockInfo(); // We set the size of the BlockBasedInput to splitLength as each split contains one block. // After reading the block info, we seek in the file to the correct position. this.readRecords = 0; this.stream.seek(this.splitStart + this.blockInfo.getFirstRecordStart()); this.blockBasedInput = new BlockBasedInput(this.stream, (int) blockInfo.getFirstRecordStart(), this.splitLength); this.dataInputStream = new DataInputViewStreamWrapper(blockBasedInput); }