@Override protected void startReading(ReadableByteChannel channel) throws IOException { this.inChannel = channel; // If the first offset is greater than zero, we need to skip bytes until we see our // first delimiter. long startOffset = getCurrentSource().getStartOffset(); if (startOffset > 0) { checkState( channel instanceof SeekableByteChannel, "%s only supports reading from a SeekableByteChannel when given a start offset" + " greater than 0.", TextSource.class.getSimpleName()); long requiredPosition = startOffset - 1; if (delimiter != null && startOffset >= delimiter.length) { // we need to move back the offset of at worse delimiter.size to be sure to see // all the bytes of the delimiter in the call to findDelimiterBounds() below requiredPosition = startOffset - delimiter.length; } ((SeekableByteChannel) channel).position(requiredPosition); findDelimiterBounds(); buffer = buffer.substring(endOfDelimiterInBuffer); startOfNextRecord = requiredPosition + endOfDelimiterInBuffer; endOfDelimiterInBuffer = 0; startOfDelimiterInBuffer = 0; } }
int bytePositionInBuffer = 0; while (true) { if (!tryToEnsureNumberOfBytesInBuffer(bytePositionInBuffer + 1)) { startOfDelimiterInBuffer = endOfDelimiterInBuffer = bytePositionInBuffer; break; endOfDelimiterInBuffer = startOfDelimiterInBuffer + 1; if (tryToEnsureNumberOfBytesInBuffer(bytePositionInBuffer + 2)) { currentByte = buffer.byteAt(bytePositionInBuffer + 1); if (currentByte == '\n') { if (tryToEnsureNumberOfBytesInBuffer(bytePositionInBuffer + i + 1)) { currentByte = buffer.byteAt(bytePositionInBuffer + i); } else {
@Override protected boolean readNextRecord() throws IOException { startOfRecord = startOfNextRecord; findDelimiterBounds(); // If we have reached EOF file and consumed all of the buffer then we know // that there are no more records. if (eof && buffer.isEmpty()) { elementIsPresent = false; return false; } decodeCurrentElement(); startOfNextRecord = startOfRecord + endOfDelimiterInBuffer; return true; }
@Override protected FileBasedReader<String> createSingleFileReader(PipelineOptions options) { return new TextBasedReader(this, delimiter); }
@Override public long getSplitPointsRemaining() { if (isStarted() && startOfNextRecord >= getCurrentSource().getEndOffset()) { return isDone() ? 0 : 1; } return super.getSplitPointsRemaining(); }