@Override public int readLine(Text str, int maxLineLength, int maxBytesToConsume) throws IOException { int bytesRead = 0; if (!finished) { // only allow at most one more record to be read after the stream // reports the split ended if (totalBytesRead > splitLength) { finished = true; } bytesRead = super.readLine(str, maxLineLength, maxBytesToConsume); } return bytesRead; }
public LineRecordReader(InputStream in, long offset, long endOffset, int maxLineLength, byte[] recordDelimiter) { this.maxLineLength = maxLineLength; this.in = new SplitLineReader(in, recordDelimiter); this.start = offset; this.pos = offset; this.end = endOffset; filePosition = null; }
public synchronized void close() throws IOException { try { if(in != null) { in.close(); } } finally { if(decompressor != null) { CodecPool.returnDecompressor(decompressor); } } } }
filePosition = cIn; } else { in = new SplitLineReader(codec.createInputStream(fileIn, decompressor), job, this.recordDelimiterBytes); filePosition = fileIn; in = new SplitLineReader(fileIn, job, this.recordDelimiterBytes); filePosition = fileIn; newSize = skipUtfByteOrderMark(); } else { newSize = in.readLine(value, 0, maxBytesToConsume(pos)); pos += newSize;
/** Read a line. */ public synchronized boolean next(LongWritable key, Text value) throws IOException { // We always read one extra line, which lies outside the upper // split limit i.e. (end - 1) while (getFilePosition() <= end || in.needAdditionalRecordAfterSplit()) { key.set(pos); int newSize = 0; if (pos == 0) { newSize = skipUtfByteOrderMark(value); } else { newSize = in.readLine(value, maxLineLength, maxBytesToConsume(pos)); pos += newSize; } if (newSize == 0) { return false; } if (newSize < maxLineLength) { return true; } // line too long. try again LOG.info("Skipped line of size " + newSize + " at pos " + (pos - newSize)); } return false; }
filePosition = cIn; } else { in = new SplitLineReader(codec.createInputStream(fileIn, decompressor), job, this.recordDelimiterBytes); filePosition = fileIn; in = new SplitLineReader(fileIn, job, this.recordDelimiterBytes); filePosition = fileIn; start += in.readLine(new Text(), 0, maxBytesToConsume(start));
while (getFilePosition() <= end || in.needAdditionalRecordAfterSplit()) { if (pos == 0) { newSize = skipUtfByteOrderMark(); } else { newSize = in.readLine(value, maxLineLength, maxBytesToConsume(pos)); pos += newSize;
filePosition = cIn; } else { in = new SplitLineReader(codec.createInputStream(fileIn, decompressor), job, this.recordDelimiterBytes); filePosition = fileIn; in = new SplitLineReader(fileIn, job, this.recordDelimiterBytes); filePosition = fileIn; newSize = skipUtfByteOrderMark(); } else { newSize = in.readLine(value, 0, maxBytesToConsume(pos)); pos += newSize;
/** Read a line. */ public synchronized boolean next(LongWritable key, Text value) throws IOException { // We always read one extra line, which lies outside the upper // split limit i.e. (end - 1) while (getFilePosition() <= end || in.needAdditionalRecordAfterSplit()) { key.set(pos); int newSize = 0; if (pos == 0) { newSize = skipUtfByteOrderMark(value); } else { newSize = in.readLine(value, maxLineLength, maxBytesToConsume(pos)); pos += newSize; } if (newSize == 0) { return false; } if (newSize < maxLineLength) { return true; } // line too long. try again LOG.info("Skipped line of size " + newSize + " at pos " + (pos - newSize)); } return false; }
@Override public int readLine(Text str, int maxLineLength, int maxBytesToConsume) throws IOException { int bytesRead = 0; if (!finished) { // only allow at most one more record to be read after the stream // reports the split ended if (totalBytesRead > splitLength) { finished = true; } bytesRead = super.readLine(str, maxLineLength, maxBytesToConsume); } return bytesRead; }
public synchronized void close() throws IOException { try { if (in != null) { in.close(); } } finally { if (decompressor != null) { CodecPool.returnDecompressor(decompressor); decompressor = null; } } } }
public LineRecordReader(InputStream in, long offset, long endOffset, int maxLineLength, byte[] recordDelimiter) { this.maxLineLength = maxLineLength; this.in = new SplitLineReader(in, recordDelimiter); this.start = offset; this.pos = offset; this.end = endOffset; filePosition = null; }
filePosition = cIn; } else { in = new SplitLineReader(codec.createInputStream(fileIn, decompressor), job, this.recordDelimiterBytes); filePosition = fileIn; in = new SplitLineReader(fileIn, job, this.recordDelimiterBytes); filePosition = fileIn; start += in.readLine(new Text(), 0, maxBytesToConsume(start));
while (getFilePosition() <= end || in.needAdditionalRecordAfterSplit()) { if (pos == 0) { newSize = skipUtfByteOrderMark(); } else { newSize = in.readLine(value, maxLineLength, maxBytesToConsume(pos)); pos += newSize;
@Override public int readLine(Text str, int maxLineLength, int maxBytesToConsume) throws IOException { int bytesRead = 0; if (!finished) { // only allow at most one more record to be read after the stream // reports the split ended if (totalBytesRead > splitLength) { finished = true; } bytesRead = super.readLine(str, maxLineLength, maxBytesToConsume); } return bytesRead; }
public synchronized void close() throws IOException { try { if (in != null) { in.close(); } } finally { if (decompressor != null) { CodecPool.returnDecompressor(decompressor); decompressor = null; } } } }
public LineRecordReader(InputStream in, long offset, long endOffset, int maxLineLength, byte[] recordDelimiter) { this.maxLineLength = maxLineLength; this.in = new SplitLineReader(in, recordDelimiter); this.start = offset; this.pos = offset; this.end = endOffset; filePosition = null; }
filePosition = cIn; } else { in = new SplitLineReader(codec.createInputStream(fileIn, decompressor), job, this.recordDelimiterBytes); filePosition = fileIn; in = new SplitLineReader(fileIn, job, this.recordDelimiterBytes); filePosition = fileIn; start += in.readLine(new Text(), 0, maxBytesToConsume(start));
while (getFilePosition() <= end || in.needAdditionalRecordAfterSplit()) { if (pos == 0) { newSize = skipUtfByteOrderMark(); } else { newSize = in.readLine(value, maxLineLength, maxBytesToConsume(pos)); pos += newSize;
@Override public int readLine(Text str, int maxLineLength, int maxBytesToConsume) throws IOException { int bytesRead = 0; if (!finished) { // only allow at most one more record to be read after the stream // reports the split ended if (totalBytesRead > splitLength) { finished = true; } bytesRead = super.readLine(str, maxLineLength, maxBytesToConsume); } return bytesRead; }