ValueBytes rawValue = (count == keyOffsets.length || rawValues[count] == null) ? in.createValueBytes() : rawValues[count]; int recordLength = in.nextRaw(rawKeys, rawValue);
public int readMarker(SequenceFile.Reader reader) throws IOException { if (valueBytes == null) { valueBytes = reader.createValueBytes(); } rawKey.reset(); rawValue.reset(); // valueBytes need not be reset since nextRaw call does it (and it is a private method) int status = reader.nextRaw(rawKey, valueBytes); // if we reach EOF, return -1 if (status == -1) { return -1; } // Check if the marker key is valid and return the count if (isMarkerValid()) { valueBytes.writeUncompressedBytes(rawValue); rawValue.flush(); // rawValue.getData() may return a larger byte array but Ints.fromByteArray will only read the first four bytes return Ints.fromByteArray(rawValue.getData()); } // EOF not reached and marker is not valid, then thrown an IOException since we can't make progress throw new IOException(String.format("Invalid key for num entries appended found %s, expected : %s", new String(rawKey.getData()), TxConstants.TransactionLog.NUM_ENTRIES_APPENDED)); }
public int readMarker(SequenceFile.Reader reader) throws IOException { if (valueBytes == null) { valueBytes = reader.createValueBytes(); } rawKey.reset(); rawValue.reset(); // valueBytes need not be reset since nextRaw call does it (and it is a private method) int status = reader.nextRaw(rawKey, valueBytes); // if we reach EOF, return -1 if (status == -1) { return -1; } // Check if the marker key is valid and return the count if (isMarkerValid()) { valueBytes.writeUncompressedBytes(rawValue); rawValue.flush(); // rawValue.getData() may return a larger byte array but Ints.fromByteArray will only read the first four bytes return Ints.fromByteArray(rawValue.getData()); } // EOF not reached and marker is not valid, then thrown an IOException since we can't make progress throw new IOException(String.format("Invalid key for num entries appended found %s, expected : %s", new String(rawKey.getData()), TxConstants.TransactionLog.NUM_ENTRIES_APPENDED)); }
public boolean next() throws IOException { if (size() == 0) return false; if (minSegment != null) { //minSegment is non-null for all invocations of next except the first //one. For the first invocation, the priority queue is ready for use //but for the subsequent invocations, first adjust the queue adjustPriorityQueue(minSegment); if (size() == 0) { minSegment = null; return false; } } minSegment = (SegmentDescriptor)top(); long startPos = minSegment.in.getPosition(); // Current position in stream //save the raw key reference rawKey = minSegment.getKey(); //load the raw value. Re-use the existing rawValue buffer if (rawValue == null) { rawValue = minSegment.in.createValueBytes(); } minSegment.nextRawValue(rawValue); long endPos = minSegment.in.getPosition(); // End position after reading value updateProgress(endPos - startPos); return true; }
@Override public boolean next() throws IOException { if (size() == 0) return false; if (minSegment != null) { //minSegment is non-null for all invocations of next except the first //one. For the first invocation, the priority queue is ready for use //but for the subsequent invocations, first adjust the queue adjustPriorityQueue(minSegment); if (size() == 0) { minSegment = null; return false; } } minSegment = (SegmentDescriptor)top(); long startPos = minSegment.in.getPosition(); // Current position in stream //save the raw key reference rawKey = minSegment.getKey(); //load the raw value. Re-use the existing rawValue buffer if (rawValue == null) { rawValue = minSegment.in.createValueBytes(); } minSegment.nextRawValue(rawValue); long endPos = minSegment.in.getPosition(); // End position after reading value updateProgress(endPos - startPos); return true; }
public boolean next() throws IOException { if (size() == 0) return false; if (minSegment != null) { //minSegment is non-null for all invocations of next except the first //one. For the first invocation, the priority queue is ready for use //but for the subsequent invocations, first adjust the queue adjustPriorityQueue(minSegment); if (size() == 0) { minSegment = null; return false; } } minSegment = (SegmentDescriptor)top(); long startPos = minSegment.in.getPosition(); // Current position in stream //save the raw key reference rawKey = minSegment.getKey(); //load the raw value. Re-use the existing rawValue buffer if (rawValue == null) { rawValue = minSegment.in.createValueBytes(); } minSegment.nextRawValue(rawValue); long endPos = minSegment.in.getPosition(); // End position after reading value updateProgress(endPos - startPos); return true; }
public int readMarker(SequenceFile.Reader reader) throws IOException { if (valueBytes == null) { valueBytes = reader.createValueBytes(); } rawKey.reset(); rawValue.reset(); // valueBytes need not be reset since nextRaw call does it (and it is a private method) int status = reader.nextRaw(rawKey, valueBytes); // if we reach EOF, return -1 if (status == -1) { return -1; } // Check if the marker key is valid and return the count if (isMarkerValid()) { valueBytes.writeUncompressedBytes(rawValue); rawValue.flush(); // rawValue.getData() may return a larger byte array but Ints.fromByteArray will only read the first four bytes return Ints.fromByteArray(rawValue.getData()); } // EOF not reached and marker is not valid, then thrown an IOException since we can't make progress throw new IOException(String.format("Invalid key for num entries appended found %s, expected : %s", new String(rawKey.getData()), TxConstants.TransactionLog.NUM_ENTRIES_APPENDED)); }
public void initialize(InputSplit split, TaskAttemptContext context) throws IOException, InterruptedException { Path path = ((FileSplit)split).getPath(); Configuration conf = context.getConfiguration(); FileSystem fs = path.getFileSystem(conf); this.in = new SequenceFile.Reader(fs, path, conf); this.end = ((FileSplit)split).getStart() + split.getLength(); if (((FileSplit)split).getStart() > in.getPosition()) { in.sync(((FileSplit)split).getStart()); // sync to start } this.start = in.getPosition(); vbytes = in.createValueBytes(); done = start >= end; }
public void initialize(InputSplit split, TaskAttemptContext context) throws IOException, InterruptedException { Path path = ((FileSplit)split).getPath(); Configuration conf = context.getConfiguration(); FileSystem fs = path.getFileSystem(conf); this.in = new SequenceFile.Reader(fs, path, conf); this.end = ((FileSplit)split).getStart() + split.getLength(); if (((FileSplit)split).getStart() > in.getPosition()) { in.sync(((FileSplit)split).getStart()); // sync to start } this.start = in.getPosition(); vbytes = in.createValueBytes(); done = start >= end; }
public void initialize(InputSplit split, TaskAttemptContext context) throws IOException, InterruptedException { Path path = ((FileSplit)split).getPath(); Configuration conf = context.getConfiguration(); FileSystem fs = path.getFileSystem(conf); this.in = new SequenceFile.Reader(fs, path, conf); this.end = ((FileSplit)split).getStart() + split.getLength(); if (((FileSplit)split).getStart() > in.getPosition()) { in.sync(((FileSplit)split).getStart()); // sync to start } this.start = in.getPosition(); vbytes = in.createValueBytes(); done = start >= end; }
public void initialize(InputSplit split, TaskAttemptContext context) throws IOException, InterruptedException { Path path = ((FileSplit)split).getPath(); Configuration conf = context.getConfiguration(); FileSystem fs = path.getFileSystem(conf); this.in = new SequenceFile.Reader(fs, path, conf); this.end = ((FileSplit)split).getStart() + split.getLength(); if (((FileSplit)split).getStart() > in.getPosition()) { in.sync(((FileSplit)split).getStart()); // sync to start } this.start = in.getPosition(); vbytes = in.createValueBytes(); done = start >= end; }
public void initialize(InputSplit split, TaskAttemptContext context) throws IOException, InterruptedException { Path path = ((FileSplit)split).getPath(); Configuration conf = context.getConfiguration(); FileSystem fs = path.getFileSystem(conf); this.in = new SequenceFile.Reader(fs, path, conf); this.end = ((FileSplit)split).getStart() + split.getLength(); if (((FileSplit)split).getStart() > in.getPosition()) { in.sync(((FileSplit)split).getStart()); // sync to start } this.start = in.getPosition(); vbytes = in.createValueBytes(); done = start >= end; }
public SequenceFileAsBinaryRecordReader(Configuration conf, FileSplit split) throws IOException { Path path = split.getPath(); FileSystem fs = path.getFileSystem(conf); this.in = new SequenceFile.Reader(fs, path, conf); this.end = split.getStart() + split.getLength(); if (split.getStart() > in.getPosition()) in.sync(split.getStart()); // sync to start this.start = in.getPosition(); vbytes = in.createValueBytes(); done = start >= end; }
public SequenceFileAsBinaryRecordReader(Configuration conf, FileSplit split) throws IOException { Path path = split.getPath(); FileSystem fs = path.getFileSystem(conf); this.in = new SequenceFile.Reader(fs, path, conf); this.end = split.getStart() + split.getLength(); if (split.getStart() > in.getPosition()) in.sync(split.getStart()); // sync to start this.start = in.getPosition(); vbytes = in.createValueBytes(); done = start >= end; }
public SequenceFileAsBinaryRecordReader(Configuration conf, FileSplit split) throws IOException { Path path = split.getPath(); FileSystem fs = path.getFileSystem(conf); this.in = new SequenceFile.Reader(fs, path, conf); this.end = split.getStart() + split.getLength(); if (split.getStart() > in.getPosition()) in.sync(split.getStart()); // sync to start this.start = in.getPosition(); vbytes = in.createValueBytes(); done = start >= end; }
public SequenceFileAsBinaryRecordReader(Configuration conf, FileSplit split) throws IOException { Path path = split.getPath(); FileSystem fs = path.getFileSystem(conf); this.in = new SequenceFile.Reader(fs, path, conf); this.end = split.getStart() + split.getLength(); if (split.getStart() > in.getPosition()) in.sync(split.getStart()); // sync to start this.start = in.getPosition(); vbytes = in.createValueBytes(); done = start >= end; }
public SequenceFileAsBinaryRecordReader(Configuration conf, FileSplit split) throws IOException { Path path = split.getPath(); FileSystem fs = path.getFileSystem(conf); this.in = new SequenceFile.Reader(fs, path, conf); this.end = split.getStart() + split.getLength(); if (split.getStart() > in.getPosition()) in.sync(split.getStart()); // sync to start this.start = in.getPosition(); vbytes = in.createValueBytes(); done = start >= end; }
public SequenceFileAsBinaryRecordReader(Configuration conf, FileSplit split) throws IOException { Path path = split.getPath(); FileSystem fs = path.getFileSystem(conf); this.in = new SequenceFile.Reader(fs, path, conf); this.end = split.getStart() + split.getLength(); if (split.getStart() > in.getPosition()) in.sync(split.getStart()); // sync to start this.start = in.getPosition(); vbytes = in.createValueBytes(); done = start >= end; }
public SequenceFileAsBinaryRecordReader(Configuration conf, FileSplit split) throws IOException { Path path = split.getPath(); FileSystem fs = path.getFileSystem(conf); this.in = new SequenceFile.Reader(fs, path, conf); this.end = split.getStart() + split.getLength(); if (split.getStart() > in.getPosition()) in.sync(split.getStart()); // sync to start this.start = in.getPosition(); vbytes = in.createValueBytes(); done = start >= end; }
@Override public boolean next() throws IOException { if (size() == 0) return false; if (minSegment != null) { //minSegment is non-null for all invocations of next except the first //one. For the first invocation, the priority queue is ready for use //but for the subsequent invocations, first adjust the queue adjustPriorityQueue(minSegment); if (size() == 0) { minSegment = null; return false; } } minSegment = (SegmentDescriptor)top(); long startPos = minSegment.in.getPosition(); // Current position in stream //save the raw key reference rawKey = minSegment.getKey(); //load the raw value. Re-use the existing rawValue buffer if (rawValue == null) { rawValue = minSegment.in.createValueBytes(); } minSegment.nextRawValue(rawValue); long endPos = minSegment.in.getPosition(); // End position after reading value updateProgress(endPos - startPos); return true; }