@Override public boolean next(K key, V value) throws IOException { if(!initDone) { throw new IOException("Hive IOContext is not inited."); } updateIOContext(); try { boolean retVal = doNext(key, value); if(retVal) { if(key instanceof RecordIdentifier) { //supports AcidInputFormat which uses the KEY pass ROW__ID info ioCxtRef.setRecordIdentifier((RecordIdentifier)key); } else if(recordReader instanceof AcidInputFormat.AcidRecordReader) { //supports AcidInputFormat which do not use the KEY pass ROW__ID info ioCxtRef.setRecordIdentifier(((AcidInputFormat.AcidRecordReader) recordReader).getRecordIdentifier()); } } return retVal; } catch (IOException e) { ioCxtRef.setIOExceptions(true); throw e; } }
public void initIOContext(FileSplit split, JobConf job, Class inputFormatClass) throws IOException { this.initIOContext(split, job, inputFormatClass, null); }
@Override public void close() throws IOException { doClose(); initDone = false; ioCxtRef = null; }
/** * This should be called after the binary search is finished and before the linear scan begins * @throws IOException */ private void beginLinearSearch() throws IOException { sync(rangeStart); this.getIOContext().setBinarySearching(false); this.wasUsingSortedSearch = false; }
stopComparisons.add(Comparison.EQUAL); stopComparisons.add(Comparison.GREATER); if (this.getIOContext().isBinarySearching()) { beginLinearSearch(); if (this.getIOContext().isBinarySearching()) { beginLinearSearch(); if (this.getIOContext().isBinarySearching()) { beginLinearSearch();
public boolean doNext(K key, V value) throws IOException { if (this.isSorted) { if (this.getIOContext().shouldEndBinarySearch() || (!this.getIOContext().useSorted() && this.wasUsingSortedSearch)) { beginLinearSearch(); this.wasUsingSortedSearch = false; this.getIOContext().setEndBinarySearch(false); if (this.getIOContext().useSorted()) { if (this.genericUDFClassName == null && this.getIOContext().getGenericUDFClassName() != null) { setGenericUDFClassName(this.getIOContext().getGenericUDFClassName()); if (this.getIOContext().isBinarySearching()) { if (this.getIOContext().getComparison() != null) { switch (this.getIOContext().getComparison()) { case GREATER: case EQUAL: sync(position); long newPosition = getSyncedPosition(); this.getIOContext().setBinarySearching(false); sync(rangeStart); } else if (foundAllTargets()) {
private boolean doNextWithExceptionHandler(K key, V value) throws IOException { return super.doNext(key, value); }
/** * Returns true if the current comparison is in the list of stop comparisons, i.e. we've found * all records which won't be filtered * @return true if the current comparison is found */ public boolean foundAllTargets() { if (this.getIOContext().getComparison() == null || !stopComparisons.contains(this.getIOContext().getComparison())) { return false; } return true; } }
public void initIOContext(FileSplit split, JobConf job, Class inputFormatClass, RecordReader recordReader) throws IOException { boolean blockPointer = false; long blockStart = -1; FileSplit fileSplit = split; Path path = fileSplit.getPath(); FileSystem fs = path.getFileSystem(job); if (inputFormatClass.getName().contains("SequenceFile")) { SequenceFile.Reader in = new SequenceFile.Reader(fs, path, job); blockPointer = in.isBlockCompressed(); in.sync(fileSplit.getStart()); blockStart = in.getPosition(); in.close(); } else if (recordReader instanceof RCFileRecordReader) { blockPointer = true; blockStart = ((RCFileRecordReader) recordReader).getStart(); } else if (inputFormatClass.getName().contains("RCFile")) { blockPointer = true; RCFile.Reader in = new RCFile.Reader(fs, path, job); in.sync(fileSplit.getStart()); blockStart = in.getPosition(); in.close(); } this.jobConf = job; this.initIOContext(blockStart, blockPointer, path.makeQualified(fs)); this.initIOContextSortedProps(split, recordReader, job); }
protected void updateIOContext() throws IOException { long pointerPos = this.getPos(); if (!ioCxtRef.isBlockPointer()) { ioCxtRef.setCurrentBlockStart(pointerPos); ioCxtRef.setCurrentRow(0); return; } ioCxtRef.setCurrentRow(ioCxtRef.getCurrentRow() + 1); if (ioCxtRef.getNextBlockStart() == -1) { ioCxtRef.setNextBlockStart(pointerPos); ioCxtRef.setCurrentRow(0); } if (pointerPos != ioCxtRef.getNextBlockStart()) { // the reader pointer has moved to the end of next block, or the end of // current record. ioCxtRef.setCurrentRow(0); if (ioCxtRef.getCurrentBlockStart() == ioCxtRef.getNextBlockStart()) { ioCxtRef.setCurrentRow(1); } ioCxtRef.setCurrentBlockStart(ioCxtRef.getNextBlockStart()); ioCxtRef.setNextBlockStart(pointerPos); } }
@Override public float getProgress() throws IOException { if (isSorted) { return super.getProgress(); } return recordReader.getProgress(); }
public boolean doNext(K key, V value) throws IOException { if (this.isSorted) { if (this.getIOContext().shouldEndBinarySearch() || (!this.getIOContext().useSorted() && this.wasUsingSortedSearch)) { beginLinearSearch(); this.wasUsingSortedSearch = false; this.getIOContext().setEndBinarySearch(false); if (this.getIOContext().useSorted()) { if (this.genericUDFClassName == null && this.getIOContext().getGenericUDFClassName() != null) { setGenericUDFClassName(this.getIOContext().getGenericUDFClassName()); if (this.getIOContext().isBinarySearching()) { if (this.getIOContext().getComparison() != null) { switch (this.getIOContext().getComparison()) { case GREATER: case EQUAL: sync(position); long newPosition = getSyncedPosition(); this.getIOContext().setBinarySearching(false); sync(rangeStart); } else if (foundAllTargets()) {
private boolean doNextWithExceptionHandler(K key, V value) throws IOException { return super.doNext(key, value); }
/** * Returns true if the current comparison is in the list of stop comparisons, i.e. we've found * all records which won't be filtered * @return true if the current comparison is found */ public boolean foundAllTargets() { if (this.getIOContext().getComparison() == null || !stopComparisons.contains(this.getIOContext().getComparison())) { return false; } return true; } }
public void initIOContext(FileSplit split, JobConf job, Class inputFormatClass, RecordReader recordReader) throws IOException { boolean blockPointer = false; long blockStart = -1; FileSplit fileSplit = split; Path path = fileSplit.getPath(); FileSystem fs = path.getFileSystem(job); if (inputFormatClass.getName().contains("SequenceFile")) { SequenceFile.Reader in = new SequenceFile.Reader(fs, path, job); blockPointer = in.isBlockCompressed(); in.sync(fileSplit.getStart()); blockStart = in.getPosition(); in.close(); } else if (recordReader instanceof RCFileRecordReader) { blockPointer = true; blockStart = ((RCFileRecordReader) recordReader).getStart(); } else if (inputFormatClass.getName().contains("RCFile")) { blockPointer = true; RCFile.Reader in = new RCFile.Reader(fs, path, job); in.sync(fileSplit.getStart()); blockStart = in.getPosition(); in.close(); } this.jobConf = job; this.initIOContext(blockStart, blockPointer, path.makeQualified(fs)); this.initIOContextSortedProps(split, recordReader, job); }
/** * This should be called after the binary search is finished and before the linear scan begins * @throws IOException */ private void beginLinearSearch() throws IOException { sync(rangeStart); this.getIOContext().setBinarySearching(false); this.wasUsingSortedSearch = false; }
stopComparisons.add(Comparison.EQUAL); stopComparisons.add(Comparison.GREATER); if (this.getIOContext().isBinarySearching()) { beginLinearSearch(); if (this.getIOContext().isBinarySearching()) { beginLinearSearch(); if (this.getIOContext().isBinarySearching()) { beginLinearSearch();
protected void updateIOContext() throws IOException { long pointerPos = this.getPos(); if (!ioCxtRef.isBlockPointer()) { ioCxtRef.setCurrentBlockStart(pointerPos); ioCxtRef.setCurrentRow(0); return; } ioCxtRef.setCurrentRow(ioCxtRef.getCurrentRow() + 1); if (ioCxtRef.getNextBlockStart() == -1) { ioCxtRef.setNextBlockStart(pointerPos); ioCxtRef.setCurrentRow(0); } if (pointerPos != ioCxtRef.getNextBlockStart()) { // the reader pointer has moved to the end of next block, or the end of // current record. ioCxtRef.setCurrentRow(0); if (ioCxtRef.getCurrentBlockStart() == ioCxtRef.getNextBlockStart()) { ioCxtRef.setCurrentRow(1); } ioCxtRef.setCurrentBlockStart(ioCxtRef.getNextBlockStart()); ioCxtRef.setNextBlockStart(pointerPos); } }
@Override public float getProgress() throws IOException { if (isSorted) { return super.getProgress(); } return recordReader.getProgress(); }
public boolean doNext(K key, V value) throws IOException { if (this.isSorted) { if (this.getIOContext().shouldEndBinarySearch() || (!this.getIOContext().useSorted() && this.wasUsingSortedSearch)) { beginLinearSearch(); this.wasUsingSortedSearch = false; this.getIOContext().setEndBinarySearch(false); if (this.getIOContext().useSorted()) { if (this.genericUDFClassName == null && this.getIOContext().getGenericUDFClassName() != null) { setGenericUDFClassName(this.getIOContext().getGenericUDFClassName()); if (this.getIOContext().isBinarySearching()) { if (this.getIOContext().getComparison() != null) { switch (this.getIOContext().getComparison()) { case GREATER: case EQUAL: sync(position); long newPosition = getSyncedPosition(); this.getIOContext().setBinarySearching(false); sync(rangeStart); } else if (foundAllTargets()) {