public OrcNewSplit(OrcSplit inner) throws IOException { super(inner.getPath(), inner.getStart(), inner.getLength(), inner.getLocations()); this.orcTail = inner.getOrcTail(); this.hasFooter = inner.hasFooter(); this.isOriginal = inner.isOriginal(); this.hasBase = inner.hasBase(); this.deltas.addAll(inner.getDeltas()); }
public OrcNewSplit(OrcSplit inner) throws IOException { super(inner.getPath(), inner.getStart(), inner.getLength(), inner.getLocations()); this.orcTail = inner.getOrcTail(); this.hasFooter = inner.hasFooter(); this.isOriginal = inner.isOriginal(); this.hasBase = inner.hasBase(); this.deltas.addAll(inner.getDeltas()); }
/** * Ensures orcReader is initialized for the split. */ private void ensureOrcReader() throws IOException { if (orcReader != null) return; path = split.getPath(); if (fileKey instanceof Long && HiveConf.getBoolVar( daemonConf, ConfVars.LLAP_IO_USE_FILEID_PATH)) { path = HdfsUtils.getFileIdPath(fs, path, (long)fileKey); } LlapIoImpl.ORC_LOGGER.trace("Creating reader for {} ({})", path, split.getPath()); long startTime = counters.startTimeCounter(); ReaderOptions opts = OrcFile.readerOptions(jobConf).filesystem(fs).fileMetadata(fileMetadata); if (split instanceof OrcSplit) { OrcTail orcTail = ((OrcSplit) split).getOrcTail(); if (orcTail != null) { LlapIoImpl.ORC_LOGGER.debug("Setting OrcTail. path={}", path); opts.orcTail(orcTail); } } orcReader = EncodedOrcFile.createReader(path, opts); counters.incrWallClockCounter(LlapIOCounters.HDFS_TIME_NS, startTime); }
static Reader createOrcReaderForSplit(Configuration conf, OrcSplit orcSplit) throws IOException { Path path = orcSplit.getPath(); Reader reader; if (orcSplit.hasBase()) { OrcFile.ReaderOptions readerOptions = OrcFile.readerOptions(conf); readerOptions.maxLength(orcSplit.getFileLength()); if (orcSplit.hasFooter()) { readerOptions.orcTail(orcSplit.getOrcTail()); } reader = OrcFile.createReader(path, readerOptions); } else { reader = null; } return reader; }
static Reader createOrcReaderForSplit(Configuration conf, OrcSplit orcSplit) throws IOException { Path path = orcSplit.getPath(); Reader reader; if (orcSplit.hasBase()) { OrcFile.ReaderOptions readerOptions = OrcFile.readerOptions(conf); readerOptions.maxLength(orcSplit.getFileLength()); if (orcSplit.hasFooter()) { readerOptions.orcTail(orcSplit.getOrcTail()); } reader = OrcFile.createReader(path, readerOptions); } else { reader = null; } return reader; }
@Override public RecordReader<NullWritable, VectorizedRowBatch> getRecordReader(InputSplit inputSplit, JobConf conf, Reporter reporter) throws IOException { FileSplit fSplit = (FileSplit)inputSplit; reporter.setStatus(fSplit.toString()); Path path = fSplit.getPath(); OrcFile.ReaderOptions opts = OrcFile.readerOptions(conf); if(fSplit instanceof OrcSplit){ OrcSplit orcSplit = (OrcSplit) fSplit; if (orcSplit.hasFooter()) { opts.orcTail(orcSplit.getOrcTail()); } opts.maxLength(orcSplit.getFileLength()); } Reader reader = OrcFile.createReader(path, opts); return new VectorizedOrcRecordReader(reader, conf, fSplit); }
@Override public RecordReader<NullWritable, VectorizedRowBatch> getRecordReader(InputSplit inputSplit, JobConf conf, Reporter reporter) throws IOException { FileSplit fSplit = (FileSplit)inputSplit; reporter.setStatus(fSplit.toString()); Path path = fSplit.getPath(); OrcFile.ReaderOptions opts = OrcFile.readerOptions(conf); if(fSplit instanceof OrcSplit){ OrcSplit orcSplit = (OrcSplit) fSplit; if (orcSplit.hasFooter()) { opts.orcTail(orcSplit.getOrcTail()); } opts.maxLength(orcSplit.getFileLength()); } Reader reader = OrcFile.createReader(path, opts); return new VectorizedOrcRecordReader(reader, conf, fSplit); }
if (inputSplit instanceof OrcSplit) { OrcSplit split = (OrcSplit) inputSplit; readerOptions.maxLength(split.getFileLength()).orcTail(split.getOrcTail());
if (inputSplit instanceof OrcSplit) { OrcSplit split = (OrcSplit) inputSplit; readerOptions.maxLength(split.getFileLength()).orcTail(split.getOrcTail());
/** * Ensures orcReader is initialized for the split. */ private void ensureOrcReader() throws IOException { if (orcReader != null) return; path = split.getPath(); if (fileKey instanceof Long && HiveConf.getBoolVar( daemonConf, ConfVars.LLAP_IO_USE_FILEID_PATH)) { path = HdfsUtils.getFileIdPath(fs, path, (long)fileKey); } LlapIoImpl.ORC_LOGGER.trace("Creating reader for {} ({})", path, split.getPath()); long startTime = counters.startTimeCounter(); ReaderOptions opts = OrcFile.readerOptions(jobConf).filesystem(fs).fileMetadata(fileMetadata); if (split instanceof OrcSplit) { OrcTail orcTail = ((OrcSplit) split).getOrcTail(); if (orcTail != null) { LlapIoImpl.ORC_LOGGER.debug("Setting OrcTail. path={}", path); opts.orcTail(orcTail); } } orcReader = EncodedOrcFile.createReader(path, opts); counters.incrTimeCounter(LlapIOCounters.HDFS_TIME_NS, startTime); }