private PartitionDesc extractSinglePartSpec(CombineHiveInputSplit hsplit) throws IOException { PartitionDesc part = null; Map<Map<Path,PartitionDesc>, Map<Path,PartitionDesc>> cache = new HashMap<>(); for (Path path : hsplit.getPaths()) { PartitionDesc otherPart = HiveFileFormatUtils.getFromPathRecursively( pathToPartInfo, path, cache); LOG.debug("Found spec for " + path + " " + otherPart + " from " + pathToPartInfo); if (part == null) { part = otherPart; } else if (otherPart != part) { // Assume we should have the exact same object. // TODO: we could also compare the schema and SerDe, and pass only those to the call // instead; most of the time these would be the same and LLAP IO can handle that. LOG.warn("Multiple partitions found; not going to pass a part spec to LLAP IO: {" + part.getPartSpec() + "} and {" + otherPart.getPartSpec() + "}"); return null; } } return part; }
private PartitionDesc extractSinglePartSpec(CombineHiveInputSplit hsplit) throws IOException { PartitionDesc part = null; Map<Map<Path,PartitionDesc>, Map<Path,PartitionDesc>> cache = new HashMap<>(); for (Path path : hsplit.getPaths()) { PartitionDesc otherPart = HiveFileFormatUtils.getPartitionDescFromPathRecursively( pathToPartInfo, path, cache); LOG.debug("Found spec for " + path + " " + otherPart + " from " + pathToPartInfo); if (part == null) { part = otherPart; } else if (otherPart != part) { // Assume we should have the exact same object. // TODO: we could also compare the schema and SerDe, and pass only those to the call // instead; most of the time these would be the same and LLAP IO can handle that. LOG.warn("Multiple partitions found; not going to pass a part spec to LLAP IO: {" + part.getPartSpec() + "} and {" + otherPart.getPartSpec() + "}"); return null; } } return part; }
FileSplit fsplit = new FileSplit(hsplit.getPaths()[partition], hsplit .getStartOffsets()[partition], hsplit.getLengths()[partition], hsplit .getLocations()); if (partition != 0 && hsplit.getPaths()[partition -1].equals(hsplit.getPaths()[partition])) this.setFooterBuffer(((CombineHiveRecordReader)preReader).getFooterBuffer());
public CombineHiveRecordReader(InputSplit split, Configuration conf, Reporter reporter, Integer partition) throws IOException { super((JobConf)conf); CombineHiveInputSplit hsplit = split instanceof CombineHiveInputSplit ? (CombineHiveInputSplit) split : new CombineHiveInputSplit(jobConf, (CombineFileSplit) split); String inputFormatClassName = hsplit.inputFormatClassName(); Class inputFormatClass = null; try { inputFormatClass = JavaUtils.loadClass(inputFormatClassName); } catch (ClassNotFoundException e) { throw new IOException("CombineHiveRecordReader: class not found " + inputFormatClassName); } InputFormat inputFormat = HiveInputFormat.getInputFormatFromCache( inputFormatClass, jobConf); // create a split for the given partition FileSplit fsplit = new FileSplit(hsplit.getPaths()[partition], hsplit .getStartOffsets()[partition], hsplit.getLengths()[partition], hsplit .getLocations()); this.setRecordReader(inputFormat.getRecordReader(fsplit, jobConf, reporter)); this.initIOContext(fsplit, jobConf, inputFormatClass, this.recordReader); }
public CombineHiveRecordReader(InputSplit split, Configuration conf, Reporter reporter, Integer partition) throws IOException { JobConf job = (JobConf) conf; CombineHiveInputSplit hsplit = new CombineHiveInputSplit(job, (InputSplitShim) split); String inputFormatClassName = hsplit.inputFormatClassName(); Class inputFormatClass = null; try { inputFormatClass = Class.forName(inputFormatClassName); } catch (ClassNotFoundException e) { throw new IOException("CombineHiveRecordReader: class not found " + inputFormatClassName); } InputFormat inputFormat = HiveInputFormat.getInputFormatFromCache( inputFormatClass, job); // create a split for the given partition FileSplit fsplit = new FileSplit(hsplit.getPaths()[partition], hsplit .getStartOffsets()[partition], hsplit.getLengths()[partition], hsplit .getLocations()); this.recordReader = inputFormat.getRecordReader(fsplit, job, reporter); this.initIOContext(fsplit, job, inputFormatClass, this.recordReader); }
FileSplit fsplit = new FileSplit(hsplit.getPaths()[partition], hsplit .getStartOffsets()[partition], hsplit.getLengths()[partition], hsplit .getLocations()); if (partition != 0 && hsplit.getPaths()[partition -1].equals(hsplit.getPaths()[partition])) this.setFooterBuffer(((CombineHiveRecordReader)preReader).getFooterBuffer());