private static Path[] getDeleteDeltaDirsFromSplit(OrcSplit orcSplit) throws IOException { Path path = orcSplit.getPath(); Path root; if (orcSplit.hasBase()) { if (orcSplit.isOriginal()) { root = path.getParent(); } else { root = path.getParent().getParent(); } } else { root = path; } return AcidUtils.deserializeDeleteDeltas(root, orcSplit.getDeltas()); }
static Path[] getDeleteDeltaDirsFromSplit(OrcSplit orcSplit) throws IOException { Path path = orcSplit.getPath(); Path root; if (orcSplit.hasBase()) { if (orcSplit.isOriginal()) { root = orcSplit.getRootDir(); } else { root = path.getParent().getParent();//todo: why not just use getRootDir()? assert root.equals(orcSplit.getRootDir()) : "root mismatch: baseDir=" + orcSplit.getRootDir() + " path.p.p=" + root; } } else { throw new IllegalStateException("Split w/o base w/Acid 2.0??: " + path); } return AcidUtils.deserializeDeleteDeltas(root, orcSplit.getDeltas()); }
public OrcNewSplit(OrcSplit inner) throws IOException { super(inner.getPath(), inner.getStart(), inner.getLength(), inner.getLocations()); this.orcTail = inner.getOrcTail(); this.hasFooter = inner.hasFooter(); this.isOriginal = inner.isOriginal(); this.hasBase = inner.hasBase(); this.deltas.addAll(inner.getDeltas()); }
public OrcNewSplit(OrcSplit inner) throws IOException { super(inner.getPath(), inner.getStart(), inner.getLength(), inner.getLocations()); this.orcTail = inner.getOrcTail(); this.hasFooter = inner.hasFooter(); this.isOriginal = inner.isOriginal(); this.hasBase = inner.hasBase(); this.deltas.addAll(inner.getDeltas()); }
private Path getSplitPath(FileSplit inputSplit, JobConf conf) throws IOException { Path path = inputSplit.getPath(); if (inputSplit instanceof OrcSplit) { OrcSplit orcSplit = (OrcSplit) inputSplit; List<Long> deltas = orcSplit.getDeltas(); if (!orcSplit.hasBase() && deltas.size() >= 2) { throw new IOException("Cannot read valid StructTypeInfo from delta only file: " + path); } } LOG.debug("Input split path: {}", path); return path; }
private static boolean hasDeltas(OrcSplit orcSplit) throws IOException { final Path path = orcSplit.getPath(); final Path root; // If the split has a base, extract the base file size, bucket and root path info. if (orcSplit.hasBase()) { if (orcSplit.isOriginal()) { root = path.getParent(); } else { root = path.getParent().getParent(); } } else { root = path; } final Path[] deltas = AcidUtils.deserializeDeltas(root, orcSplit.getDeltas()); return deltas.length > 0; } }
public OrcNewSplit(OrcSplit inner) throws IOException { super(inner.getPath(), inner.getStart(), inner.getLength(), inner.getLocations()); this.fileMetaInfo = inner.getFileMetaInfo(); this.hasFooter = inner.hasFooter(); this.isOriginal = inner.isOriginal(); this.hasBase = inner.hasBase(); this.deltas.addAll(inner.getDeltas()); }
final Path[] deltas = AcidUtils.deserializeDeltas(root, orcSplit.getDeltas());
if (split.isOriginal() && split.getDeltas().isEmpty()) { if (vectorMode) { return createVectorizedReader(inputSplit, conf, reporter);
root = path; final Path[] deltas = AcidUtils.deserializeDeltas(root, split.getDeltas()); final Configuration conf = options.getConfiguration(); final Reader reader;