@VisibleForTesting public SplitInfo(Context context, FileSystem fs, FileStatus fileStatus, OrcTail orcTail, List<OrcProto.Type> readerTypes, boolean isOriginal, ArrayList<DeltaMetaData> deltas, boolean hasBase, Path dir, boolean[] covered) throws IOException { this(context, fs, AcidUtils.createOriginalObj(null, fileStatus), orcTail, readerTypes, isOriginal, deltas, hasBase, dir, covered, null); } }
@VisibleForTesting public SplitInfo(Context context, FileSystem fs, FileStatus fileStatus, OrcTail orcTail, List<OrcProto.Type> readerTypes, boolean isOriginal, ArrayList<DeltaMetaData> deltas, boolean hasBase, Path dir, boolean[] covered) throws IOException { this(context, fs, AcidUtils.createOriginalObj(null, fileStatus), orcTail, readerTypes, isOriginal, deltas, hasBase, dir, covered, null); } }
private List<HdfsFileStatusWithId> findBaseFiles( Path base, Ref<Boolean> useFileIds) throws IOException { Boolean val = useFileIds.value; if (val == null || val) { try { List<HdfsFileStatusWithId> result = SHIMS.listLocatedHdfsStatus( fs, base, AcidUtils.hiddenFileFilter); if (val == null) { useFileIds.value = true; // The call succeeded, so presumably the API is there. } return result; } catch (Throwable t) { LOG.error("Failed to get files with ID; using regular API: " + t.getMessage()); if (val == null && t instanceof UnsupportedOperationException) { useFileIds.value = false; } } } // Fall back to regular API and create states without ID. List<FileStatus> children = HdfsUtils.listLocatedStatus(fs, base, AcidUtils.hiddenFileFilter); List<HdfsFileStatusWithId> result = new ArrayList<>(children.size()); for (FileStatus child : children) { result.add(AcidUtils.createOriginalObj(null, child)); } return result; } }
private List<HdfsFileStatusWithId> findBaseFiles( Path base, Ref<Boolean> useFileIds) throws IOException { Boolean val = useFileIds.value; if (val == null || val) { try { List<HdfsFileStatusWithId> result = SHIMS.listLocatedHdfsStatus( fs, base, AcidUtils.hiddenFileFilter); if (val == null) { useFileIds.value = true; // The call succeeded, so presumably the API is there. } return result; } catch (Throwable t) { LOG.error("Failed to get files with ID; using regular API: " + t.getMessage()); if (val == null && t instanceof UnsupportedOperationException) { useFileIds.value = false; } } } // Fall back to regular API and create states without ID. List<FileStatus> children = HdfsUtils.listLocatedStatus(fs, base, AcidUtils.hiddenFileFilter); List<HdfsFileStatusWithId> result = new ArrayList<>(children.size()); for (FileStatus child : children) { result.add(AcidUtils.createOriginalObj(null, child)); } return result; } }
original.add(createOriginalObj(childWithId, child));
original.add(createOriginalObj(null, child));
if (!child.isDirectory()) { if (!ignoreEmptyFiles || child.getLen() != 0) { original.add(createOriginalObj(childWithId, child));
findOriginals(fs, child, original, useFileIds); } else { original.add(createOriginalObj(null, child));
HdfsFileStatusWithId fileId = AcidUtils.createOriginalObj(null, child); baseFiles.add(new AcidBaseFileInfo(fileId, AcidUtils.AcidBaseFileType.INSERT_DELTA));
HdfsFileStatusWithId fileId = AcidUtils.createOriginalObj(null, child); baseFiles.add(new AcidBaseFileInfo(fileId, deltaType));