public boolean containsDirectories() throws IOException { if (dirStatus == StatusType.EXPANDED) { for (final FileStatus status : getStatuses()) { if (status.isDirectory()) { return true; } } } return dirStatus == StatusType.HAS_DIRS; }
public FileStatus getFirstPath() throws IOException { return getStatuses().get(0); }
public int getMaxDepth(FileStatus rootStatus) { final int selectionDepth = rootStatus.getPath().depth(); int maxDepth = 0; for (final FileStatus status : getStatuses()) { maxDepth = Math.max(maxDepth, status.getPath().depth() - selectionDepth); } return maxDepth; }
public ParquetGroupScanUtils( String userName, FileSelection selection, FileSystemPlugin plugin, ParquetFormatPlugin formatPlugin, String selectionRoot, List<SchemaPath> columns, BatchSchema schema, Map<String, GlobalDictionaryFieldInfo> globalDictionaryColumns, List<ParquetFilterCondition> conditions, SystemOptionManager optionManager) throws IOException { this.schema = schema; this.formatPlugin = formatPlugin; this.conditions = conditions; this.columns = columns; this.fs = ImpersonationUtil.createFileSystem(userName, plugin.getFsConf()); this.plugin = plugin; this.selectionRoot = selectionRoot; this.entries = selection.getStatuses(); this.globalDictionaryColumns = (globalDictionaryColumns == null)? Collections.<String, GlobalDictionaryFieldInfo>emptyMap() : globalDictionaryColumns; this.optionManager = optionManager; init(); }
private boolean isHomogeneous(FileSystemWrapper fs, FileSelection fileSelection) throws IOException { FormatMatcher matcher = null; FileSelection noDir = fileSelection.minusDirectories(); for(FileStatus s : noDir.getStatuses()) { FileSelection subSelection = FileSelection.create(s); if (matcher == null) { matcher = findMatcher(fs, subSelection); if(matcher == null) { return false; } } if(!matcher.matches(fs, subSelection, codecFactory)) { return false; } } return true; }
private void initFromSelection(FileSelection selection, EasyFormatPlugin<?> formatPlugin) throws IOException { final FileSystemWrapper dfs = ImpersonationUtil.createFileSystem(userName, plugin.getFsConf()); this.selection = selection; BlockMapBuilder b = new BlockMapBuilder(dfs, plugin.getContext().getExecutors()); this.chunks = b.generateFileWork(selection.getStatuses(), formatPlugin.isBlockSplittable()); }
final ImplicitFilesystemColumnFinder explorer = new ImplicitFilesystemColumnFinder(context.getOptionManager(), dfs, GroupScan.ALL_COLUMNS); Optional<FileStatus> fileName = Iterables.tryFind(selection.getStatuses(), new Predicate<FileStatus>() { @Override public boolean apply(@Nullable FileStatus input) { final FileStatus file = fileName.or(selection.getStatuses().get(0));