@Override public Iterable<FileStatus> apply(Refresh input) { try { FileSelection selection = FileSelection.create(getFs(), resolveTablePathToValidPath(input.getPath())); if(selection != null) { return selection.minusDirectories().getFileStatuses(); } throw new IllegalStateException("Unable to retrieve selection for path." + input.getPath()); } catch (IOException e) { throw Throwables.propagate(e); } }}).toList();
public static FileSelection create(final FileSystemWrapper fs, Path combined) throws IOException { Stopwatch timer = Stopwatch.createStarted(); final ImmutableList<FileStatus> statuses = fs.listRecursive(combined, false); if (statuses == null || statuses.isEmpty()) { return null; } final FileSelection fileSel = createFromExpanded(statuses, combined.toUri().getPath()); logger.debug("FileSelection.create() took {} ms ", timer.elapsed(TimeUnit.MILLISECONDS)); return fileSel; }
public static FileSelection create(final FileSystemWrapper fs, final List<String> fullPath) throws IOException { return create(fs, getPathBasedOnFullPath(fullPath)); }
private boolean isHomogeneous(FileSystemWrapper fs, FileSelection fileSelection) throws IOException { FormatMatcher matcher = null; FileSelection noDir = fileSelection.minusDirectories(); for(FileStatus s : noDir.getStatuses()) { FileSelection subSelection = FileSelection.create(s); if (matcher == null) { matcher = findMatcher(fs, subSelection); if(matcher == null) { return false; } } if(!matcher.matches(fs, subSelection, codecFactory)) { return false; } } return true; }
List<String> parentSchemaPath = new ArrayList<>(fullPath.subList(0, fullPath.size() - 1)); FileSystemWrapper fs = getFS((user != null) ? user : userName); FileSelection fileSelection = FileSelection.create(fs, fullPath); String tableName = datasetPath.getName(); fileSelection = FileSelection.createWithFullSchema(fs, PathUtils.toFSPathString(parentSchemaPath), tableName); if (fileSelection == null) { return null; // no table found final boolean hasDirectories = fileSelection.containsDirectories(); final FileStatus rootStatus = fs.getFileStatus(new Path(fileSelection.getSelectionRoot())); for (FileStatus dirStatus: fileSelection.getAllDirectories()) { cachedEntities.add(fromFileStatus(dirStatus)); final FileSelection fileSelectionWithoutDir = hasDirectories? new FileSelection(fileSelection).minusDirectories(): fileSelection; if(fileSelectionWithoutDir == null || fileSelectionWithoutDir.isEmpty()){
private SourceTableDefinition getDataset(NamespaceKey datasetPath, DatasetConfig oldConfig, FormatPlugin formatPlugin, FileSystemWrapper fs, FileConfig fileConfig) throws IOException { final List<FileSystemCachedEntity> cachedEntities = Lists.newArrayList(); final FileStatus rootStatus = fs.getFileStatus(new Path(fileConfig.getLocation())); final Path combined = new Path("/", PathUtils.removeLeadingSlash(fileConfig.getLocation())); final FileSelection fileSelection = FileSelection.create(fs, combined); if (fileSelection == null) { return null; } // first entity is always a root if (rootStatus.isDirectory()) { cachedEntities.add(fromFileStatus(rootStatus)); } for (FileStatus dirStatus: fileSelection.getAllDirectories()) { cachedEntities.add(fromFileStatus(dirStatus)); } if(cachedEntities.isEmpty()){ // this is a single file. cachedEntities.add(fromFileStatus(rootStatus)); } final FileUpdateKey updateKey = new FileUpdateKey().setCachedEntitiesList(cachedEntities); final boolean hasDirectories = fileSelection.containsDirectories(); // Expand selection by copying it first used to check extensions of files in directory. final FileSelection fileSelectionWithoutDir = hasDirectories? fileSelection.minusDirectories(): fileSelection; if(fileSelectionWithoutDir == null){ // no files in the found directory, not a table. return null; } return formatPlugin.getDatasetAccessor(oldConfig, fs, fileSelectionWithoutDir, this, datasetPath, datasetPath.getName(), updateKey); }
public static FileSelection createFromExpanded(final ImmutableList<FileStatus> statuses, final String root) { return create(StatusType.EXPANDED, statuses, root); }
SampleMutator mutator = new SampleMutator(sampleAllocator) ){ final Optional<FileStatus> firstFileO = selection.getFirstFile(); if (!firstFileO.isPresent()) { throw UserException.dataReadError().message("Unable to find any files for datasets.").build(logger); for (FileStatus firstFile : selection.getFileStatuses()) { ParquetMetadata footer = ParquetFileReader.readFooter(fsPlugin.getFsConf(), firstFile, ParquetMetadataConverter.NO_FILTER);
final FileSystemPlugin plugin = getPlugin(key); final FileSystemWrapper fs = plugin.getFS(securityContext.getUserPrincipal().getName()); final Path path = FileSelection.getPathBasedOnFullPath(plugin.resolveTableNameToValidPath(key.getPathComponents()));
/** * Creates a {@link FileSelection selection} with the given file statuses and selection root. * * @param statuses list of file statuses * @param root root path for selections * * @return null if creation of {@link FileSelection} fails with an {@link IllegalArgumentException} * otherwise a new selection. * */ private static FileSelection create(StatusType status, final ImmutableList<FileStatus> statuses, final String root) { if (statuses == null || statuses.size() == 0) { return null; } final String selectionRoot; if (Strings.isNullOrEmpty(root)) { throw new IllegalArgumentException("Selection root is null or empty" + root); } final Path rootPath = handleWildCard(root); final URI uri = statuses.get(0).getPath().toUri(); final Path path = new Path(uri.getScheme(), uri.getAuthority(), rootPath.toUri().getPath()); selectionRoot = Path.getPathWithoutSchemeAndAuthority(path).toString(); return new FileSelection(status, statuses, selectionRoot); }
public static FileSelection create(FileStatus status) throws IOException { return new FileSelection(StatusType.EXPANDED, ImmutableList.of(status), status.getPath().toString()); }
final String pathString = buildPath(folders[0], latest); return new Path(uri.getScheme(), uri.getAuthority(), pathString).toString();
public FileSelection minusDirectories() throws IOException { if (dirStatus == StatusType.NO_DIRS) { return this; } Stopwatch timer = Stopwatch.createStarted(); final int total = statuses.size(); final ImmutableList<FileStatus> nonDirectories = FluentIterable.from(statuses).filter(new Predicate<FileStatus>() { @Override public boolean apply(@Nullable FileStatus status) { return !status.isDirectory(); } }).toList(); final FileSelection fileSel = create(StatusType.NO_DIRS, nonDirectories, selectionRoot); logger.debug("FileSelection.minusDirectories() took {} ms, numFiles: {}", timer.elapsed(TimeUnit.MILLISECONDS), total); return fileSel; }
private FileFormat detectFileFormat(NamespaceKey key) { final FileSystemPlugin plugin = getPlugin(key); final FileSystemWrapper fs = plugin.getFS(securityContext.getUserPrincipal().getName()); final Path path = FileSelection.getPathBasedOnFullPath(plugin.resolveTableNameToValidPath(key.getPathComponents()));
public static FileSelection createWithFullSchema(final FileSystemWrapper fs, final String parent, final String fullSchemaPath) throws IOException { final Path combined = Path.mergePaths(new Path(parent), PathUtils.toFSPath(fullSchemaPath)); return create(fs, combined); }
@Override public SourceTableDefinition getDataset( NamespaceKey datasetPath, DatasetConfig oldConfig, DatasetRetrievalOptions ignored ) throws Exception { FluentIterable<Refresh> refreshes = getSlices(datasetPath.getPathComponents()); if(refreshes == null) { return null; } final String selectionRoot = new Path(getConfig().getPath(), refreshes.first().get().getReflectionId().getId()).toString(); ImmutableList<FileStatus> allStatus = refreshes.transformAndConcat(new Function<Refresh, Iterable<FileStatus>>(){ @Override public Iterable<FileStatus> apply(Refresh input) { try { FileSelection selection = FileSelection.create(getFs(), resolveTablePathToValidPath(input.getPath())); if(selection != null) { return selection.minusDirectories().getFileStatuses(); } throw new IllegalStateException("Unable to retrieve selection for path." + input.getPath()); } catch (IOException e) { throw Throwables.propagate(e); } }}).toList(); FileSelection selection = FileSelection.createFromExpanded(allStatus, selectionRoot); return new ParquetFormatDatasetAccessor(oldConfig, getFs(), selection, this, datasetPath, null, EMPTY, formatPlugin); }
FileSelection fileSelection; try { fileSelection = FileSelection.create(fs, fullPath); } catch (IOException e) { throw new RuntimeException(e);