/** * Convert fs path to list of strings. * /a/b/c -> [a,b,c] * @param fsPath filesystem path * @return list of path components */ public static List<String> toPathComponents(Path fsPath) { if (fsPath == null ) { return EMPTY_SCHEMA_PATHS; } return toPathComponents(fsPath.toUri().getPath()); }
/** * Get the name of the file or folder at Path * @param path * @return */ public static String getQuotedFileName(Path path) { List<String> components = toPathComponents(path); return String.format("%1$s%2$s%1$s", SqlUtils.QUOTE, components.get(components.size() - 1)); }
public static List<String> getPathFromInternalId(String id) { return com.dremio.common.utils.PathUtils.toPathComponents(id.substring(INTERNAL_ID_PREFIX.length())); }
private static List<String> relativePath(List<String> tableSchemaPath, Path rootPath) { List<String> rootPathComponents = PathUtils.toPathComponents(rootPath); List<String> tablePathComponents = PathUtils.toPathComponents(PathUtils.toFSPathSkipRoot(tableSchemaPath, null)); return tablePathComponents.subList(rootPathComponents.size(), tablePathComponents.size()); } }
/** * Convert fs path to dotted schema path. * /a/b/c -> a.b.c * @param fsPath filesystem path * @return schema path. */ public static String toDottedPath(Path fsPath) { return constructFullPath(toPathComponents(fsPath)); }
/** * Convert fs path relative to parent to dotted schema path. * parent: /a/b, child: /a/b/c/d -> c.d * @param parent parent path * @param child full path of child inside parent path * @return dotted schema name of child relative to parent. * @throws IOException if child does not belong under parent */ public static String toDottedPath(final Path parent, final Path child) throws IOException { final List<String> parentPathComponents = toPathComponents(parent); final List<String> childPathComponents = toPathComponents(child); for (int i = 0; i < parentPathComponents.size(); ++i) { if (!parentPathComponents.get(i).equals(childPathComponents.get(i))) { throw new IOException(String.format("Invalid file/directory %s listed under %s", child, parent)); } } return constructFullPath(childPathComponents.subList(parentPathComponents.size(), childPathComponents.size())); }
/** * Creates a FilePath from a root entity and a path component from a URL * * @param root the root entity * @param path a relative path represented as a string * @return a new FilePath instance */ public static FilePath fromURLPath(RootEntity root, String path) { List<String> components = PathUtils.toPathComponents(path); return new FilePath(ImmutableList.<String> builder().add(root.getName()).addAll(components).build()); }
public static String relativePath(Path absolutePath, Path basePath) { Preconditions.checkArgument(absolutePath.isAbsolute(), "absolutePath must be an absolute path"); Preconditions.checkArgument(basePath.isAbsolute(), "basePath must be an absolute path"); List<String> absolutePathComponents = Lists.newArrayList(toPathComponents(absolutePath)); // make a copy List<String> basePathComponents = toPathComponents(basePath); boolean hasCommonPrefix = basePathComponents.isEmpty(); // when basePath is "/", we always have a common prefix for (String base : basePathComponents) { if (absolutePathComponents.get(0).equals(base)) { absolutePathComponents.remove(0); hasCommonPrefix = true; } else { break; } } if (hasCommonPrefix) { return PATH_JOINER.join(absolutePathComponents); } else { return absolutePath.toString(); } }
/** * Get summary for dataset (physical datasets, virtual datasets) * @param path relative path to the summary * @return * @throws NamespaceException */ @GET @Path("/summary/{path: .*}") @Produces(MediaType.APPLICATION_JSON) public DatasetSummary getDatasetSummary(@PathParam("path") String path) throws NamespaceException { final DatasetPath datasetPath = new DatasetPath(PathUtils.toPathComponents(path)); return getDatasetSummary(datasetPath); }
public static DatasetPath fromURLPath(RootEntity root, String path) { List<String> components = PathUtils.toPathComponents(path); return new DatasetPath(ImmutableList.<String> builder().add(root.getName()).addAll(components).build()); }
/** * Convert fully or partially dotted path to file system path. * a.b.c -> /a/b/c * a.`b/c` -> /a/b/c * @param path a fully or partially dotted path * @return {@code fs Path} */ public static Path toFSPath(String path) { final List<String> pathComponents = Lists.newArrayList(); for (String component: parseFullPath(path)) { if (component.contains(SLASH)) { pathComponents.addAll(toPathComponents(new Path(component))); } else { pathComponents.add(component); } } return toFSPath(pathComponents); }
public static List<String> getRefreshPath(final JobId jobId, final JobData jobData, final Path accelerationBasePath) { // extract written path from writer's metadata JobDataFragment data = jobData.range(0, 1); Text text = (Text) Preconditions.checkNotNull(data.extractValue(RecordWriter.PATH_COLUMN, 0), "Empty write path for job %s", jobId.getId()); // relative path to the acceleration base path final String path = PathUtils.relativePath(new Path(text.toString()), accelerationBasePath); // extract first 2 components of the path "<reflection-id>."<modified-materialization-id>" List<String> components = PathUtils.toPathComponents(path); Preconditions.checkState(components.size() >= 2, "Refresh path %s is incomplete", path); return ImmutableList.of(ACCELERATOR_STORAGEPLUGIN_NAME, components.get(0), components.get(1)); }
/** * Helper method which resolves the table name to actual file/folder on the filesystem. * If the resolved path refers to an entity not under the base of the source then a permission error is thrown. * * Ex. For given source named "dfs" which has base path of "/base" tableSchemaPath * [dfs, tmp, a] -> "/base/tmp/a" * [dfs, "/tmp/b"] -> "/base/tmp/b" * [dfs, "value", tbl] -> "/base/value/tbl" * @param tableSchemaPath * @return */ public List<String> resolveTableNameToValidPath(List<String> tableSchemaPath) { List<String> fullPath = new ArrayList<>(); fullPath.addAll(PathUtils.toPathComponents(basePath)); for (String pathComponent : tableSchemaPath.subList(1 /* need to skip the source name */, tableSchemaPath.size())) { fullPath.add(PathUtils.removeQuotes(pathComponent)); } PathUtils.verifyNoAccessOutsideBase(basePath, PathUtils.toFSPath(fullPath)); return fullPath; }
private void deleteOwnedRefreshes(Materialization materialization, SchemaConfig schemaConfig) { Iterable<Refresh> refreshes = materializationStore.getRefreshesExclusivelyOwnedBy(materialization); if (Iterables.isEmpty(refreshes)) { logger.debug("deleted materialization {} has no associated refresh"); return; } for (Refresh r : refreshes) { try { //TODO once DX-10850 is fixed we should no longer need to split the refresh path into separate components final List<String> tableSchemaPath = ImmutableList.<String>builder() .add(ReflectionServiceImpl.ACCELERATOR_STORAGEPLUGIN_NAME) .addAll(PathUtils.toPathComponents(r.getPath())) .build(); logger.debug("deleting refresh {}", tableSchemaPath); super.dropTable(tableSchemaPath, schemaConfig); } catch (Exception e) { logger.warn("Couldn't delete refresh {}", r.getId().getId(), e); } finally { materializationStore.delete(r.getId()); } } }
@Test public void testPathComponents() throws Exception { assertEquals(ImmutableList.of("a", "b", "c"), PathUtils.toPathComponents(new Path("/a/b/c"))); assertEquals(ImmutableList.of("a", "b", "c"), PathUtils.toPathComponents(new Path("a/b/c"))); assertEquals(ImmutableList.of("a", "b", "c/"), PathUtils.toPathComponents(new Path("a/b/\"c/\""))); }
private static String getSchemaPath(String file) throws IOException { return "dacfs_test." + PathUtils.constructFullPath(PathUtils.toPathComponents(new Path(FileUtils.getResourceAsFile(file).getAbsolutePath()))); }
/** * Check with namespace format setting on a file/directory. * @param namespaceService PhysicalDatasetService namespaceService * @param schemaPath parent path * @param tableName table name (file/directory name) * @param fileSelection file/files selected under directory. * @return {@code FormatPluginConfig} that should be used for creating Dremio table. */ public static FormatPluginConfig getPhysicalDatasetProperties(NamespaceService namespaceService, final List<String> schemaPath, final String tableName, final FileSelection fileSelection) { // If the execution engine is being tested outside the context of a Dremio instance, there won't be a dataset service. if(namespaceService == null) { return null; } final List<String> tableSchemaPath = new ArrayList<>(schemaPath); tableSchemaPath.addAll(PathUtils.toPathComponents(new Path(tableName))); try { final DatasetConfig config = namespaceService.getDataset(new NamespaceKey(tableSchemaPath)); return toFormatPlugin(config.getPhysicalDataset().getFormatSettings(), fileSelection.getExtensions()); } catch (NamespaceException e) { logger.debug("Failed to get physical dataset properties for table {} error {}", PathUtils.constructFullPath(tableSchemaPath), e); } return null; }
/** * Fetches a single item from the filesystem plugin */ public SchemaEntity get(List<String> path, String userName) { try { final FileStatus status = getFS(userName).getFileStatus(PathUtils.toFSPath(resolveTableNameToValidPath(path))); final Set<List<String>> tableNames = Sets.newHashSet(); final NamespaceService ns = context.getNamespaceService(userName); final NamespaceKey folderNSKey = new NamespaceKey(path); if (ns.exists(folderNSKey)) { for(NameSpaceContainer entity : ns.list(folderNSKey)) { if (entity.getType() == Type.DATASET) { tableNames.add(resolveTableNameToValidPath(entity.getDataset().getFullPathList())); } } } List<String> p = PathUtils.toPathComponents(status.getPath()); return getSchemaEntity(status, tableNames, p); } catch (IOException | NamespaceException e) { throw new RuntimeException(e); } }
new NamespaceKey(ImmutableList.<String>builder() .add("__home") // TODO (AH) hack. .addAll(PathUtils.toPathComponents(datasetConfig.getPhysicalDataset().getFormatSettings().getLocation())).build()), datasetConfig, false); if (datasetAccessor == null) {
case FOLDER_TABLE: { try { final NamespaceKey namespaceKey = new NamespaceKey(PathUtils.toPathComponents(PathUtils.toFSPath(entityPath))); final DatasetConfig dataset = namespaceService.getDataset(namespaceKey); catalogItem = new CatalogItem.Builder()