/** * Convert list of path components into fs path, skip root of path if its same as given root. * schema path : [a,b,c] and root: a -> /b/c * @param schemaPath list of path components * @param root of path to be ignored. * @return {@code fs Path} */ public static Path toFSPathSkipRoot(final List<String> schemaPath, final String root) { if (schemaPath == null || schemaPath.isEmpty()) { return ROOT_PATH; } if (root == null || !root.equals(schemaPath.get(0))) { return toFSPath(schemaPath); } else { return toFSPath(schemaPath.subList(1, schemaPath.size())); } }
public static String toFSPathString(final List<String> schemaPath) { return Path.getPathWithoutSchemeAndAuthority(toFSPath(schemaPath)).toString(); }
@Override public boolean containerExists(NamespaceKey key) { List<String> folderPath = key.getPathComponents(); try { return getFS(SystemUser.SYSTEM_USERNAME).exists(PathUtils.toFSPath( ImmutableList.<String>builder() .addAll(folderPath.subList(1, folderPath.size())) .build())); }catch(IOException e) { logger.info("IOException while trying to retrieve home files.", e); return false; } }
/** * Convert fully or partially dotted path to file system path. * a.b.c -> /a/b/c * a.`b/c` -> /a/b/c * @param path a fully or partially dotted path * @return {@code fs Path} */ public static Path toFSPath(String path) { final List<String> pathComponents = Lists.newArrayList(); for (String component: parseFullPath(path)) { if (component.contains(SLASH)) { pathComponents.addAll(toPathComponents(new Path(component))); } else { pathComponents.add(component); } } return toFSPath(pathComponents); }
public static FileSelection createWithFullSchema(final FileSystemWrapper fs, final String parent, final String fullSchemaPath) throws IOException { final Path combined = Path.mergePaths(new Path(parent), PathUtils.toFSPath(fullSchemaPath)); return create(fs, combined); }
/** * Helper method which resolves the table name to actual file/folder on the filesystem. * If the resolved path refers to an entity not under the base of the source then a permission error is thrown. * * Ex. For given source named "dfs" which has base path of "/base" tableSchemaPath * [dfs, tmp, a] -> "/base/tmp/a" * [dfs, "/tmp/b"] -> "/base/tmp/b" * [dfs, "value", tbl] -> "/base/value/tbl" * @param tableSchemaPath * @return */ public List<String> resolveTableNameToValidPath(List<String> tableSchemaPath) { List<String> fullPath = new ArrayList<>(); fullPath.addAll(PathUtils.toPathComponents(basePath)); for (String pathComponent : tableSchemaPath.subList(1 /* need to skip the source name */, tableSchemaPath.size())) { fullPath.add(PathUtils.removeQuotes(pathComponent)); } PathUtils.verifyNoAccessOutsideBase(basePath, PathUtils.toFSPath(fullPath)); return fullPath; }
public Map<String, String> getApiLinks() throws UnsupportedEncodingException { final NamespaceKey datasetPath = new NamespaceKey(fullPath); final String dottedFullPath = datasetPath.toUrlEncodedString(); final String fullPathString = PathUtils.toFSPath(fullPath).toString(); Map<String, String> links = new HashMap<String, String>(); switch (datasetType) { case VIRTUAL_DATASET: links.put("edit", "/dataset/" + dottedFullPath + "/version/" + datasetVersion + "?view=explore"); //edit dataset links.put("run", "/datasets/new_untitled?parentDataset=" + dottedFullPath + "&newVersion=" + DatasetVersion.newVersion()); //create new dataset break; case PHYSICAL_DATASET_HOME_FILE: links.put("run", "/home/" + fullPath.get(0) + "new_untitled_from_file" + fullPathString); break; case PHYSICAL_DATASET_HOME_FOLDER: // Folder not supported yet break; case PHYSICAL_DATASET_SOURCE_FILE: links.put("run", "/source/" + fullPath.get(0) + "new_untitled_from_file" + fullPathString); break; case PHYSICAL_DATASET_SOURCE_FOLDER: links.put("run", "/source/" + fullPath.get(0) + "new_untitled_from_folder" + fullPathString); break; case PHYSICAL_DATASET: links.put("run", "/source/" + fullPath.get(0) + "new_untitled_from_physical_dataset" + fullPathString); break; default: break; } return links; }
final NamespaceKey datasetPath = new NamespaceKey(fullPath); final String dottedFullPath = datasetPath.toUrlEncodedString(); final String fullPathString = PathUtils.toFSPath(fullPath).toString();
@Override public boolean containerExists(NamespaceKey key) { final List<String> folderPath = key.getPathComponents(); try { return getFS(SYSTEM_USERNAME).isDirectory(PathUtils.toFSPath(resolveTableNameToValidPath(folderPath))); } catch (IOException e) { logger.debug("Failure reading path.", e); return false; } }
/** * Temporary location for file upload. * Add uuid so that this location remains unique even across file renames. * @param filePath file path in under home space * @param extension file extension * @return location of staging dir where user file is uploaded. */ private Path getStagingLocation(FilePath filePath, String extension) { FilePath uniquePath = filePath.rename(format("%s_%s-%s", filePath.getFileName().toString(), extension, UUID.randomUUID().toString())); return Path.mergePaths(config.getStagingPath(hostname), PathUtils.toFSPath(uniquePath.toPathList())); }
@Override public boolean datasetExists(NamespaceKey key) { final List<String> filePath = key.getPathComponents(); try { return getFS(SYSTEM_USERNAME).exists(PathUtils.toFSPath(resolveTableNameToValidPath(filePath))); } catch (IOException e) { logger.debug("Failure reading path.", e); return false; } }
@Test public void testToFSPath() throws Exception { assertEquals(new Path("/a/b/c"), PathUtils.toFSPath("a.b.c")); assertEquals(new Path("/a/b"), PathUtils.toFSPath("a.b")); assertEquals(new Path("/c.txt"), PathUtils.toFSPath("\"c.txt\"")); assertEquals(new Path("/a/b/c.txt"), PathUtils.toFSPath("a.b.\"c.txt\"")); }
protected boolean fileExists(String username, List<String> filePath) throws IOException { return getFS(username).isFile(PathUtils.toFSPath(resolveTableNameToValidPath(filePath))); }
@Test public void testHybridSchemaPathToFSPath() throws Exception { assertEquals(new Path("/dfs/tmp/a/b/c"), PathUtils.toFSPath("dfs.tmp.\"a/b/c")); assertEquals(new Path("/dfs/tmp/a/b/c.json"), PathUtils.toFSPath("dfs.tmp.\"a/b/c.json")); assertEquals(new Path("/dfs/tmp/a.txt"), PathUtils.toFSPath("dfs.tmp.\"a.txt")); }
private Path getUploadLocation(FilePath filePath, String extension) { FilePath filePathWithExtension = filePath.rename(format("%s_%s", filePath.getFileName().getName(), extension)); return Path.mergePaths(config.getInnerUploads(), PathUtils.toFSPath(filePathWithExtension.toPathList())); }
public List<SchemaEntity> list(List<String> folderPath, String userName) { try { final List<FileStatus> files = Lists.newArrayList(getFS(userName).listStatus(PathUtils.toFSPath(resolveTableNameToValidPath(folderPath)))); final Set<List<String>> tableNames = Sets.newHashSet(); final NamespaceService ns = context.getNamespaceService(userName);
public Iterable<String> getSubPartitions(List<String> table, List<String> partitionColumns, List<String> partitionValues, SchemaConfig schemaConfig ) throws PartitionNotFoundException { List<FileStatus> fileStatuses; try { Path fullPath = PathUtils.toFSPath(resolveTableNameToValidPath(table)); fileStatuses = getFS(schemaConfig.getUserName()).list(fullPath, false); } catch (IOException e) { throw new PartitionNotFoundException("Error finding partitions for table " + table, e); } return new SubDirectoryList(fileStatuses); }
/** * Fetches a single item from the filesystem plugin */ public SchemaEntity get(List<String> path, String userName) { try { final FileStatus status = getFS(userName).getFileStatus(PathUtils.toFSPath(resolveTableNameToValidPath(path))); final Set<List<String>> tableNames = Sets.newHashSet(); final NamespaceService ns = context.getNamespaceService(userName); final NamespaceKey folderNSKey = new NamespaceKey(path); if (ns.exists(folderNSKey)) { for(NameSpaceContainer entity : ns.list(folderNSKey)) { if (entity.getType() == Type.DATASET) { tableNames.add(resolveTableNameToValidPath(entity.getDataset().getFullPathList())); } } } List<String> p = PathUtils.toPathComponents(status.getPath()); return getSchemaEntity(status, tableNames, p); } catch (IOException | NamespaceException e) { throw new RuntimeException(e); } }
case FOLDER_TABLE: { try { final NamespaceKey namespaceKey = new NamespaceKey(PathUtils.toPathComponents(PathUtils.toFSPath(entityPath))); final DatasetConfig dataset = namespaceService.getDataset(namespaceKey); catalogItem = new CatalogItem.Builder()
final Path fsPath = PathUtils.toFSPath(fullPath);