if (!this.util().exists(f)) { return false;
checkDependencies(qSrc, qDst); mkdir(qDst, FsPermission.getDirDefault(), true); FileStatus[] contents = listStatus(qSrc); for (FileStatus content : contents) { copy(makeQualified(content.getPath()), makeQualified(new Path(qDst, content.getPath().getName())), deleteSource, overwrite);
/** * Returns all jars that are in the directory. It is useful in expanding a * wildcard path to return all jars from the directory to use in a classpath. * * @param path the path to the directory. The path may include the wildcard. * @return the list of jars as URLs, or an empty list if there are no jars, or * the directory does not exist */ public static List<Path> getJarsInDirectory(String path, boolean useLocal) { List<Path> paths = new ArrayList<>(); try { // add the wildcard if it is not provided if (!path.endsWith("*")) { path += File.separator + "*"; } Path globPath = new Path(path).suffix("{.jar,.JAR}"); FileContext context = useLocal ? FileContext.getLocalFSFileContext() : FileContext.getFileContext(globPath.toUri()); FileStatus[] files = context.util().globStatus(globPath); if (files != null) { for (FileStatus file: files) { paths.add(file.getPath()); } } } catch (IOException ignore) {} // return the empty list return paths; }
checkDependencies(qSrc, qDst); mkdir(qDst, FsPermission.getDirDefault(), true); FileStatus[] contents = listStatus(qSrc); for (FileStatus content : contents) { copy(makeQualified(content.getPath()), makeQualified(new Path(qDst, content.getPath().getName())), deleteSource, overwrite);
checkDependencies(qSrc, qDst); mkdir(qDst, FsPermission.getDirDefault(), true); FileStatus[] contents = listStatus(qSrc); for (FileStatus content : contents) { copy(makeQualified(content.getPath()), makeQualified(new Path(qDst, content.getPath().getName())), deleteSource, overwrite);
checkDependencies(qSrc, qDst); mkdir(qDst, FsPermission.getDirDefault(), true); FileStatus[] contents = listStatus(qSrc); for (FileStatus content : contents) { copy(makeQualified(content.getPath()), makeQualified(new Path(qDst, content.getPath().getName())), deleteSource, overwrite);
checkDependencies(qSrc, qDst); mkdir(qDst, FsPermission.getDirDefault(), true); FileStatus[] contents = listStatus(qSrc); for (FileStatus content : contents) { copy(makeQualified(content.getPath()), makeQualified(new Path(qDst, content.getPath().getName())), deleteSource, overwrite);
FileStatus s = statusIterator.next(); long length = s.getLen(); ContentSummary c = s.isDirectory() ? getContentSummary(s.getPath()) : new ContentSummary.Builder().length(length).fileCount(1). directoryCount(0).spaceConsumed(length).build();
FileStatus s = statusIterator.next(); long length = s.getLen(); ContentSummary c = s.isDirectory() ? getContentSummary(s.getPath()) : new ContentSummary.Builder().length(length).fileCount(1). directoryCount(0).spaceConsumed(length).build();
FileStatus s = statusIterator.next(); long length = s.getLen(); ContentSummary c = s.isDirectory() ? getContentSummary(s.getPath()) : new ContentSummary.Builder().length(length).fileCount(1). directoryCount(0).spaceConsumed(length).build();
FileStatus s = statusIterator.next(); long length = s.getLen(); ContentSummary c = s.isDirectory() ? getContentSummary(s.getPath()) : new ContentSummary.Builder().length(length).fileCount(1). directoryCount(0).spaceConsumed(length).build();
private FileContext(final AbstractFileSystem defFs, final FsPermission theUmask, final Configuration aConf) { defaultFS = defFs; umask = FsPermission.getUMask(aConf); conf = aConf; try { ugi = UserGroupInformation.getCurrentUser(); } catch (IOException e) { LOG.error("Exception in getCurrentUser: ",e); throw new RuntimeException("Failed to get the current user " + "while creating a FileContext", e); } /* * Init the wd. * WorkingDir is implemented at the FileContext layer * NOT at the AbstractFileSystem layer. * If the DefaultFS, such as localFilesystem has a notion of * builtin WD, we use that as the initial WD. * Otherwise the WD is initialized to the home directory. */ workingDir = defaultFS.getInitialWorkingDirectory(); if (workingDir == null) { workingDir = defaultFS.getHomeDirectory(); } resolveSymlinks = conf.getBoolean( CommonConfigurationKeys.FS_CLIENT_RESOLVE_REMOTE_SYMLINKS_KEY, CommonConfigurationKeys.FS_CLIENT_RESOLVE_REMOTE_SYMLINKS_DEFAULT); util = new Util(); // for the inner class }
private FileContext(final AbstractFileSystem defFs, final FsPermission theUmask, final Configuration aConf) { defaultFS = defFs; umask = FsPermission.getUMask(aConf); conf = aConf; try { ugi = UserGroupInformation.getCurrentUser(); } catch (IOException e) { LOG.error("Exception in getCurrentUser: ",e); throw new RuntimeException("Failed to get the current user " + "while creating a FileContext", e); } /* * Init the wd. * WorkingDir is implemented at the FileContext layer * NOT at the AbstractFileSystem layer. * If the DefaultFS, such as localFilesystem has a notion of * builtin WD, we use that as the initial WD. * Otherwise the WD is initialized to the home directory. */ workingDir = defaultFS.getInitialWorkingDirectory(); if (workingDir == null) { workingDir = defaultFS.getHomeDirectory(); } resolveSymlinks = conf.getBoolean( CommonConfigurationKeys.FS_CLIENT_RESOLVE_REMOTE_SYMLINKS_KEY, CommonConfigurationKeys.FS_CLIENT_RESOLVE_REMOTE_SYMLINKS_DEFAULT); util = new Util(); // for the inner class }
private FileContext(final AbstractFileSystem defFs, final FsPermission theUmask, final Configuration aConf) { defaultFS = defFs; umask = FsPermission.getUMask(aConf); conf = aConf; try { ugi = UserGroupInformation.getCurrentUser(); } catch (IOException e) { LOG.error("Exception in getCurrentUser: ",e); throw new RuntimeException("Failed to get the current user " + "while creating a FileContext", e); } /* * Init the wd. * WorkingDir is implemented at the FileContext layer * NOT at the AbstractFileSystem layer. * If the DefaultFS, such as localFilesystem has a notion of * builtin WD, we use that as the initial WD. * Otherwise the WD is initialized to the home directory. */ workingDir = defaultFS.getInitialWorkingDirectory(); if (workingDir == null) { workingDir = defaultFS.getHomeDirectory(); } resolveSymlinks = conf.getBoolean( CommonConfigurationKeys.FS_CLIENT_RESOLVE_REMOTE_SYMLINKS_KEY, CommonConfigurationKeys.FS_CLIENT_RESOLVE_REMOTE_SYMLINKS_DEFAULT); util = new Util(); // for the inner class }
private FileContext(final AbstractFileSystem defFs, final FsPermission theUmask, final Configuration aConf) { defaultFS = defFs; umask = theUmask; conf = aConf; tracer = FsTracer.get(aConf); try { ugi = UserGroupInformation.getCurrentUser(); } catch (IOException e) { LOG.error("Exception in getCurrentUser: ",e); throw new RuntimeException("Failed to get the current user " + "while creating a FileContext", e); } /* * Init the wd. * WorkingDir is implemented at the FileContext layer * NOT at the AbstractFileSystem layer. * If the DefaultFS, such as localFilesystem has a notion of * builtin WD, we use that as the initial WD. * Otherwise the WD is initialized to the home directory. */ workingDir = defaultFS.getInitialWorkingDirectory(); if (workingDir == null) { workingDir = defaultFS.getHomeDirectory(); } resolveSymlinks = conf.getBoolean( CommonConfigurationKeys.FS_CLIENT_RESOLVE_REMOTE_SYMLINKS_KEY, CommonConfigurationKeys.FS_CLIENT_RESOLVE_REMOTE_SYMLINKS_DEFAULT); util = new Util(); // for the inner class }
@Test public void testGlobStatusSomeMatchesInDirectories() throws Exception { Path[] testDirs = { getTestRootPath(fc, TEST_DIR_AAA), getTestRootPath(fc, TEST_DIR_AXA), getTestRootPath(fc, TEST_DIR_AXX), getTestRootPath(fc, TEST_DIR_AAA2), }; if (exists(fc, testDirs[0]) == false) { for (Path path : testDirs) { fc.mkdir(path, FsPermission.getDefault(), true); } } // Should return two items ("/test/hadoop" and "/test/hadoop2") FileStatus[] paths = fc.util().globStatus( getTestRootPath(fc, "test/hadoop*")); Assert.assertEquals(2, paths.length); Assert.assertTrue(containsPath(getTestRootPath(fc, "test/hadoop"), paths)); Assert.assertTrue(containsPath(getTestRootPath(fc, "test/hadoop2"), paths)); }
@Test public void testGlobStatusSomeMatchesInDirectories() throws Exception { Path[] testDirs = { getTestRootPath(fc, TEST_DIR_AAA), getTestRootPath(fc, TEST_DIR_AXA), getTestRootPath(fc, TEST_DIR_AXX), getTestRootPath(fc, TEST_DIR_AAA2), }; if (exists(fc, testDirs[0]) == false) { for (Path path : testDirs) { fc.mkdir(path, FsPermission.getDefault(), true); } } // Should return two items ("/test/hadoop" and "/test/hadoop2") FileStatus[] paths = fc.util().globStatus( getTestRootPath(fc, "test/hadoop*")); Assert.assertEquals(2, paths.length); Assert.assertTrue(containsPath(getTestRootPath(fc, "test/hadoop"), paths)); Assert.assertTrue(containsPath(getTestRootPath(fc, "test/hadoop2"), paths)); }
@Test public void testGlobStatusWithMultipleMatchesOfSingleChar() throws Exception { Path[] testDirs = { getTestRootPath(fc, TEST_DIR_AAA), getTestRootPath(fc, TEST_DIR_AXA), getTestRootPath(fc, TEST_DIR_AXX), getTestRootPath(fc, TEST_DIR_AAA2), }; if (exists(fc, testDirs[0]) == false) { for (Path path : testDirs) { fc.mkdir(path, FsPermission.getDefault(), true); } } //Should return only 2 items ("/test/hadoop/axa", "/test/hadoop/axx") FileStatus[] paths = fc.util().globStatus( getTestRootPath(fc, "test/hadoop/ax?")); Assert.assertEquals(2, paths.length); Assert.assertTrue(containsPath(getTestRootPath(fc, TEST_DIR_AXA), paths)); Assert.assertTrue(containsPath(getTestRootPath(fc, TEST_DIR_AXX), paths)); }
@Test public void testGlobStatusSomeMatchesInDirectories() throws Exception { Path[] testDirs = { getTestRootPath(fc, TEST_DIR_AAA), getTestRootPath(fc, TEST_DIR_AXA), getTestRootPath(fc, TEST_DIR_AXX), getTestRootPath(fc, TEST_DIR_AAA2), }; if (exists(fc, testDirs[0]) == false) { for (Path path : testDirs) { fc.mkdir(path, FsPermission.getDefault(), true); } } // Should return two items ("/test/hadoop" and "/test/hadoop2") FileStatus[] paths = fc.util().globStatus( getTestRootPath(fc, "test/hadoop*")); Assert.assertEquals(2, paths.length); Assert.assertTrue(containsPath(getTestRootPath(fc, "test/hadoop"), paths)); Assert.assertTrue(containsPath(getTestRootPath(fc, "test/hadoop2"), paths)); }