/** * Append additional path elements to the end of an {@link AlluxioURI}. * * @param suffix the suffix to add * @return the new {@link AlluxioURI} */ public AlluxioURI join(AlluxioURI suffix) { return join(suffix.toString()); }
private void createPersistedDirectoryLevel(int levels, AlluxioURI ufsTop) throws Exception { if (levels <= 0) { return; } // Create files and nested directories in UFS. for (int i = 0; i < DIR_WIDTH; ++i) { Files.createFile(Paths.get(ufsTop.join(FILE_PREFIX + i).getPath())); Files.createDirectories(Paths.get(ufsTop.join(DIR_PREFIX + i).getPath())); createPersistedDirectoryLevel(levels - 1, ufsTop.join(DIR_PREFIX + i)); } }
private AlluxioURI createPersistedDirectories(int levels) throws Exception { AlluxioURI ufsMount = new AlluxioURI(mTestFolder.newFolder().getAbsolutePath()); // Create top-level directory in UFS. Files.createDirectory(Paths.get(ufsMount.join(DIR_TOP_LEVEL).getPath())); createPersistedDirectoryLevel(levels, ufsMount.join(DIR_TOP_LEVEL)); return ufsMount; }
private void checkPersistedDirectoryDeletedLevel(int levels, AlluxioURI alluxioURI, AlluxioURI ufsURI, List<AlluxioURI> except) throws Exception { if (levels <= 0) { return; } if (except.contains(alluxioURI)) { assertTrue(Files.exists(Paths.get(ufsURI.getPath()))); assertNotEquals(IdUtils.INVALID_FILE_ID, mFileSystemMaster.getFileId(alluxioURI)); } else { assertFalse(Files.exists(Paths.get(ufsURI.getPath()))); assertEquals(IdUtils.INVALID_FILE_ID, mFileSystemMaster.getFileId(alluxioURI)); } for (int i = 0; i < DIR_WIDTH; ++i) { // Files can always be deleted. assertFalse(Files.exists(Paths.get(ufsURI.join(FILE_PREFIX + i).getPath()))); assertEquals(IdUtils.INVALID_FILE_ID, mFileSystemMaster.getFileId(alluxioURI.join(FILE_PREFIX + i))); checkPersistedDirectoryDeletedLevel(levels - 1, alluxioURI.join(DIR_PREFIX + i), ufsURI.join(DIR_PREFIX + i), except); } }
/** * Tests the {@link FileSystemMaster#delete(AlluxioURI, DeleteContext)} method for * a multi-level directory with un-synced persistent entries without a sync check. */ @Test public void deleteUnsyncedPersistedMultilevelDirectoryWithoutCheck() throws Exception { AlluxioURI ufsMount = createPersistedDirectories(3); mountPersistedDirectories(ufsMount); loadPersistedDirectories(3); // Add a file to the UFS down the tree. Files.createFile(Paths.get(ufsMount.join(DIR_TOP_LEVEL).join(DIR_PREFIX + 0) .join(FILE_PREFIX + (DIR_WIDTH)).getPath())); // delete top-level directory mFileSystemMaster.delete(new AlluxioURI(MOUNT_URI).join(DIR_TOP_LEVEL), DeleteContext.defaults( DeletePOptions.newBuilder().setRecursive(true).setAlluxioOnly(false).setUnchecked(true))); checkPersistedDirectoriesDeleted(3, ufsMount, Collections.EMPTY_LIST); }
@Test public void listStatusWithLoadMetadataNever() throws Exception { AlluxioURI ufsMount = new AlluxioURI(mTestFolder.newFolder().getAbsolutePath()); mFileSystemMaster.createDirectory(new AlluxioURI("/mnt/"), CreateDirectoryContext.defaults()); // Create ufs file. Files.createDirectory(Paths.get(ufsMount.join("dir1").getPath())); Files.createFile(Paths.get(ufsMount.join("dir1").join("file1").getPath())); Files.createFile(Paths.get(ufsMount.join("dir1").join("file2").getPath())); mFileSystemMaster.mount(new AlluxioURI("/mnt/local"), ufsMount, MountContext.defaults()); // 3 directories exist. assertEquals(3, countPaths()); // getFileId should load metadata automatically. AlluxioURI uri = new AlluxioURI("/mnt/local/dir1"); try { mFileSystemMaster.listStatus(uri, ListStatusContext .defaults(ListStatusPOptions.newBuilder().setLoadMetadataType(LoadMetadataPType.NEVER))); fail("Exception expected"); } catch (FileDoesNotExistException e) { // Expected case. } assertEquals(3, countPaths()); }
/** * Tests the {@link FileSystemMaster#delete(AlluxioURI, DeleteContext)} method for * a directory with un-synced persistent entries without a sync check. */ @Test public void deleteUnsyncedPersistedDirectoryWithoutCheck() throws Exception { AlluxioURI ufsMount = createPersistedDirectories(1); mountPersistedDirectories(ufsMount); loadPersistedDirectories(1); // Add a file to the UFS. Files.createFile( Paths.get(ufsMount.join(DIR_TOP_LEVEL).join(FILE_PREFIX + (DIR_WIDTH)).getPath())); // delete top-level directory mFileSystemMaster.delete(new AlluxioURI(MOUNT_URI).join(DIR_TOP_LEVEL), DeleteContext.defaults( DeletePOptions.newBuilder().setRecursive(true).setAlluxioOnly(false).setUnchecked(true))); checkPersistedDirectoriesDeleted(1, ufsMount, Collections.EMPTY_LIST); }
/** * Tests unmount operation. */ @Test public void unmount() throws Exception { AlluxioURI alluxioURI = new AlluxioURI("/hello"); AlluxioURI ufsURI = createTempUfsDir("ufs/hello"); mFileSystemMaster.mount(alluxioURI, ufsURI, MountContext.defaults()); mFileSystemMaster.createDirectory(alluxioURI.join("dir"), CreateDirectoryContext .defaults().setPersisted(true)); mFileSystemMaster.unmount(alluxioURI); // after unmount, ufs path under previous mount point should still exist File file = new File(ufsURI.join("dir").toString()); assertTrue(file.exists()); // after unmount, alluxio path under previous mount point should not exist mThrown.expect(FileDoesNotExistException.class); mFileSystemMaster.getFileInfo(alluxioURI.join("dir"), GET_STATUS_CONTEXT); }
private void loadPersistedDirectoryLevel(int levels, AlluxioURI alluxioTop) throws Exception { if (levels <= 0) { return; } mFileSystemMaster.listStatus(alluxioTop, ListStatusContext .defaults(ListStatusPOptions.newBuilder().setLoadMetadataType(LoadMetadataPType.ALWAYS))); for (int i = 0; i < DIR_WIDTH; ++i) { loadPersistedDirectoryLevel(levels - 1, alluxioTop.join(DIR_PREFIX + i)); } }
/** * Checks for absent paths (descendants) and existing paths (ancestors) in the UFS. * * @param firstAbsent the first Alluxio path which should not exist in the UFS */ private void checkAbsentPaths(AlluxioURI firstAbsent) throws Exception { // Check for additional non-existing paths as descendants of the first absent path for (int level = 1; level <= 2; level++) { AlluxioURI levelUri = firstAbsent.join("level" + level); for (int dir = 1; dir <= 2; dir++) { AlluxioURI uri = levelUri.join("dir" + dir); Assert.assertTrue(uri.toString(), mUfsAbsentPathCache.isAbsent(uri)); } } // Check all ancestors AlluxioURI existing = firstAbsent.getParent(); while (existing != null) { Assert.assertFalse(existing.toString(), mUfsAbsentPathCache.isAbsent(existing)); existing = existing.getParent(); } } }
@Test public void get() throws Exception { final long blockId = IdUtils.getRandomNonNegativeLong(); final AlluxioURI fileUri = new AlluxioURI("/mnt/file"); final String localFilePath = new AlluxioURI(mLocalUfsPath).join("file").getPath(); mLocalUfs.create(localFilePath); final List<String> ufsLocations = mLocalUfs.getFileLocations(localFilePath); for (String location : ufsLocations) { System.out.println(location); } Assert.assertNull(mUfsBlockLocationCache.get(blockId)); List<String> locations = mUfsBlockLocationCache.get(blockId, fileUri, 0); Assert.assertArrayEquals(ufsLocations.toArray(), locations.toArray()); locations = mUfsBlockLocationCache.get(blockId); Assert.assertArrayEquals(ufsLocations.toArray(), locations.toArray()); mUfsBlockLocationCache.invalidate(blockId); Assert.assertNull(mUfsBlockLocationCache.get(blockId)); } }
private void loadPersistedDirectories(int levels) throws Exception { // load persisted ufs entries to alluxio mFileSystemMaster.listStatus(new AlluxioURI(MOUNT_URI), ListStatusContext .defaults(ListStatusPOptions.newBuilder().setLoadMetadataType(LoadMetadataPType.ALWAYS))); loadPersistedDirectoryLevel(levels, new AlluxioURI(MOUNT_URI).join(DIR_TOP_LEVEL)); }
@Test public void getFileIdWithLoadMetadata() throws Exception { AlluxioURI ufsMount = new AlluxioURI(mTestFolder.newFolder().getAbsolutePath()); mFileSystemMaster.createDirectory(new AlluxioURI("/mnt/"), CreateDirectoryContext.defaults()); // Create ufs file. Files.createFile(Paths.get(ufsMount.join("file").getPath())); mFileSystemMaster.mount(new AlluxioURI("/mnt/local"), ufsMount, MountContext.defaults()); // 3 directories exist. assertEquals(3, countPaths()); // getFileId should load metadata automatically. AlluxioURI uri = new AlluxioURI("/mnt/local/file"); assertNotEquals(IdUtils.INVALID_FILE_ID, mFileSystemMaster.getFileId(uri)); // getFileId should have loaded another file, so now 4 paths exist. assertEquals(4, countPaths()); }
@Test public void getFileInfoWithLoadMetadata() throws Exception { AlluxioURI ufsMount = new AlluxioURI(mTestFolder.newFolder().getAbsolutePath()); mFileSystemMaster.createDirectory(new AlluxioURI("/mnt/"), CreateDirectoryContext.defaults()); // Create ufs file. Files.createFile(Paths.get(ufsMount.join("file").getPath())); mFileSystemMaster.mount(new AlluxioURI("/mnt/local"), ufsMount, MountContext.defaults()); // 3 directories exist. assertEquals(3, countPaths()); // getFileInfo should load metadata automatically. AlluxioURI uri = new AlluxioURI("/mnt/local/file"); assertEquals(uri.getPath(), mFileSystemMaster.getFileInfo(uri, GET_STATUS_CONTEXT).getPath()); // getFileInfo should have loaded another file, so now 4 paths exist. assertEquals(4, countPaths()); }
/** * Tests the method for getting a copy of the current mount table. */ @Test public void getMountTable() throws Exception { Map<String, MountInfo> mountTable = new HashMap<>(2); mountTable.put("/mnt/foo", new MountInfo(new AlluxioURI("/mnt/foo"), new AlluxioURI("hdfs://localhost:5678/foo"), 2L, MountContext.defaults().getOptions().build())); mountTable.put("/mnt/bar", new MountInfo(new AlluxioURI("/mnt/bar"), new AlluxioURI("hdfs://localhost:5678/bar"), 3L, MountContext.defaults().getOptions().build())); AlluxioURI masterAddr = new AlluxioURI("alluxio://localhost:1234"); for (Map.Entry<String, MountInfo> mountPoint : mountTable.entrySet()) { MountInfo mountInfo = mountPoint.getValue(); mMountTable.add(NoopJournalContext.INSTANCE, masterAddr.join(mountPoint.getKey()), mountInfo.getUfsUri(), mountInfo.getMountId(), mountInfo.getOptions()); } // Add root mountpoint mountTable.put("/", new MountInfo(new AlluxioURI("/"), new AlluxioURI("s3a://bucket/"), IdUtils.ROOT_MOUNT_ID, MountContext.defaults().getOptions().build())); Assert.assertEquals(mountTable, mMountTable.getMountTable()); }
/** * Tests unmount operation failed when unmounting non-mount point. */ @Test public void unmountNonMountPointWithException() throws Exception { AlluxioURI alluxioURI = new AlluxioURI("/hello"); AlluxioURI ufsURI = createTempUfsDir("ufs/hello"); mFileSystemMaster.mount(alluxioURI, ufsURI, MountContext.defaults()); AlluxioURI dirURI = alluxioURI.join("dir"); mFileSystemMaster.createDirectory(dirURI, CreateDirectoryContext .defaults().setPersisted(true)); mThrown.expect(InvalidPathException.class); mFileSystemMaster.unmount(dirURI); }
private void deleteSyncedPersistedDirectory(int levels, boolean unchecked) throws Exception { AlluxioURI ufsMount = createPersistedDirectories(levels); mountPersistedDirectories(ufsMount); loadPersistedDirectories(levels); // delete top-level directory mFileSystemMaster.delete(new AlluxioURI(MOUNT_URI).join(DIR_TOP_LEVEL), DeleteContext.defaults(DeletePOptions.newBuilder().setRecursive(true).setAlluxioOnly(false) .setUnchecked(unchecked))); checkPersistedDirectoriesDeleted(levels, ufsMount, Collections.EMPTY_LIST); }
/** * Tests the {@link AlluxioURI#AlluxioURI(String)} constructor for an empty URI. */ @Test public void emptyURI() { AlluxioURI uri = new AlluxioURI(""); assertEquals("", uri.getAuthority().toString()); assertTrue(uri.getAuthority() instanceof NoAuthority); assertEquals(0, uri.getDepth()); assertEquals("", uri.getName()); assertEquals("", uri.getPath()); assertEquals(null, uri.getScheme()); assertFalse(uri.hasAuthority()); assertFalse(uri.hasScheme()); assertFalse(uri.isAbsolute()); assertFalse(uri.isPathAbsolute()); assertEquals("/d", uri.join("/d").toString()); assertEquals("/d", uri.join(new AlluxioURI("/d")).toString()); assertEquals("", uri.toString()); }
/** * Tests the {@link FileSystemMaster#getFileId(AlluxioURI)} method. */ @Test public void getFileId() throws Exception { createFileWithSingleBlock(NESTED_FILE_URI); // These URIs exist. assertNotEquals(IdUtils.INVALID_FILE_ID, mFileSystemMaster.getFileId(ROOT_URI)); assertEquals(ROOT_URI, mFileSystemMaster.getPath(mFileSystemMaster.getFileId(ROOT_URI))); assertNotEquals(IdUtils.INVALID_FILE_ID, mFileSystemMaster.getFileId(NESTED_URI)); assertEquals(NESTED_URI, mFileSystemMaster.getPath(mFileSystemMaster.getFileId(NESTED_URI))); assertNotEquals(IdUtils.INVALID_FILE_ID, mFileSystemMaster.getFileId(NESTED_FILE_URI)); assertEquals(NESTED_FILE_URI, mFileSystemMaster.getPath(mFileSystemMaster.getFileId(NESTED_FILE_URI))); // These URIs do not exist. assertEquals(IdUtils.INVALID_FILE_ID, mFileSystemMaster.getFileId(ROOT_FILE_URI)); assertEquals(IdUtils.INVALID_FILE_ID, mFileSystemMaster.getFileId(TEST_URI)); assertEquals(IdUtils.INVALID_FILE_ID, mFileSystemMaster.getFileId(NESTED_FILE_URI.join("DNE"))); }