/** * Creates a file at the given path. * * @param fs a filesystem client * @param path the file path */ public static void createFile(FileSystem fs, AlluxioURI path) throws Exception { try (FileOutStream out = fs.createFile(path, CreateFilePOptions.newBuilder() .setBlockSizeBytes(Constants.KB).setRecursive(true).build())) { out.write("test".getBytes()); } }
@Override public void apply(FileSystem fs) throws Exception { try (FileOutStream out = fs.createFile(FILE, CreateFilePOptions.newBuilder() .setBlockSizeBytes(Constants.KB).setWriteType(WritePType.ASYNC_THROUGH).build())) { out.write("test".getBytes()); } // Nested file try (FileOutStream out = fs.createFile(NESTED_FILE, CreateFilePOptions.newBuilder().setBlockSizeBytes(Constants.KB) .setWriteType(WritePType.ASYNC_THROUGH).setRecursive(true).build())) { out.write("test".getBytes()); } CommonUtils.waitFor("files to be persisted", () -> { try { return fs.getStatus(FILE).isPersisted() && fs.getStatus(NESTED_FILE).isPersisted(); } catch (IOException | AlluxioException e) { throw new RuntimeException(e); } }); }
/** * Sets up the dependencies before a test runs. */ @Before public void before() throws Exception { GroupMappingServiceTestUtils.resetCache(); MetricsSystem.clearAllMetrics(); // This makes sure that the mount point of the UFS corresponding to the Alluxio root ("/") // doesn't exist by default (helps loadRootTest). mUnderFS = ServerConfiguration.get(PropertyKey.MASTER_MOUNT_TABLE_ROOT_UFS); mNestedFileContext = CreateFileContext.defaults( CreateFilePOptions.newBuilder().setBlockSizeBytes(Constants.KB).setRecursive(true)); mJournalFolder = mTestFolder.newFolder().getAbsolutePath(); startServices(); }
mFileSystemMaster.createFile(new AlluxioURI("/testDir/file"), CreateFileContext .defaults(CreateFilePOptions.newBuilder().setBlockSizeBytes(Constants.KB) .setMode(TEST_FILE_MODE.toProto())) .setOwner(TEST_USER_1.getUser()).setGroup(TEST_USER_1.getGroup())); mFileSystemMaster.createFile(new AlluxioURI("/testFile"), CreateFileContext .defaults(CreateFilePOptions.newBuilder().setBlockSizeBytes(Constants.KB) .setMode(TEST_FILE_MODE.toProto())) .setOwner(TEST_USER_2.getUser()).setGroup(TEST_USER_2.getGroup()));
/** * Tests that an exception is thrown when trying to get information about a Directory after * it has been deleted because of a TTL of 0. */ @Test public void setTtlForDirectoryWithNoTtl() throws Exception { CreateDirectoryContext directoryContext = CreateDirectoryContext .defaults(CreateDirectoryPOptions.newBuilder().setRecursive(true)); mFileSystemMaster.createDirectory(NESTED_URI, directoryContext); mFileSystemMaster.createDirectory(NESTED_DIR_URI, directoryContext); CreateFileContext createFileContext = CreateFileContext.defaults( CreateFilePOptions.newBuilder().setBlockSizeBytes(Constants.KB).setRecursive(true)); long fileId = mFileSystemMaster.createFile(NESTED_FILE_URI, createFileContext); HeartbeatScheduler.execute(HeartbeatContext.MASTER_TTL_CHECK); // Since no TTL is set, the file should not be deleted. assertEquals(fileId, mFileSystemMaster.getFileInfo(NESTED_FILE_URI, GET_STATUS_CONTEXT).getFileId()); // Set ttl. mFileSystemMaster.setAttribute(NESTED_URI, SetAttributeContext.defaults(SetAttributePOptions.newBuilder().setTtl(0))); HeartbeatScheduler.execute(HeartbeatContext.MASTER_TTL_CHECK); // TTL is set to 0, the file and directory should have been deleted during last TTL check. mThrown.expect(FileDoesNotExistException.class); mFileSystemMaster.getFileInfo(NESTED_URI, GET_STATUS_CONTEXT); mFileSystemMaster.getFileInfo(NESTED_DIR_URI, GET_STATUS_CONTEXT); mFileSystemMaster.getFileInfo(NESTED_FILE_URI, GET_STATUS_CONTEXT); }
/** * Tests that an exception is thrown when trying to get information about a file after it * has been deleted after the TTL has been set to 0. */ @Test public void setSmallerTtlForFileWithTtl() throws Exception { CreateFileContext context = CreateFileContext.defaults(CreateFilePOptions.newBuilder() .setCommonOptions(FileSystemMasterCommonPOptions.newBuilder().setTtl(Constants.HOUR_MS)) .setBlockSizeBytes(Constants.KB).setRecursive(true)); long fileId = mFileSystemMaster.createFile(NESTED_FILE_URI, context); HeartbeatScheduler.execute(HeartbeatContext.MASTER_TTL_CHECK); // Since TTL is 1 hour, the file won't be deleted during last TTL check. assertEquals(fileId, mFileSystemMaster.getFileInfo(NESTED_FILE_URI, GET_STATUS_CONTEXT).getFileId()); mFileSystemMaster.setAttribute(NESTED_FILE_URI, SetAttributeContext.defaults(SetAttributePOptions.newBuilder().setTtl(0))); HeartbeatScheduler.execute(HeartbeatContext.MASTER_TTL_CHECK); // TTL is reset to 0, the file should have been deleted during last TTL check. mThrown.expect(FileDoesNotExistException.class); mFileSystemMaster.getFileInfo(fileId); }
/** * Sets up dependencies before a single test runs. */ @BeforeClass public static void beforeClass() throws Exception { sFileContext = CreateFileContext .defaults(CreateFilePOptions.newBuilder().setBlockSizeBytes(Constants.KB) .setMode(TEST_FILE_MODE.toProto())) .setOwner(TEST_OWNER).setGroup(TEST_GROUP); sDirectoryContext = CreateDirectoryContext .defaults(CreateDirectoryPOptions.newBuilder().setMode(TEST_DIR_MODE.toProto())) .setOwner(TEST_OWNER).setGroup(TEST_GROUP); sNestedFileContext = CreateFileContext .defaults(CreateFilePOptions.newBuilder().setBlockSizeBytes(Constants.KB) .setMode(TEST_FILE_MODE.toProto()).setRecursive(true)) .setOwner(TEST_OWNER).setGroup(TEST_GROUP); sNestedDirectoryContext = CreateDirectoryContext.defaults( CreateDirectoryPOptions.newBuilder().setMode(TEST_DIR_MODE.toProto()).setRecursive(true)) .setOwner(TEST_OWNER).setGroup(TEST_GROUP); }
/** * @return Master side defaults for {@link CreateFilePOptions} */ public static CreateFilePOptions createFileDefaults() { return CreateFilePOptions.newBuilder() .setCommonOptions(commonDefaults()) .setRecursive(false) .setBlockSizeBytes(ServerConfiguration.getBytes(PropertyKey.USER_BLOCK_SIZE_BYTES_DEFAULT)) .setFileWriteLocationPolicy( ServerConfiguration.get(PropertyKey.USER_FILE_WRITE_LOCATION_POLICY)) .setWriteTier(ServerConfiguration.getInt(PropertyKey.USER_FILE_WRITE_TIER_DEFAULT)) .setWriteType(ServerConfiguration .getEnum(PropertyKey.USER_FILE_WRITE_TYPE_DEFAULT, WriteType.class).toProto()) .setMode(ModeUtils.applyFileUMask(Mode.defaults(), ServerConfiguration.get(PropertyKey.SECURITY_AUTHORIZATION_PERMISSION_UMASK)).toProto()) .setReplicationDurable(ServerConfiguration .getInt(PropertyKey.USER_FILE_REPLICATION_DURABLE)) .setReplicationMin(ServerConfiguration.getInt(PropertyKey.USER_FILE_REPLICATION_MIN)) .setReplicationMax(ServerConfiguration.getInt(PropertyKey.USER_FILE_REPLICATION_MAX)) .build(); }
/** * Tests that an exception is thrown when trying to get information about a file after it * has been deleted because of a TTL of 0. */ @Test public void setTtlForFileWithNoTtl() throws Exception { CreateFileContext context = CreateFileContext.defaults( CreateFilePOptions.newBuilder().setBlockSizeBytes(Constants.KB).setRecursive(true)); long fileId = mFileSystemMaster.createFile(NESTED_FILE_URI, context); HeartbeatScheduler.execute(HeartbeatContext.MASTER_TTL_CHECK); // Since no TTL is set, the file should not be deleted. assertEquals(fileId, mFileSystemMaster.getFileInfo(NESTED_FILE_URI, GET_STATUS_CONTEXT).getFileId()); mFileSystemMaster.setAttribute(NESTED_FILE_URI, SetAttributeContext.defaults(SetAttributePOptions.newBuilder().setTtl(0))); HeartbeatScheduler.execute(HeartbeatContext.MASTER_TTL_CHECK); // TTL is set to 0, the file should have been deleted during last TTL check. mThrown.expect(FileDoesNotExistException.class); mFileSystemMaster.getFileInfo(fileId); }
/** * Tests that a file has not been deleted after the TTL has been reset to a valid value. */ @Test public void setLargerTtlForFileWithTtl() throws Exception { mFileSystemMaster.createDirectory(NESTED_URI, CreateDirectoryContext .defaults(CreateDirectoryPOptions.newBuilder().setRecursive(true))); CreateFileContext context = CreateFileContext.defaults(CreateFilePOptions.newBuilder() .setCommonOptions(FileSystemMasterCommonPOptions.newBuilder().setTtl(0)) .setBlockSizeBytes(Constants.KB).setRecursive(true)); long fileId = mFileSystemMaster.createFile(NESTED_FILE_URI, context); assertEquals(fileId, mFileSystemMaster.getFileInfo(NESTED_FILE_URI, GET_STATUS_CONTEXT).getFileId()); mFileSystemMaster.setAttribute(NESTED_FILE_URI, SetAttributeContext .defaults(SetAttributePOptions.newBuilder().setTtl(Constants.HOUR_MS))); HeartbeatScheduler.execute(HeartbeatContext.MASTER_TTL_CHECK); // TTL is reset to 1 hour, the file should not be deleted during last TTL check. assertEquals(fileId, mFileSystemMaster.getFileInfo(fileId).getFileId()); }
/** * Tests that TTL delete of a file is not forgotten across restarts. */ @Test public void ttlFileDeleteReplay() throws Exception { CreateFileContext context = CreateFileContext.defaults(); context.getOptions().setBlockSizeBytes(Constants.KB); context.getOptions().setRecursive(true); context.getOptions().setCommonOptions(FileSystemMasterCommonPOptions.newBuilder().setTtl(0)); long fileId = mFileSystemMaster.createFile(NESTED_FILE_URI, context); // Simulate restart. stopServices(); startServices(); FileInfo fileInfo = mFileSystemMaster.getFileInfo(fileId); assertEquals(fileInfo.getFileId(), fileId); HeartbeatScheduler.execute(HeartbeatContext.MASTER_TTL_CHECK); mThrown.expect(FileDoesNotExistException.class); mFileSystemMaster.getFileInfo(fileId); }
/** * Tests that the original TTL is removed after setting it to {@link Constants#NO_TTL} for a file. */ @Test public void setNoTtlForFileWithTtl() throws Exception { mFileSystemMaster.createDirectory(NESTED_URI, CreateDirectoryContext .defaults(CreateDirectoryPOptions.newBuilder().setRecursive(true))); CreateFileContext context = CreateFileContext.defaults(CreateFilePOptions.newBuilder() .setCommonOptions(FileSystemMasterCommonPOptions.newBuilder().setTtl(0)) .setBlockSizeBytes(Constants.KB).setRecursive(true)); long fileId = mFileSystemMaster.createFile(NESTED_FILE_URI, context); // After setting TTL to NO_TTL, the original TTL will be removed, and the file will not be // deleted during next TTL check. mFileSystemMaster.setAttribute(NESTED_FILE_URI, SetAttributeContext .defaults(SetAttributePOptions.newBuilder().setTtl(Constants.NO_TTL))); HeartbeatScheduler.execute(HeartbeatContext.MASTER_TTL_CHECK); assertEquals(fileId, mFileSystemMaster.getFileInfo(fileId).getFileId()); }
/** * Tests that an exception is in the * {@link FileSystemMaster#createFile(AlluxioURI, CreateFileContext)} with a * TTL set in the {@link CreateFileContext} after the TTL check was done once. */ @Test public void ttlFileDelete() throws Exception { CreateFileContext context = CreateFileContext.defaults(); context.getOptions().setBlockSizeBytes(Constants.KB); context.getOptions().setRecursive(true); context.getOptions().setCommonOptions(FileSystemMasterCommonPOptions.newBuilder().setTtl(0)); long fileId = mFileSystemMaster.createFile(NESTED_FILE_URI, context); FileInfo fileInfo = mFileSystemMaster.getFileInfo(fileId); assertEquals(fileInfo.getFileId(), fileId); HeartbeatScheduler.execute(HeartbeatContext.MASTER_TTL_CHECK); mThrown.expect(FileDoesNotExistException.class); mFileSystemMaster.getFileInfo(fileId); }
/** * Creates a simple file with {@code len} bytes. * * @param fs a {@link FileSystem} handler * @param fileName the name of the file to be created * @param writeType {@link WritePType} used to create the file * @param len file size * @param blockCapacityByte block size of the file */ public static void createByteFile(FileSystem fs, String fileName, WritePType writeType, int len, long blockCapacityByte) { CreateFilePOptions options = CreateFilePOptions.newBuilder().setWriteType(writeType) .setBlockSizeBytes(blockCapacityByte).setRecursive(true).build(); createByteFile(fs, new AlluxioURI(fileName), options, len); }
@Override public void apply(FileSystem fs) throws Exception { Utils.createFile(fs, PATH); Utils.createFile(fs, NESTED); Utils.createFile(fs, MODE, CreateFilePOptions.newBuilder().setBlockSizeBytes(Constants.KB) .setRecursive(true).setMode(TEST_MODE.toProto()).build()); Utils.createFile(fs, THROUGH, CreateFilePOptions.newBuilder().setBlockSizeBytes(Constants.KB) .setRecursive(true).setWriteType(WritePType.THROUGH).build()); Utils.createFile(fs, TTL, CreateFilePOptions.newBuilder().setBlockSizeBytes(Constants.KB).setRecursive(true) .setCommonOptions(FileSystemMasterCommonPOptions.newBuilder().setTtl(TEST_TTL) .setTtlAction(alluxio.grpc.TtlAction.FREE)) .build()); }
/** * Tests that an exception is thrown when trying to create a file in a non-existing directory * without setting the {@code recursive} flag. */ @Test public void renameUnderNonexistingDir() throws Exception { mThrown.expect(FileDoesNotExistException.class); mThrown.expectMessage(ExceptionMessage.PATH_DOES_NOT_EXIST.getMessage("/nested/test")); CreateFileContext context = CreateFileContext .defaults(CreateFilePOptions.newBuilder().setBlockSizeBytes(Constants.KB)); mFileSystemMaster.createFile(TEST_URI, context); // nested dir mFileSystemMaster.rename(TEST_URI, NESTED_FILE_URI, RenameContext.defaults()); }
/** * @param id block container id of this inode * @return the inode file representation */ protected MutableInodeFile createInodeFile(long id) { return MutableInodeFile.create(id, 1, "testFile" + id, 0, CreateFileContext .defaults(CreateFilePOptions.newBuilder().setBlockSizeBytes(Constants.KB) .setMode(TEST_FILE_MODE.toProto())) .setOwner(TEST_OWNER).setGroup(TEST_GROUP)); } }
/** * Tests that an exception is thrown when trying to create a file with a negative block size. */ @Test public void createFileWithNegativeBlockSize() throws Exception { mThrown.expect(BlockInfoException.class); mThrown.expectMessage("Invalid block size -1"); CreateFileContext context = CreateFileContext.defaults(CreateFilePOptions.newBuilder().setBlockSizeBytes(-1)); createPath(mTree, TEST_URI, context); }
@Test public void renameToNonExistentParent() throws Exception { CreateFileContext context = CreateFileContext.defaults( CreateFilePOptions.newBuilder().setBlockSizeBytes(Constants.KB).setRecursive(true)); mFileSystemMaster.createFile(NESTED_URI, context); try { mFileSystemMaster.rename(NESTED_URI, new AlluxioURI("/testDNE/b"), RenameContext.defaults()); fail("Rename to a non-existent parent path should not succeed."); } catch (FileDoesNotExistException e) { // Expected case. } }
/** * Tests that an exception is thrown when trying to create a file with invalid block size. */ @Test public void createFileWithInvalidBlockSize() throws Exception { mThrown.expect(BlockInfoException.class); mThrown.expectMessage("Invalid block size 0"); CreateFileContext context = CreateFileContext.defaults(CreateFilePOptions.newBuilder().setBlockSizeBytes(0)); createPath(mTree, TEST_URI, context); }