.setLoadDescendantType(GrpcUtils.toProto(loadDescendantType)) .setCommonOptions(FileSystemMasterCommonPOptions.newBuilder() .setTtl(context.getOptions().getCommonOptions().getTtl()) .setTtlAction(context.getOptions().getCommonOptions().getTtlAction()))); Inode inode;
LoadMetadataContext.defaults(LoadMetadataPOptions.newBuilder().setCreateAncestors(true) .setCommonOptions(FileSystemMasterCommonPOptions.newBuilder() .setTtl(context.getOptions().getCommonOptions().getTtl()) .setTtlAction(context.getOptions().getCommonOptions().getTtlAction())))); ensureFullPathAndUpdateCache(inodePath);
createFileContext.getOptions() .setCommonOptions(FileSystemMasterCommonPOptions.newBuilder() .setTtl(context.getOptions().getCommonOptions().getTtl()) .setTtlAction(context.getOptions().getCommonOptions().getTtlAction())); createFileContext.setMetadataLoad(true);
.setRecursive(context.getOptions().getCreateAncestors()).setAllowExists(false) .setCommonOptions(FileSystemMasterCommonPOptions.newBuilder() .setTtl(context.getOptions().getCommonOptions().getTtl()) .setTtlAction(context.getOptions().getCommonOptions().getTtlAction())); createDirectoryContext.setMountPoint(mMountTable.isMountPoint(inodePath.getUri()));
@Override public void apply(FileSystem fs) throws Exception { fs.createDirectory(DIR); fs.createDirectory(NESTED_DIR); fs.createDirectory(NESTED_NESTED_DIR); fs.createDirectory(RECURSIVE, CreateDirectoryPOptions.newBuilder().setRecursive(true).build()); fs.createDirectory(RECURSIVE, CreateDirectoryPOptions.newBuilder().setAllowExists(true).build()); fs.createDirectory(MODE_DIR, CreateDirectoryPOptions.newBuilder().setMode(TEST_MODE.toProto()) .setRecursive(true).build()); // Set TTL via common options instead (should have the same effect). fs.createDirectory(COMMON_TTL_DIR, CreateDirectoryPOptions.newBuilder().setRecursive(true).setCommonOptions( FileSystemMasterCommonPOptions.newBuilder().setTtl(TTL).setTtlAction(TtlAction.DELETE)) .build()); fs.createDirectory(TTL_DIR, CreateDirectoryPOptions.newBuilder().setCommonOptions( FileSystemMasterCommonPOptions.newBuilder().setTtl(TTL).setTtlAction(TtlAction.DELETE)) .setRecursive(true).build()); fs.createDirectory(THROUGH_DIR, CreateDirectoryPOptions.newBuilder() .setWriteType(WritePType.THROUGH).setRecursive(true).build()); fs.createDirectory(ALL_OPTS_DIR, CreateDirectoryPOptions.newBuilder().setRecursive(true) .setMode(TEST_MODE.toProto()).setAllowExists(true).setWriteType(WritePType.THROUGH) .setCommonOptions( FileSystemMasterCommonPOptions.newBuilder().setTtl(TTL).setTtlAction(TtlAction.DELETE)) .build()); }
/** * Tests that an exception is thrown when trying to get information about a file after it * has been deleted after the TTL has been set to 0. */ @Test public void setSmallerTtlForFileWithTtl() throws Exception { CreateFileContext context = CreateFileContext.defaults(CreateFilePOptions.newBuilder() .setCommonOptions(FileSystemMasterCommonPOptions.newBuilder().setTtl(Constants.HOUR_MS)) .setBlockSizeBytes(Constants.KB).setRecursive(true)); long fileId = mFileSystemMaster.createFile(NESTED_FILE_URI, context); HeartbeatScheduler.execute(HeartbeatContext.MASTER_TTL_CHECK); // Since TTL is 1 hour, the file won't be deleted during last TTL check. assertEquals(fileId, mFileSystemMaster.getFileInfo(NESTED_FILE_URI, GET_STATUS_CONTEXT).getFileId()); mFileSystemMaster.setAttribute(NESTED_FILE_URI, SetAttributeContext.defaults(SetAttributePOptions.newBuilder().setTtl(0))); HeartbeatScheduler.execute(HeartbeatContext.MASTER_TTL_CHECK); // TTL is reset to 0, the file should have been deleted during last TTL check. mThrown.expect(FileDoesNotExistException.class); mFileSystemMaster.getFileInfo(fileId); }
/** * Tests that the original TTL is removed after setting it to {@link Constants#NO_TTL} for * a directory. */ @Test public void setNoTtlForDirectoryWithTtl() throws Exception { mFileSystemMaster.createDirectory(new AlluxioURI("/nested"), CreateDirectoryContext.defaults(CreateDirectoryPOptions.newBuilder().setRecursive(true))); mFileSystemMaster.createDirectory(NESTED_URI, CreateDirectoryContext.defaults(CreateDirectoryPOptions.newBuilder() .setCommonOptions(FileSystemMasterCommonPOptions.newBuilder().setTtl(0)) .setRecursive(true))); // After setting TTL to NO_TTL, the original TTL will be removed, and the file will not be // deleted during next TTL check. mFileSystemMaster.setAttribute(NESTED_URI, SetAttributeContext .defaults(SetAttributePOptions.newBuilder().setTtl(Constants.NO_TTL))); HeartbeatScheduler.execute(HeartbeatContext.MASTER_TTL_CHECK); assertEquals(NESTED_URI.getName(), mFileSystemMaster.getFileInfo(NESTED_URI, GET_STATUS_CONTEXT).getName()); }
/** * Tests that a file has not been deleted after the TTL has been reset to a valid value. */ @Test public void setLargerTtlForFileWithTtl() throws Exception { mFileSystemMaster.createDirectory(NESTED_URI, CreateDirectoryContext .defaults(CreateDirectoryPOptions.newBuilder().setRecursive(true))); CreateFileContext context = CreateFileContext.defaults(CreateFilePOptions.newBuilder() .setCommonOptions(FileSystemMasterCommonPOptions.newBuilder().setTtl(0)) .setBlockSizeBytes(Constants.KB).setRecursive(true)); long fileId = mFileSystemMaster.createFile(NESTED_FILE_URI, context); assertEquals(fileId, mFileSystemMaster.getFileInfo(NESTED_FILE_URI, GET_STATUS_CONTEXT).getFileId()); mFileSystemMaster.setAttribute(NESTED_FILE_URI, SetAttributeContext .defaults(SetAttributePOptions.newBuilder().setTtl(Constants.HOUR_MS))); HeartbeatScheduler.execute(HeartbeatContext.MASTER_TTL_CHECK); // TTL is reset to 1 hour, the file should not be deleted during last TTL check. assertEquals(fileId, mFileSystemMaster.getFileInfo(fileId).getFileId()); }
/** * Tests that TTL delete of a file is not forgotten across restarts. */ @Test public void ttlFileDeleteReplay() throws Exception { CreateFileContext context = CreateFileContext.defaults(); context.getOptions().setBlockSizeBytes(Constants.KB); context.getOptions().setRecursive(true); context.getOptions().setCommonOptions(FileSystemMasterCommonPOptions.newBuilder().setTtl(0)); long fileId = mFileSystemMaster.createFile(NESTED_FILE_URI, context); // Simulate restart. stopServices(); startServices(); FileInfo fileInfo = mFileSystemMaster.getFileInfo(fileId); assertEquals(fileInfo.getFileId(), fileId); HeartbeatScheduler.execute(HeartbeatContext.MASTER_TTL_CHECK); mThrown.expect(FileDoesNotExistException.class); mFileSystemMaster.getFileInfo(fileId); }
/** * Tests that an exception is thrown when trying to get information about a Directory after * it has been deleted after the TTL has been set to 0. */ @Test public void setSmallerTtlForDirectoryWithTtl() throws Exception { CreateDirectoryContext directoryContext = CreateDirectoryContext.defaults(CreateDirectoryPOptions.newBuilder() .setCommonOptions(FileSystemMasterCommonPOptions.newBuilder().setTtl(Constants.HOUR_MS)) .setRecursive(true)); mFileSystemMaster.createDirectory(NESTED_URI, directoryContext); HeartbeatScheduler.execute(HeartbeatContext.MASTER_TTL_CHECK); assertTrue( mFileSystemMaster.getFileInfo(NESTED_URI, GET_STATUS_CONTEXT).getName() != null); mFileSystemMaster.setAttribute(NESTED_URI, SetAttributeContext.defaults(SetAttributePOptions.newBuilder().setTtl(0))); HeartbeatScheduler.execute(HeartbeatContext.MASTER_TTL_CHECK); // TTL is reset to 0, the file should have been deleted during last TTL check. mThrown.expect(FileDoesNotExistException.class); mFileSystemMaster.getFileInfo(NESTED_URI, GET_STATUS_CONTEXT); }
/** * Tests that the original TTL is removed after setting it to {@link Constants#NO_TTL} for a file. */ @Test public void setNoTtlForFileWithTtl() throws Exception { mFileSystemMaster.createDirectory(NESTED_URI, CreateDirectoryContext .defaults(CreateDirectoryPOptions.newBuilder().setRecursive(true))); CreateFileContext context = CreateFileContext.defaults(CreateFilePOptions.newBuilder() .setCommonOptions(FileSystemMasterCommonPOptions.newBuilder().setTtl(0)) .setBlockSizeBytes(Constants.KB).setRecursive(true)); long fileId = mFileSystemMaster.createFile(NESTED_FILE_URI, context); // After setting TTL to NO_TTL, the original TTL will be removed, and the file will not be // deleted during next TTL check. mFileSystemMaster.setAttribute(NESTED_FILE_URI, SetAttributeContext .defaults(SetAttributePOptions.newBuilder().setTtl(Constants.NO_TTL))); HeartbeatScheduler.execute(HeartbeatContext.MASTER_TTL_CHECK); assertEquals(fileId, mFileSystemMaster.getFileInfo(fileId).getFileId()); }
/** * Tests that an exception is in the * {@link FileSystemMaster#createFile(AlluxioURI, CreateFileContext)} with a * TTL set in the {@link CreateFileContext} after the TTL check was done once. */ @Test public void ttlFileDelete() throws Exception { CreateFileContext context = CreateFileContext.defaults(); context.getOptions().setBlockSizeBytes(Constants.KB); context.getOptions().setRecursive(true); context.getOptions().setCommonOptions(FileSystemMasterCommonPOptions.newBuilder().setTtl(0)); long fileId = mFileSystemMaster.createFile(NESTED_FILE_URI, context); FileInfo fileInfo = mFileSystemMaster.getFileInfo(fileId); assertEquals(fileInfo.getFileId(), fileId); HeartbeatScheduler.execute(HeartbeatContext.MASTER_TTL_CHECK); mThrown.expect(FileDoesNotExistException.class); mFileSystemMaster.getFileInfo(fileId); }
/** * Tests that TTL delete of a directory is not forgotten across restarts. */ @Test public void ttlDirectoryDeleteReplay() throws Exception { CreateDirectoryContext context = CreateDirectoryContext.defaults(CreateDirectoryPOptions.newBuilder().setRecursive(true) .setCommonOptions(FileSystemMasterCommonPOptions.newBuilder().setTtl(0))); long dirId = mFileSystemMaster.createDirectory(NESTED_DIR_URI, context); // Simulate restart. stopServices(); startServices(); FileInfo fileInfo = mFileSystemMaster.getFileInfo(dirId); assertEquals(fileInfo.getFileId(), dirId); HeartbeatScheduler.execute(HeartbeatContext.MASTER_TTL_CHECK); mThrown.expect(FileDoesNotExistException.class); mFileSystemMaster.getFileInfo(dirId); }
/** * Tests that an exception is in the * {@literal FileSystemMaster#createDirectory(AlluxioURI, CreateDirectoryOptions)} * with a TTL set in the {@link CreateDirectoryContext} after the TTL check was done once. */ @Test public void ttlDirectoryDelete() throws Exception { CreateDirectoryContext context = CreateDirectoryContext.defaults(CreateDirectoryPOptions.newBuilder().setRecursive(true) .setCommonOptions(FileSystemMasterCommonPOptions.newBuilder().setTtl(0))); long dirId = mFileSystemMaster.createDirectory(NESTED_DIR_URI, context); FileInfo fileInfo = mFileSystemMaster.getFileInfo(dirId); assertEquals(fileInfo.getFileId(), dirId); HeartbeatScheduler.execute(HeartbeatContext.MASTER_TTL_CHECK); mThrown.expect(FileDoesNotExistException.class); mFileSystemMaster.getFileInfo(dirId); }
/** * Tests that a directory has not been deleted after the TTL has been reset to a valid value. */ @Test public void setLargerTtlForDirectoryWithTtl() throws Exception { mFileSystemMaster.createDirectory(new AlluxioURI("/nested"), CreateDirectoryContext.defaults(CreateDirectoryPOptions.newBuilder().setRecursive(true))); mFileSystemMaster.createDirectory(NESTED_URI, CreateDirectoryContext.defaults(CreateDirectoryPOptions.newBuilder() .setCommonOptions(FileSystemMasterCommonPOptions.newBuilder().setTtl(0)) .setRecursive(true))); mFileSystemMaster.setAttribute(NESTED_URI, SetAttributeContext .defaults(SetAttributePOptions.newBuilder().setTtl(Constants.HOUR_MS))); HeartbeatScheduler.execute(HeartbeatContext.MASTER_TTL_CHECK); // TTL is reset to 1 hour, the directory should not be deleted during last TTL check. assertEquals(NESTED_URI.getName(), mFileSystemMaster.getFileInfo(NESTED_URI, GET_STATUS_CONTEXT).getName()); }
@Override public void apply(FileSystem fs) throws Exception { Utils.createFile(fs, PATH); Utils.createFile(fs, NESTED); Utils.createFile(fs, MODE, CreateFilePOptions.newBuilder().setBlockSizeBytes(Constants.KB) .setRecursive(true).setMode(TEST_MODE.toProto()).build()); Utils.createFile(fs, THROUGH, CreateFilePOptions.newBuilder().setBlockSizeBytes(Constants.KB) .setRecursive(true).setWriteType(WritePType.THROUGH).build()); Utils.createFile(fs, TTL, CreateFilePOptions.newBuilder().setBlockSizeBytes(Constants.KB).setRecursive(true) .setCommonOptions(FileSystemMasterCommonPOptions.newBuilder().setTtl(TEST_TTL) .setTtlAction(alluxio.grpc.TtlAction.FREE)) .build()); }
public Builder mergeFrom(alluxio.grpc.FileSystemMasterCommonPOptions other) { if (other == alluxio.grpc.FileSystemMasterCommonPOptions.getDefaultInstance()) return this; if (other.hasSyncIntervalMs()) { setSyncIntervalMs(other.getSyncIntervalMs()); } if (other.hasTtl()) { setTtl(other.getTtl()); } if (other.hasTtlAction()) { setTtlAction(other.getTtlAction()); } this.mergeUnknownFields(other.unknownFields); onChanged(); return this; }
/** * @return {@link FileSystemMasterCommonPOptions} with default values for master */ private static FileSystemMasterCommonPOptions commonDefaults() { return FileSystemMasterCommonPOptions.newBuilder() .setTtl(ServerConfiguration.getLong(PropertyKey.USER_FILE_CREATE_TTL)) .setTtlAction( ServerConfiguration.getEnum(PropertyKey.USER_FILE_CREATE_TTL_ACTION, TtlAction.class)) .setSyncIntervalMs(ServerConfiguration.getMs(PropertyKey.USER_FILE_METADATA_SYNC_INTERVAL)) .build(); }
public static Inode createDirectoryWithIdAndTtl(long id, long ttl) { return Inode.wrap(MutableInodeFile.create(id, 0, "ignored", 0, CreateFileContext.defaults(CreateFilePOptions.newBuilder() .setCommonOptions(FileSystemMasterCommonPOptions.newBuilder().setTtl(ttl))))); } }
public static Inode createFileWithIdAndTtl(long id, long ttl) { return Inode.wrap(MutableInodeFile.create(id, 0, "ignored", 0, CreateFileContext.defaults(CreateFilePOptions.newBuilder() .setCommonOptions(FileSystemMasterCommonPOptions.newBuilder().setTtl(ttl))))); }