/** * Attempts to create a folder with the specified path. Parent directories will be created. * * @param path path to create * @param permission permissions to grant the created folder * @return true if the indicated folder is created successfully or already exists */ @Override public boolean mkdirs(Path path, FsPermission permission) throws IOException { LOG.debug("mkdirs({}, {})", path, permission); if (mStatistics != null) { mStatistics.incrementWriteOps(1); } AlluxioURI uri = new AlluxioURI(HadoopUtils.getPathWithoutScheme(path)); CreateDirectoryPOptions options = CreateDirectoryPOptions.newBuilder().setRecursive(true) .setAllowExists(true).setMode(new Mode(permission.toShort()).toProto()).build(); try { mFileSystem.createDirectory(uri, options); return true; } catch (AlluxioException e) { throw new IOException(e); } }
/** * <code>optional .alluxio.grpc.file.FileSystemMasterCommonPOptions commonOptions = 5;</code> */ private com.google.protobuf.SingleFieldBuilderV3< alluxio.grpc.FileSystemMasterCommonPOptions, alluxio.grpc.FileSystemMasterCommonPOptions.Builder, alluxio.grpc.FileSystemMasterCommonPOptionsOrBuilder> getCommonOptionsFieldBuilder() { if (commonOptionsBuilder_ == null) { commonOptionsBuilder_ = new com.google.protobuf.SingleFieldBuilderV3< alluxio.grpc.FileSystemMasterCommonPOptions, alluxio.grpc.FileSystemMasterCommonPOptions.Builder, alluxio.grpc.FileSystemMasterCommonPOptionsOrBuilder>( getCommonOptions(), getParentForChildren(), isClean()); commonOptions_ = null; } return commonOptionsBuilder_; } public final Builder setUnknownFields(
/** * <code>optional .alluxio.grpc.PMode mode = 3;</code> */ private com.google.protobuf.SingleFieldBuilderV3< alluxio.grpc.PMode, alluxio.grpc.PMode.Builder, alluxio.grpc.PModeOrBuilder> getModeFieldBuilder() { if (modeBuilder_ == null) { modeBuilder_ = new com.google.protobuf.SingleFieldBuilderV3< alluxio.grpc.PMode, alluxio.grpc.PMode.Builder, alluxio.grpc.PModeOrBuilder>( getMode(), getParentForChildren(), isClean()); mode_ = null; } return modeBuilder_; }
&& ((CreateDirectoryContext) context).getOptions().getAllowExists()) { return new ArrayList<>(); } else { missingDirContext.getOptions().setCommonOptions(FileSystemMasterCommonPOptions.newBuilder() .setTtl(context.getTtl()).setTtlAction(context.getTtlAction())); missingDirContext.setPersisted(context.isPersisted());
.setRecursive(context.getOptions().getCreateAncestors()).setAllowExists(false) .setCommonOptions(FileSystemMasterCommonPOptions.newBuilder() .setTtl(context.getOptions().getCommonOptions().getTtl()) .setTtlAction(context.getOptions().getCommonOptions().getTtlAction())); mode.setOtherBits(mode.getOtherBits().or(mode.getOwnerBits())); createDirectoryContext.getOptions().setMode(mode.toProto()); createDirectoryContext.setOwner(ufsOwner).setGroup(ufsGroup) .setUfsStatus(context.getUfsStatus());
subBuilder.mergeFrom(options_); options_ = subBuilder.buildPartial();
CreateDirectoryPOptions.newBuilder().setRecursive(true).setMode(TEST_DIR_MODE.toProto())) .setOwner(TEST_OWNER).setGroup(TEST_GROUP);
ServerConfiguration.global()).toResource()) { mFileSystemMaster.createDirectory(new AlluxioURI("/testDir"), CreateDirectoryContext .defaults(CreateDirectoryPOptions.newBuilder().setMode(TEST_DIR_MODE.toProto())) .setOwner(TEST_USER_1.getUser()).setGroup(TEST_USER_1.getGroup()));
LOG.debug("createDirectory {} ", path); Metrics.CREATE_DIRECTORIES_OPS.inc(); LockingScheme lockingScheme = createLockingScheme(path, context.getOptions().getCommonOptions(), LockPattern.WRITE_EDGE); try (RpcContext rpcContext = createRpcContext(); FileSystemMasterAuditContext auditContext = createAuditContext("mkdir", path, null, inodePath.getParentInodeOrNull())) { if (context.getOptions().getRecursive()) { auditContext.setSrcInode(inodePath.getLastExistingInode());
/** * Tests that an exception is thrown when trying to get information about a Directory after * it has been deleted because of a TTL of 0. */ @Test public void setTtlForDirectoryWithNoTtl() throws Exception { CreateDirectoryContext directoryContext = CreateDirectoryContext .defaults(CreateDirectoryPOptions.newBuilder().setRecursive(true)); mFileSystemMaster.createDirectory(NESTED_URI, directoryContext); mFileSystemMaster.createDirectory(NESTED_DIR_URI, directoryContext); CreateFileContext createFileContext = CreateFileContext.defaults( CreateFilePOptions.newBuilder().setBlockSizeBytes(Constants.KB).setRecursive(true)); long fileId = mFileSystemMaster.createFile(NESTED_FILE_URI, createFileContext); HeartbeatScheduler.execute(HeartbeatContext.MASTER_TTL_CHECK); // Since no TTL is set, the file should not be deleted. assertEquals(fileId, mFileSystemMaster.getFileInfo(NESTED_FILE_URI, GET_STATUS_CONTEXT).getFileId()); // Set ttl. mFileSystemMaster.setAttribute(NESTED_URI, SetAttributeContext.defaults(SetAttributePOptions.newBuilder().setTtl(0))); HeartbeatScheduler.execute(HeartbeatContext.MASTER_TTL_CHECK); // TTL is set to 0, the file and directory should have been deleted during last TTL check. mThrown.expect(FileDoesNotExistException.class); mFileSystemMaster.getFileInfo(NESTED_URI, GET_STATUS_CONTEXT); mFileSystemMaster.getFileInfo(NESTED_DIR_URI, GET_STATUS_CONTEXT); mFileSystemMaster.getFileInfo(NESTED_FILE_URI, GET_STATUS_CONTEXT); }
@Test public void createPathWithWhiteSpaces() throws Exception { String[] paths = new String[]{ "/ ", "/ ", "/ path", "/path ", "/pa th", "/ pa th ", "/pa/ th", "/pa / th", "/ pa / th ", }; for (String path : paths) { AlluxioURI uri = new AlluxioURI(path); long id = mFileSystemMaster.createFile(uri, CreateFileContext.defaults(CreateFilePOptions.newBuilder().setRecursive(true))); Assert.assertEquals(id, mFileSystemMaster.getFileId(uri)); mFileSystemMaster.delete(uri, DeleteContext.defaults()); id = mFileSystemMaster.createDirectory(uri, CreateDirectoryContext .defaults(CreateDirectoryPOptions.newBuilder().setRecursive(true))); Assert.assertEquals(id, mFileSystemMaster.getFileId(uri)); } }
@Override public void apply(FileSystem fs) throws Exception { fs.createDirectory(DIR); fs.createDirectory(NESTED_DIR); fs.createDirectory(NESTED_NESTED_DIR); fs.createDirectory(RECURSIVE, CreateDirectoryPOptions.newBuilder().setRecursive(true).build()); fs.createDirectory(RECURSIVE, CreateDirectoryPOptions.newBuilder().setAllowExists(true).build()); fs.createDirectory(MODE_DIR, CreateDirectoryPOptions.newBuilder().setMode(TEST_MODE.toProto()) .setRecursive(true).build()); // Set TTL via common options instead (should have the same effect). fs.createDirectory(COMMON_TTL_DIR, CreateDirectoryPOptions.newBuilder().setRecursive(true).setCommonOptions( FileSystemMasterCommonPOptions.newBuilder().setTtl(TTL).setTtlAction(TtlAction.DELETE)) .build()); fs.createDirectory(TTL_DIR, CreateDirectoryPOptions.newBuilder().setCommonOptions( FileSystemMasterCommonPOptions.newBuilder().setTtl(TTL).setTtlAction(TtlAction.DELETE)) .setRecursive(true).build()); fs.createDirectory(THROUGH_DIR, CreateDirectoryPOptions.newBuilder() .setWriteType(WritePType.THROUGH).setRecursive(true).build()); fs.createDirectory(ALL_OPTS_DIR, CreateDirectoryPOptions.newBuilder().setRecursive(true) .setMode(TEST_MODE.toProto()).setAllowExists(true).setWriteType(WritePType.THROUGH) .setCommonOptions( FileSystemMasterCommonPOptions.newBuilder().setTtl(TTL).setTtlAction(TtlAction.DELETE)) .build()); }
@Test public void removeExtendedDefaultAclMask() throws Exception { mFileSystemMaster.createDirectory(NESTED_URI, CreateDirectoryContext .defaults(CreateDirectoryPOptions.newBuilder().setRecursive(true))); AclEntry newAcl = AclEntry.fromCliString("default:user:newuser:rwx"); // Add an ACL addAcl(NESTED_URI, newAcl); assertThat(getInfo(NESTED_URI).getDefaultAcl().getEntries(), hasItem(newAcl)); // Attempt to remove the ACL mask AclEntry maskEntry = AclEntry.fromCliString("default:mask::rwx"); assertThat(getInfo(NESTED_URI).getDefaultAcl().getEntries(), hasItem(maskEntry)); try { removeAcl(NESTED_URI, maskEntry); fail("Expected removing the mask from an extended ACL to fail"); } catch (IOException e) { assertThat(e.getMessage(), containsString("mask")); } // Remove the extended ACL removeAcl(NESTED_URI, newAcl); // Now we can add and remove a mask addAcl(NESTED_URI, maskEntry); removeAcl(NESTED_URI, maskEntry); }
@Test public void removeExtendedAclMask() throws Exception { mFileSystemMaster.createDirectory(NESTED_URI, CreateDirectoryContext .defaults(CreateDirectoryPOptions.newBuilder().setRecursive(true))); AclEntry newAcl = AclEntry.fromCliString("user:newuser:rwx"); // Add an ACL addAcl(NESTED_URI, newAcl); assertThat(getInfo(NESTED_URI).getAcl().getEntries(), hasItem(newAcl)); // Attempt to remove the ACL mask AclEntry maskEntry = AclEntry.fromCliString("mask::rwx"); assertThat(getInfo(NESTED_URI).getAcl().getEntries(), hasItem(maskEntry)); try { removeAcl(NESTED_URI, maskEntry); fail("Expected removing the mask from an extended ACL to fail"); } catch (IOException e) { assertThat(e.getMessage(), containsString("mask")); } // Remove the extended ACL removeAcl(NESTED_URI, newAcl); // Now we can add and remove a mask addAcl(NESTED_URI, maskEntry); removeAcl(NESTED_URI, maskEntry); }
/** * Tests that TTL free of a directory is not forgotten across restarts. */ @Test public void ttlDirectoryFreeReplay() throws Exception { CreateDirectoryContext directoryContext = CreateDirectoryContext .defaults(CreateDirectoryPOptions.newBuilder().setRecursive(true)); mFileSystemMaster.createDirectory(NESTED_URI, directoryContext); long blockId = createFileWithSingleBlock(NESTED_FILE_URI); assertEquals(1, mBlockMaster.getBlockInfo(blockId).getLocations().size()); // Set ttl & operation. mFileSystemMaster.setAttribute(NESTED_URI, SetAttributeContext.defaults( SetAttributePOptions.newBuilder().setTtl(0).setTtlAction(alluxio.grpc.TtlAction.FREE))); // Simulate restart. stopServices(); startServices(); Command heartbeat = mBlockMaster.workerHeartbeat(mWorkerId1, null, ImmutableMap.of("MEM", (long) Constants.KB), ImmutableList.of(blockId), ImmutableMap.<String, List<Long>>of(), mMetrics); // Verify the muted Free command on worker1. assertEquals(Command.newBuilder().setCommandType(CommandType.Nothing).build(), heartbeat); assertEquals(0, mBlockMaster.getBlockInfo(blockId).getLocations().size()); }
/** * Tests that an exception is thrown when trying to get information about a Directory after * it has been deleted after the TTL has been set to 0. */ @Test public void setSmallerTtlForDirectoryWithTtl() throws Exception { CreateDirectoryContext directoryContext = CreateDirectoryContext.defaults(CreateDirectoryPOptions.newBuilder() .setCommonOptions(FileSystemMasterCommonPOptions.newBuilder().setTtl(Constants.HOUR_MS)) .setRecursive(true)); mFileSystemMaster.createDirectory(NESTED_URI, directoryContext); HeartbeatScheduler.execute(HeartbeatContext.MASTER_TTL_CHECK); assertTrue( mFileSystemMaster.getFileInfo(NESTED_URI, GET_STATUS_CONTEXT).getName() != null); mFileSystemMaster.setAttribute(NESTED_URI, SetAttributeContext.defaults(SetAttributePOptions.newBuilder().setTtl(0))); HeartbeatScheduler.execute(HeartbeatContext.MASTER_TTL_CHECK); // TTL is reset to 0, the file should have been deleted during last TTL check. mThrown.expect(FileDoesNotExistException.class); mFileSystemMaster.getFileInfo(NESTED_URI, GET_STATUS_CONTEXT); }
/** * Sets up dependencies before a single test runs. */ @BeforeClass public static void beforeClass() throws Exception { sFileContext = CreateFileContext .defaults(CreateFilePOptions.newBuilder().setBlockSizeBytes(Constants.KB) .setMode(TEST_FILE_MODE.toProto())) .setOwner(TEST_OWNER).setGroup(TEST_GROUP); sDirectoryContext = CreateDirectoryContext .defaults(CreateDirectoryPOptions.newBuilder().setMode(TEST_DIR_MODE.toProto())) .setOwner(TEST_OWNER).setGroup(TEST_GROUP); sNestedFileContext = CreateFileContext .defaults(CreateFilePOptions.newBuilder().setBlockSizeBytes(Constants.KB) .setMode(TEST_FILE_MODE.toProto()).setRecursive(true)) .setOwner(TEST_OWNER).setGroup(TEST_GROUP); sNestedDirectoryContext = CreateDirectoryContext.defaults( CreateDirectoryPOptions.newBuilder().setMode(TEST_DIR_MODE.toProto()).setRecursive(true)) .setOwner(TEST_OWNER).setGroup(TEST_GROUP); }
/** * Tests that file information is still present after it has been freed after the parent * directory's TTL has been set to 0. */ @Test public void ttlDirectoryFree() throws Exception { CreateDirectoryContext directoryContext = CreateDirectoryContext .defaults(CreateDirectoryPOptions.newBuilder().setRecursive(true)); mFileSystemMaster.createDirectory(NESTED_URI, directoryContext); long blockId = createFileWithSingleBlock(NESTED_FILE_URI); assertEquals(1, mBlockMaster.getBlockInfo(blockId).getLocations().size()); // Set ttl & operation. mFileSystemMaster.setAttribute(NESTED_URI, SetAttributeContext.defaults( SetAttributePOptions.newBuilder().setTtl(0).setTtlAction(alluxio.grpc.TtlAction.FREE))); Command heartbeat = mBlockMaster.workerHeartbeat(mWorkerId1, null, ImmutableMap.of("MEM", (long) Constants.KB), ImmutableList.of(blockId), ImmutableMap.<String, List<Long>>of(), mMetrics); // Verify the muted Free command on worker1. assertEquals(Command.newBuilder().setCommandType(CommandType.Nothing).build(), heartbeat); assertEquals(0, mBlockMaster.getBlockInfo(blockId).getLocations().size()); }
/** * Tests that the original TTL is removed after setting it to {@link Constants#NO_TTL} for * a directory. */ @Test public void setNoTtlForDirectoryWithTtl() throws Exception { mFileSystemMaster.createDirectory(new AlluxioURI("/nested"), CreateDirectoryContext.defaults(CreateDirectoryPOptions.newBuilder().setRecursive(true))); mFileSystemMaster.createDirectory(NESTED_URI, CreateDirectoryContext.defaults(CreateDirectoryPOptions.newBuilder() .setCommonOptions(FileSystemMasterCommonPOptions.newBuilder().setTtl(0)) .setRecursive(true))); // After setting TTL to NO_TTL, the original TTL will be removed, and the file will not be // deleted during next TTL check. mFileSystemMaster.setAttribute(NESTED_URI, SetAttributeContext .defaults(SetAttributePOptions.newBuilder().setTtl(Constants.NO_TTL))); HeartbeatScheduler.execute(HeartbeatContext.MASTER_TTL_CHECK); assertEquals(NESTED_URI.getName(), mFileSystemMaster.getFileInfo(NESTED_URI, GET_STATUS_CONTEXT).getName()); }
/** * Initializes the root of the inode tree. * * @param owner the root owner * @param group the root group * @param mode the root mode * @param context the journal context to journal the initialization to */ public void initializeRoot(String owner, String group, Mode mode, JournalContext context) throws UnavailableException { if (mState.getRoot() == null) { MutableInodeDirectory root = MutableInodeDirectory.create( mDirectoryIdGenerator.getNewDirectoryId(context), NO_PARENT, ROOT_INODE_NAME, CreateDirectoryContext .defaults(CreateDirectoryPOptions.newBuilder().setMode(mode.toProto())) .setOwner(owner).setGroup(group)); root.setPersistenceState(PersistenceState.PERSISTED); mState.applyAndJournal(context, root); } }