.setWriteType(WritePType.CACHE_THROUGH).setRecursive(true).build(); FileOutStream os = fileSystem.createFile(outputPath, options); System.out.println("done");
CreateFilePOptions options = CreateFilePOptions.newBuilder().setBlockSizeBytes(blockSize) .setMode(new Mode(permission.toShort()).toProto()).setRecursive(true).build();
@Test public void heartbeatFileUnderReplicatedAndLost() throws Exception { mFileContext.getOptions().setReplicationMin(2); long blockId = createBlockHelper(TEST_FILE_1, mFileContext); // Create a worker. long workerId = mBlockMaster.getWorkerId(new WorkerNetAddress().setHost("localhost") .setRpcPort(80).setDataPort(81).setWebPort(82)); mBlockMaster.workerRegister(workerId, Collections.singletonList("MEM"), ImmutableMap.of("MEM", 100L), ImmutableMap.of("MEM", 0L), NO_BLOCKS_ON_TIERS, RegisterWorkerPOptions.getDefaultInstance()); mBlockMaster.commitBlock(workerId, 50L, "MEM", blockId, 20L); // Indicate that blockId is removed on the worker. mBlockMaster.workerHeartbeat(workerId, null, ImmutableMap.of("MEM", 0L), ImmutableList.of(blockId), NO_BLOCKS_ON_TIERS, NO_METRICS); mReplicationChecker.heartbeat(); Assert.assertEquals(EMPTY, mMockReplicationHandler.getEvictRequests()); Assert.assertEquals(EMPTY, mMockReplicationHandler.getReplicateRequests()); }
createFileContext.getOptions().setBlockSizeBytes(ufsBlockSizeByte); createFileContext.getOptions().setRecursive(context.getOptions().getCreateAncestors()); createFileContext.getOptions() .setCommonOptions(FileSystemMasterCommonPOptions.newBuilder() .setTtl(context.getOptions().getCommonOptions().getTtl()) .setTtlAction(context.getOptions().getCommonOptions().getTtlAction())); mode.setOtherBits(mode.getOtherBits().or(mode.getOwnerBits())); createFileContext.getOptions().setMode(mode.toProto()); if (acl != null) { createFileContext.setAcl(acl.getEntries());
CreateFilePOptions.newBuilder().setWriteType(WritePType.NONE).build();
@BeforeClass public static void beforeClass() throws Exception { sFileContext = CreateFileContext .defaults(CreateFilePOptions.newBuilder().setBlockSizeBytes(Constants.KB) .setMode(TEST_NORMAL_MODE.toProto())) .setOwner(TEST_USER_2.getUser()).setGroup(TEST_USER_2.getGroup()); sWeirdFileContext = CreateFileContext .defaults(CreateFilePOptions.newBuilder().setBlockSizeBytes(Constants.KB) .setMode(TEST_WEIRD_MODE.toProto())) .setOwner(TEST_USER_1.getUser()).setGroup(TEST_USER_1.getGroup()); sNestedFileContext = CreateFileContext .defaults(CreateFilePOptions.newBuilder().setBlockSizeBytes(Constants.KB) .setMode(TEST_NORMAL_MODE.toProto()).setRecursive(true)) .setOwner(TEST_USER_1.getUser()).setGroup(TEST_USER_1.getGroup());
CreateFilePOptions.newBuilder().setBlockSizeBytes(Constants.KB).setRecursive(true)); created = createPath(mTree, NESTED_FILE_URI, options);
try (Closer closer = Closer.create()) { CreateFilePOptions createOptions = CreateFilePOptions.newBuilder() .setFileWriteLocationPolicy(mFsContext.getConf().get( PropertyKey.USER_FILE_COPY_FROM_LOCAL_WRITE_LOCATION_POLICY)) .build(); os = closer.register(mFileSystem.createFile(dstPath, createOptions)); FileInputStream in = closer.register(new FileInputStream(src));
mFileSystemMaster.createFile(new AlluxioURI("/testDir/file"), CreateFileContext .defaults(CreateFilePOptions.newBuilder().setBlockSizeBytes(Constants.KB) .setMode(TEST_FILE_MODE.toProto())) .setOwner(TEST_USER_1.getUser()).setGroup(TEST_USER_1.getGroup())); mFileSystemMaster.createFile(new AlluxioURI("/testFile"), CreateFileContext .defaults(CreateFilePOptions.newBuilder().setBlockSizeBytes(Constants.KB) .setMode(TEST_FILE_MODE.toProto())) .setOwner(TEST_USER_2.getUser()).setGroup(TEST_USER_2.getGroup()));
public Builder mergeFrom(alluxio.grpc.CreateFilePOptions other) { if (other == alluxio.grpc.CreateFilePOptions.getDefaultInstance()) return this; if (other.hasBlockSizeBytes()) { setBlockSizeBytes(other.getBlockSizeBytes()); setRecursive(other.getRecursive()); mergeMode(other.getMode()); setReplicationMax(other.getReplicationMax()); setReplicationMin(other.getReplicationMin()); setReplicationDurable(other.getReplicationDurable()); onChanged(); setWriteTier(other.getWriteTier()); setWriteType(other.getWriteType()); mergeCommonOptions(other.getCommonOptions()); this.mergeUnknownFields(other.unknownFields); onChanged(); return this;
/** * @param command the move command to execute * @param writeType the write type to use for the moved file * @param fileSystem the Alluxio file system */ private static void move(MoveCommand command, WritePType writeType, FileSystem fileSystem) throws Exception { String source = command.getSource(); String destination = command.getDestination(); LOG.debug("Moving {} to {}", source, destination); CreateFilePOptions createOptions = CreateFilePOptions.newBuilder().setWriteType(writeType).build(); try (FileOutStream out = fileSystem.createFile(new AlluxioURI(destination), createOptions)) { try (FileInStream in = fileSystem.openFile(new AlluxioURI(source))) { IOUtils.copy(in, out); } catch (Throwable t) { try { out.cancel(); } catch (Throwable t2) { t.addSuppressed(t2); } throw t; } } fileSystem.delete(new AlluxioURI(source)); }
@Test public void createPathWithWhiteSpaces() throws Exception { String[] paths = new String[]{ "/ ", "/ ", "/ path", "/path ", "/pa th", "/ pa th ", "/pa/ th", "/pa / th", "/ pa / th ", }; for (String path : paths) { AlluxioURI uri = new AlluxioURI(path); long id = mFileSystemMaster.createFile(uri, CreateFileContext.defaults(CreateFilePOptions.newBuilder().setRecursive(true))); Assert.assertEquals(id, mFileSystemMaster.getFileId(uri)); mFileSystemMaster.delete(uri, DeleteContext.defaults()); id = mFileSystemMaster.createDirectory(uri, CreateDirectoryContext .defaults(CreateDirectoryPOptions.newBuilder().setRecursive(true))); Assert.assertEquals(id, mFileSystemMaster.getFileId(uri)); } }
/** * Tests that an exception is thrown when trying to get information about a Directory after * it has been deleted because of a TTL of 0. */ @Test public void setTtlForDirectoryWithNoTtl() throws Exception { CreateDirectoryContext directoryContext = CreateDirectoryContext .defaults(CreateDirectoryPOptions.newBuilder().setRecursive(true)); mFileSystemMaster.createDirectory(NESTED_URI, directoryContext); mFileSystemMaster.createDirectory(NESTED_DIR_URI, directoryContext); CreateFileContext createFileContext = CreateFileContext.defaults( CreateFilePOptions.newBuilder().setBlockSizeBytes(Constants.KB).setRecursive(true)); long fileId = mFileSystemMaster.createFile(NESTED_FILE_URI, createFileContext); HeartbeatScheduler.execute(HeartbeatContext.MASTER_TTL_CHECK); // Since no TTL is set, the file should not be deleted. assertEquals(fileId, mFileSystemMaster.getFileInfo(NESTED_FILE_URI, GET_STATUS_CONTEXT).getFileId()); // Set ttl. mFileSystemMaster.setAttribute(NESTED_URI, SetAttributeContext.defaults(SetAttributePOptions.newBuilder().setTtl(0))); HeartbeatScheduler.execute(HeartbeatContext.MASTER_TTL_CHECK); // TTL is set to 0, the file and directory should have been deleted during last TTL check. mThrown.expect(FileDoesNotExistException.class); mFileSystemMaster.getFileInfo(NESTED_URI, GET_STATUS_CONTEXT); mFileSystemMaster.getFileInfo(NESTED_DIR_URI, GET_STATUS_CONTEXT); mFileSystemMaster.getFileInfo(NESTED_FILE_URI, GET_STATUS_CONTEXT); }
/** * Writes a partition. */ public void writePartition() throws IOException, AlluxioException { if (sDebugMode) { mBuf.flip(); LOG.info(FormatUtils.byteBufferToString(mBuf)); } mBuf.flip(); for (int pId = mLeft; pId < mRight; pId++) { final long startTimeMs = System.currentTimeMillis(); FileOutStream os = mFileSystem.createFile(new AlluxioURI(sFileName + (pId + sBaseFileNumber)), CreateFilePOptions.newBuilder().setRecursive(true).build()); for (int k = 0; k < sBlocksPerFile; k++) { mBuf.putInt(0, k + mWorkerId); os.write(mBuf.array()); } os.close(); logPerIteration(startTimeMs, pId, "th WriteAlluxioFile @ Worker ", pId); } }
@Test public void heartbeatFileWithinRange() throws Exception { mFileContext.getOptions().setReplicationMin(1).setReplicationMax(3); long blockId = createBlockHelper(TEST_FILE_1, mFileContext); // One replica, meeting replication min addBlockLocationHelper(blockId, 1); mReplicationChecker.heartbeat(); Assert.assertEquals(EMPTY, mMockReplicationHandler.getEvictRequests()); Assert.assertEquals(EMPTY, mMockReplicationHandler.getReplicateRequests()); // Two replicas, good heartbeatToAddLocationHelper(blockId, createWorkerHelper(1)); mReplicationChecker.heartbeat(); Assert.assertEquals(EMPTY, mMockReplicationHandler.getEvictRequests()); Assert.assertEquals(EMPTY, mMockReplicationHandler.getReplicateRequests()); // Three replicas, meeting replication max, still good heartbeatToAddLocationHelper(blockId, createWorkerHelper(2)); mReplicationChecker.heartbeat(); Assert.assertEquals(EMPTY, mMockReplicationHandler.getEvictRequests()); Assert.assertEquals(EMPTY, mMockReplicationHandler.getReplicateRequests()); }
/** * Sets up dependencies before a single test runs. */ @BeforeClass public static void beforeClass() throws Exception { sFileContext = CreateFileContext .defaults(CreateFilePOptions.newBuilder().setBlockSizeBytes(Constants.KB) .setMode(TEST_FILE_MODE.toProto())) .setOwner(TEST_OWNER).setGroup(TEST_GROUP); sDirectoryContext = CreateDirectoryContext .defaults(CreateDirectoryPOptions.newBuilder().setMode(TEST_DIR_MODE.toProto())) .setOwner(TEST_OWNER).setGroup(TEST_GROUP); sNestedFileContext = CreateFileContext .defaults(CreateFilePOptions.newBuilder().setBlockSizeBytes(Constants.KB) .setMode(TEST_FILE_MODE.toProto()).setRecursive(true)) .setOwner(TEST_OWNER).setGroup(TEST_GROUP); sNestedDirectoryContext = CreateDirectoryContext.defaults( CreateDirectoryPOptions.newBuilder().setMode(TEST_DIR_MODE.toProto()).setRecursive(true)) .setOwner(TEST_OWNER).setGroup(TEST_GROUP); }
/** * Tests that an exception is thrown when trying to get information about a file after it * has been deleted after the TTL has been set to 0. */ @Test public void setSmallerTtlForFileWithTtl() throws Exception { CreateFileContext context = CreateFileContext.defaults(CreateFilePOptions.newBuilder() .setCommonOptions(FileSystemMasterCommonPOptions.newBuilder().setTtl(Constants.HOUR_MS)) .setBlockSizeBytes(Constants.KB).setRecursive(true)); long fileId = mFileSystemMaster.createFile(NESTED_FILE_URI, context); HeartbeatScheduler.execute(HeartbeatContext.MASTER_TTL_CHECK); // Since TTL is 1 hour, the file won't be deleted during last TTL check. assertEquals(fileId, mFileSystemMaster.getFileInfo(NESTED_FILE_URI, GET_STATUS_CONTEXT).getFileId()); mFileSystemMaster.setAttribute(NESTED_FILE_URI, SetAttributeContext.defaults(SetAttributePOptions.newBuilder().setTtl(0))); HeartbeatScheduler.execute(HeartbeatContext.MASTER_TTL_CHECK); // TTL is reset to 0, the file should have been deleted during last TTL check. mThrown.expect(FileDoesNotExistException.class); mFileSystemMaster.getFileInfo(fileId); }
/** * Tests that a file has not been deleted after the TTL has been reset to a valid value. */ @Test public void setLargerTtlForFileWithTtl() throws Exception { mFileSystemMaster.createDirectory(NESTED_URI, CreateDirectoryContext .defaults(CreateDirectoryPOptions.newBuilder().setRecursive(true))); CreateFileContext context = CreateFileContext.defaults(CreateFilePOptions.newBuilder() .setCommonOptions(FileSystemMasterCommonPOptions.newBuilder().setTtl(0)) .setBlockSizeBytes(Constants.KB).setRecursive(true)); long fileId = mFileSystemMaster.createFile(NESTED_FILE_URI, context); assertEquals(fileId, mFileSystemMaster.getFileInfo(NESTED_FILE_URI, GET_STATUS_CONTEXT).getFileId()); mFileSystemMaster.setAttribute(NESTED_FILE_URI, SetAttributeContext .defaults(SetAttributePOptions.newBuilder().setTtl(Constants.HOUR_MS))); HeartbeatScheduler.execute(HeartbeatContext.MASTER_TTL_CHECK); // TTL is reset to 1 hour, the file should not be deleted during last TTL check. assertEquals(fileId, mFileSystemMaster.getFileInfo(fileId).getFileId()); }
/** * Tests that TTL delete of a file is not forgotten across restarts. */ @Test public void ttlFileDeleteReplay() throws Exception { CreateFileContext context = CreateFileContext.defaults(); context.getOptions().setBlockSizeBytes(Constants.KB); context.getOptions().setRecursive(true); context.getOptions().setCommonOptions(FileSystemMasterCommonPOptions.newBuilder().setTtl(0)); long fileId = mFileSystemMaster.createFile(NESTED_FILE_URI, context); // Simulate restart. stopServices(); startServices(); FileInfo fileInfo = mFileSystemMaster.getFileInfo(fileId); assertEquals(fileInfo.getFileId(), fileId); HeartbeatScheduler.execute(HeartbeatContext.MASTER_TTL_CHECK); mThrown.expect(FileDoesNotExistException.class); mFileSystemMaster.getFileInfo(fileId); }
/** * Tests that an exception is thrown when trying to get information about a file after it * has been deleted because of a TTL of 0. */ @Test public void setTtlForFileWithNoTtl() throws Exception { CreateFileContext context = CreateFileContext.defaults( CreateFilePOptions.newBuilder().setBlockSizeBytes(Constants.KB).setRecursive(true)); long fileId = mFileSystemMaster.createFile(NESTED_FILE_URI, context); HeartbeatScheduler.execute(HeartbeatContext.MASTER_TTL_CHECK); // Since no TTL is set, the file should not be deleted. assertEquals(fileId, mFileSystemMaster.getFileInfo(NESTED_FILE_URI, GET_STATUS_CONTEXT).getFileId()); mFileSystemMaster.setAttribute(NESTED_FILE_URI, SetAttributeContext.defaults(SetAttributePOptions.newBuilder().setTtl(0))); HeartbeatScheduler.execute(HeartbeatContext.MASTER_TTL_CHECK); // TTL is set to 0, the file should have been deleted during last TTL check. mThrown.expect(FileDoesNotExistException.class); mFileSystemMaster.getFileInfo(fileId); }