@Override public OutputStream create(String path) throws IOException { return create(path, CreateOptions.defaults(mUfsConf).setCreateParent(true)); }
/** * Completes the current log and rotates in a new log. */ private void rotateLog() throws IOException { mDataOutputStream.close(); mJournalWriter.completeCurrentLog(); mRawOutputStream = mUfs.create(mCurrentLog.toString(), CreateOptions.defaults(ServerConfiguration.global()).setEnsureAtomic(false) .setCreateParent(true)); LOG.info("Opened current log file: {}", mCurrentLog); mDataOutputStream = new DataOutputStream(mRawOutputStream); } }
/** * @param ufs the under storage holding the journal * @param log the location to write the log to * @param journalFormatter the journal formatter to use when writing journal entries * @param journalWriter the journal writer to use to get journal entry sequence numbers and * complete the log when it needs to be rotated */ public EntryOutputStream(UnderFileSystem ufs, URI log, JournalFormatter journalFormatter, UfsJournalWriter journalWriter) throws IOException { mUfs = ufs; mCurrentLog = log; mJournalFormatter = journalFormatter; mJournalWriter = journalWriter; mMaxLogSize = ServerConfiguration.getBytes(PropertyKey.MASTER_JOURNAL_LOG_SIZE_BYTES_MAX); mRawOutputStream = mUfs.create(mCurrentLog.toString(), CreateOptions.defaults(ServerConfiguration.global()) .setEnsureAtomic(false).setCreateParent(true)); LOG.info("Opened current log file: {}", mCurrentLog); mDataOutputStream = new DataOutputStream(mRawOutputStream); }
private void createNewLogFile() throws IOException { URI newLog = UfsJournalFile .encodeLogFileLocation(mJournal, mNextSequenceNumber, UfsJournal.UNKNOWN_SEQUENCE_NUMBER); UfsJournalFile currentLog = UfsJournalFile.createLogFile(newLog, mNextSequenceNumber, UfsJournal.UNKNOWN_SEQUENCE_NUMBER); OutputStream outputStream = mUfs.create(currentLog.getLocation().toString(), CreateOptions.defaults(ServerConfiguration.global()).setEnsureAtomic(false) .setCreateParent(true)); mJournalOutputStream = new JournalOutputStream(currentLog, outputStream); LOG.info("Created current log file: {}", currentLog); }
String dstPath = prepareUfsFilePath(fileInfo, ufs); OutputStream outputStream = ufs.create(dstPath, CreateOptions.defaults(ServerConfiguration.global()) .setOwner(fileInfo.getOwner()).setGroup(fileInfo.getGroup()) .setMode(new Mode((short) fileInfo.getMode())));
@Test public void createParent() throws IOException { String testFile = PathUtils.concatPath(mUnderfsAddress, "createParent/testFile"); OutputStream o = mUfs.create(testFile, CreateOptions.defaults(mConfiguration) .setCreateParent(true)); o.close(); assertTrue(mUfs.exists(testFile)); }
@Test public void createAtomic() throws IOException { String testFile = PathUtils.concatPath(mUnderfsAddress, "createAtomic"); OutputStream stream = mUfs.create(testFile, CreateOptions.defaults(mConfiguration) .setEnsureAtomic(true)); stream.write(TEST_BYTES); assertFalse(mUfs.isFile(testFile)); stream.close(); assertTrue(mUfs.isFile(testFile)); }
@Test public void createNoParent() throws IOException { // Run the test only for local UFS. Other UFSs succeed if no parents are present Assume.assumeTrue(UnderFileSystemUtils.isLocal(mUfs)); mThrown.expect(IOException.class); String testFile = PathUtils.concatPath(mUnderfsAddress, "createNoParent/testFile"); OutputStream o = mUfs.create(testFile, CreateOptions.defaults(mConfiguration) .setCreateParent(false)); o.close(); }
OutputStream out = closer.register( ufs.create(dstPath.toString(), CreateOptions.defaults(ServerConfiguration.global()).setOwner(uriStatus.getOwner()) .setGroup(uriStatus.getGroup()).setMode(new Mode((short) uriStatus.getMode())))); bytesWritten = IOUtils.copyLarge(in, out);
/** * Tests getting and setting fields. */ @Test public void fields() { Random random = new Random(); boolean createParent = random.nextBoolean(); boolean ensureAtomic = random.nextBoolean(); String owner = CommonUtils.randomAlphaNumString(10); String group = CommonUtils.randomAlphaNumString(10); Mode mode = new Mode((short) random.nextInt()); CreateOptions options = CreateOptions.defaults(mConfiguration); options.setCreateParent(createParent); options.setEnsureAtomic(ensureAtomic); options.setOwner(owner); options.setGroup(group); options.setMode(mode); assertEquals(createParent, options.getCreateParent()); assertEquals(ensureAtomic, options.isEnsureAtomic()); assertEquals(owner, options.getOwner()); assertEquals(group, options.getGroup()); assertEquals(mode, options.getMode()); }
private void createUfsFile(UfsFileWriteRequestContext context) throws IOException { UfsFileWriteRequest request = context.getRequest(); Preconditions.checkState(request != null); Protocol.CreateUfsFileOptions createUfsFileOptions = request.getCreateUfsFileOptions(); UfsManager.UfsClient ufsClient = mUfsManager.get(createUfsFileOptions.getMountId()); CloseableResource<UnderFileSystem> ufsResource = ufsClient.acquireUfsResource(); context.setUfsResource(ufsResource); UnderFileSystem ufs = ufsResource.get(); CreateOptions createOptions = CreateOptions.defaults(ServerConfiguration.global()) .setOwner(createUfsFileOptions.getOwner()).setGroup(createUfsFileOptions.getGroup()) .setMode(new Mode((short) createUfsFileOptions.getMode())); if (createUfsFileOptions.hasAcl()) { // This acl information will be ignored by all but HDFS implementations createOptions.setAcl(ProtoUtils.fromProto(createUfsFileOptions.getAcl())); } context.setOutputStream(ufs.create(request.getUfsPath(), createOptions)); context.setCreateOptions(createOptions); String ufsString = MetricsSystem.escape(ufsClient.getUfsMountPointUri()); String counterName = Metric.getMetricNameWithTags(WorkerMetrics.BYTES_WRITTEN_UFS, WorkerMetrics.TAG_UFS, ufsString); Counter counter = MetricsSystem.counter(counterName); context.setCounter(counter); String meterName = Metric.getMetricNameWithTags(WorkerMetrics.BYTES_WRITTEN_UFS_THROUGHPUT, WorkerMetrics.TAG_UFS, ufsString); context.setMeter(MetricsSystem.meter(meterName)); } }
/** * Tests for default {@link CreateOptions}. */ @Test public void defaults() throws IOException { CreateOptions options = CreateOptions.defaults(mConfiguration); assertFalse(options.getCreateParent()); assertFalse(options.isEnsureAtomic()); assertNull(options.getOwner()); assertNull(options.getGroup()); String umask = mConfiguration.get(PropertyKey.SECURITY_AUTHORIZATION_PERMISSION_UMASK); assertEquals(ModeUtils.applyFileUMask(Mode.defaults(), umask), options.getMode()); }
/** * Tests for building an {@link CreateOptions} with a security enabled * configuration. */ @Test public void securityEnabled() throws IOException { mConfiguration.set(PropertyKey.SECURITY_AUTHENTICATION_TYPE, AuthType.SIMPLE.getAuthName()); mConfiguration.set(PropertyKey.SECURITY_LOGIN_USERNAME, "foo"); // Use IdentityUserGroupMapping to map user "foo" to group "foo". mConfiguration.set(PropertyKey.SECURITY_GROUP_MAPPING_CLASS, IdentityUserGroupsMapping.class.getName()); CreateOptions options = CreateOptions.defaults(mConfiguration); assertFalse(options.getCreateParent()); assertFalse(options.isEnsureAtomic()); assertNull(options.getOwner()); assertNull(options.getGroup()); String umask = mConfiguration.get(PropertyKey.SECURITY_AUTHORIZATION_PERMISSION_UMASK); assertEquals(ModeUtils.applyFileUMask(Mode.defaults(), umask), options.getMode()); }
/** * Creates a UFS block and initialize it with bytes read from block store. * * @param context context of this request */ private void createUfsBlock(BlockWriteRequestContext context) throws Exception { BlockWriteRequest request = context.getRequest(); Protocol.CreateUfsBlockOptions createUfsBlockOptions = request.getCreateUfsBlockOptions(); UfsManager.UfsClient ufsClient = mUfsManager.get(createUfsBlockOptions.getMountId()); alluxio.resource.CloseableResource<UnderFileSystem> ufsResource = ufsClient.acquireUfsResource(); context.setUfsResource(ufsResource); String ufsString = MetricsSystem.escape(ufsClient.getUfsMountPointUri()); String ufsPath = BlockUtils.getUfsBlockPath(ufsClient, request.getId()); UnderFileSystem ufs = ufsResource.get(); // Set the atomic flag to be true to ensure only the creation of this file is atomic on close. OutputStream ufsOutputStream = ufs.create(ufsPath, CreateOptions.defaults(ServerConfiguration.global()).setEnsureAtomic(true) .setCreateParent(true)); context.setOutputStream(ufsOutputStream); context.setUfsPath(ufsPath); String counterName = Metric.getMetricNameWithTags(WorkerMetrics.BYTES_WRITTEN_UFS, WorkerMetrics.TAG_UFS, ufsString); String meterName = Metric.getMetricNameWithTags(WorkerMetrics.BYTES_WRITTEN_UFS_THROUGHPUT, WorkerMetrics.TAG_UFS, ufsString); context.setCounter(MetricsSystem.counter(counterName)); context.setMeter(MetricsSystem.meter(meterName)); }
/** * Completes the current log and rotates in a new log. */ private void rotateLog() throws IOException { mDataOutputStream.close(); mJournalWriter.completeCurrentLog(); mRawOutputStream = mUfs.create(mCurrentLog.toString(), CreateOptions.defaults().setEnsureAtomic(false).setCreateParent(true)); LOG.info("Opened current log file: {}", mCurrentLog); mDataOutputStream = new DataOutputStream(mRawOutputStream); } }
@Override public OutputStream create(String path) throws IOException { return create(path, CreateOptions.defaults().setCreateParent(true)); }
/** * @param ufs the under storage holding the journal * @param log the location to write the log to * @param journalFormatter the journal formatter to use when writing journal entries * @param journalWriter the journal writer to use to get journal entry sequence numbers and * complete the log when it needs to be rotated */ public EntryOutputStream(UnderFileSystem ufs, URI log, JournalFormatter journalFormatter, UfsJournalWriter journalWriter) throws IOException { mUfs = ufs; mCurrentLog = log; mJournalFormatter = journalFormatter; mJournalWriter = journalWriter; mMaxLogSize = Configuration.getBytes(PropertyKey.MASTER_JOURNAL_LOG_SIZE_BYTES_MAX); mRawOutputStream = mUfs.create(mCurrentLog.toString(), CreateOptions.defaults().setEnsureAtomic(false).setCreateParent(true)); LOG.info("Opened current log file: {}", mCurrentLog); mDataOutputStream = new DataOutputStream(mRawOutputStream); }
private void createNewLogFile() throws IOException { URI newLog = UfsJournalFile .encodeLogFileLocation(mJournal, mNextSequenceNumber, UfsJournal.UNKNOWN_SEQUENCE_NUMBER); UfsJournalFile currentLog = UfsJournalFile.createLogFile(newLog, mNextSequenceNumber, UfsJournal.UNKNOWN_SEQUENCE_NUMBER); OutputStream outputStream = mUfs.create(currentLog.getLocation().toString(), CreateOptions.defaults().setEnsureAtomic(false).setCreateParent(true)); mJournalOutputStream = new JournalOutputStream(currentLog, outputStream); LOG.info("Created current log file: {}", currentLog); }