/** * Tests getting and setting fields. */ @Test public void fields() { Random random = new Random(); boolean createParent = random.nextBoolean(); boolean ensureAtomic = random.nextBoolean(); String owner = CommonUtils.randomAlphaNumString(10); String group = CommonUtils.randomAlphaNumString(10); Mode mode = new Mode((short) random.nextInt()); CreateOptions options = CreateOptions.defaults(mConfiguration); options.setCreateParent(createParent); options.setEnsureAtomic(ensureAtomic); options.setOwner(owner); options.setGroup(group); options.setMode(mode); assertEquals(createParent, options.getCreateParent()); assertEquals(ensureAtomic, options.isEnsureAtomic()); assertEquals(owner, options.getOwner()); assertEquals(group, options.getGroup()); assertEquals(mode, options.getMode()); }
@Override public OutputStream create(String path) throws IOException { return create(path, CreateOptions.defaults(mUfsConf).setCreateParent(true)); }
/** * Completes the current log and rotates in a new log. */ private void rotateLog() throws IOException { mDataOutputStream.close(); mJournalWriter.completeCurrentLog(); mRawOutputStream = mUfs.create(mCurrentLog.toString(), CreateOptions.defaults(ServerConfiguration.global()).setEnsureAtomic(false) .setCreateParent(true)); LOG.info("Opened current log file: {}", mCurrentLog); mDataOutputStream = new DataOutputStream(mRawOutputStream); } }
@Override public void close() throws IOException { if (mClosed) { return; } mTemporaryOutputStream.close(); if (!mUfs.renameFile(mTemporaryPath, mPermanentPath)) { if (!mUfs.deleteFile(mTemporaryPath)) { LOG.error("Failed to delete temporary file {}", mTemporaryPath); } throw new IOException( ExceptionMessage.FAILED_UFS_RENAME.getMessage(mTemporaryPath, mPermanentPath)); } // Preserve owner and group in case delegation was used to create the path if (mOptions.getOwner() != null || mOptions.getGroup() != null) { try { mUfs.setOwner(mPermanentPath, mOptions.getOwner(), mOptions.getGroup()); } catch (Exception e) { LOG.warn("Failed to update the ufs ownership, default values will be used. " + e); } } // TODO(chaomin): consider setMode of the ufs file. mClosed = true; } }
/** * Tests for default {@link CreateOptions}. */ @Test public void defaults() throws IOException { CreateOptions options = CreateOptions.defaults(mConfiguration); assertFalse(options.getCreateParent()); assertFalse(options.isEnsureAtomic()); assertNull(options.getOwner()); assertNull(options.getGroup()); String umask = mConfiguration.get(PropertyKey.SECURITY_AUTHORIZATION_PERMISSION_UMASK); assertEquals(ModeUtils.applyFileUMask(Mode.defaults(), umask), options.getMode()); }
String dstPath = prepareUfsFilePath(fileInfo, ufs); OutputStream outputStream = ufs.create(dstPath, CreateOptions.defaults(ServerConfiguration.global()) .setOwner(fileInfo.getOwner()).setGroup(fileInfo.getGroup()) .setMode(new Mode((short) fileInfo.getMode()))); final WritableByteChannel outputChannel = Channels.newChannel(outputStream); List<Throwable> errors = new ArrayList<>();
@Override public OutputStream createDirect(String path, CreateOptions options) throws IOException { path = stripPath(path); if (options.getCreateParent()) { File parent = new File(path).getParentFile(); if (parent != null && !parent.mkdirs() && !parent.isDirectory()) { throw new IOException(ExceptionMessage.PARENT_CREATION_FAILED.getMessage(path)); } } OutputStream stream = new FileOutputStream(path); try { setMode(path, options.getMode().toShort()); } catch (IOException e) { stream.close(); throw e; } return stream; }
@Test public void createAtomic() throws IOException { String testFile = PathUtils.concatPath(mUnderfsAddress, "createAtomic"); OutputStream stream = mUfs.create(testFile, CreateOptions.defaults(mConfiguration) .setEnsureAtomic(true)); stream.write(TEST_BYTES); assertFalse(mUfs.isFile(testFile)); stream.close(); assertTrue(mUfs.isFile(testFile)); }
@Override public OutputStream create(String path, CreateOptions options) throws IOException { if (!options.isEnsureAtomic()) { return createDirect(path, options); } return new AtomicFileOutputStream(path, this, options); }
@Override public OutputStream createDirect(String path, CreateOptions options) throws IOException { IOException te = null; FileSystem hdfs = getFs(); RetryPolicy retryPolicy = new CountingRetry(MAX_TRY); while (retryPolicy.attempt()) { try { // TODO(chaomin): support creating HDFS files with specified block size and replication. OutputStream outputStream = new HdfsUnderFileOutputStream( FileSystem.create(hdfs, new Path(path), new FsPermission(options.getMode().toShort()))); if (options.getAcl() != null) { setAclEntries(path, options.getAcl().getEntries()); } return outputStream; } catch (IOException e) { LOG.warn("Attempt count {} : {} ", retryPolicy.getAttemptCount(), e.getMessage()); te = e; } } throw te; }
@Override public OutputStream create(String path, CreateOptions options) throws IOException { if (options.getCreateParent() && !mkdirs(getParentPath(path))) { throw new IOException(ExceptionMessage.PARENT_CREATION_FAILED.getMessage(path)); } return createObject(stripPrefixIfPresent(path)); }
@Override public OutputStream createDirect(String path, CreateOptions options) throws IOException { IOException te = null; RetryPolicy retryPolicy = new CountingRetry(MAX_TRY); while (retryPolicy.attemptRetry()) { try { // TODO(chaomin): support creating HDFS files with specified block size and replication. return FileSystem.create(mFileSystem, new Path(path), new FsPermission(options.getMode().toShort())); } catch (IOException e) { LOG.warn("Retry count {} : {} ", retryPolicy.getRetryCount(), e.getMessage()); te = e; } } throw te; }
/** * @param ufs the under storage holding the journal * @param log the location to write the log to * @param journalFormatter the journal formatter to use when writing journal entries * @param journalWriter the journal writer to use to get journal entry sequence numbers and * complete the log when it needs to be rotated */ public EntryOutputStream(UnderFileSystem ufs, URI log, JournalFormatter journalFormatter, UfsJournalWriter journalWriter) throws IOException { mUfs = ufs; mCurrentLog = log; mJournalFormatter = journalFormatter; mJournalWriter = journalWriter; mMaxLogSize = ServerConfiguration.getBytes(PropertyKey.MASTER_JOURNAL_LOG_SIZE_BYTES_MAX); mRawOutputStream = mUfs.create(mCurrentLog.toString(), CreateOptions.defaults(ServerConfiguration.global()) .setEnsureAtomic(false).setCreateParent(true)); LOG.info("Opened current log file: {}", mCurrentLog); mDataOutputStream = new DataOutputStream(mRawOutputStream); }
/** * Tests for building an {@link CreateOptions} with a security enabled * configuration. */ @Test public void securityEnabled() throws IOException { mConfiguration.set(PropertyKey.SECURITY_AUTHENTICATION_TYPE, AuthType.SIMPLE.getAuthName()); mConfiguration.set(PropertyKey.SECURITY_LOGIN_USERNAME, "foo"); // Use IdentityUserGroupMapping to map user "foo" to group "foo". mConfiguration.set(PropertyKey.SECURITY_GROUP_MAPPING_CLASS, IdentityUserGroupsMapping.class.getName()); CreateOptions options = CreateOptions.defaults(mConfiguration); assertFalse(options.getCreateParent()); assertFalse(options.isEnsureAtomic()); assertNull(options.getOwner()); assertNull(options.getGroup()); String umask = mConfiguration.get(PropertyKey.SECURITY_AUTHORIZATION_PERMISSION_UMASK); assertEquals(ModeUtils.applyFileUMask(Mode.defaults(), umask), options.getMode()); }
private void createUfsFile(UfsFileWriteRequestContext context) throws IOException { UfsFileWriteRequest request = context.getRequest(); Preconditions.checkState(request != null); Protocol.CreateUfsFileOptions createUfsFileOptions = request.getCreateUfsFileOptions(); UfsManager.UfsClient ufsClient = mUfsManager.get(createUfsFileOptions.getMountId()); CloseableResource<UnderFileSystem> ufsResource = ufsClient.acquireUfsResource(); context.setUfsResource(ufsResource); UnderFileSystem ufs = ufsResource.get(); CreateOptions createOptions = CreateOptions.defaults(ServerConfiguration.global()) .setOwner(createUfsFileOptions.getOwner()).setGroup(createUfsFileOptions.getGroup()) .setMode(new Mode((short) createUfsFileOptions.getMode())); if (createUfsFileOptions.hasAcl()) { // This acl information will be ignored by all but HDFS implementations createOptions.setAcl(ProtoUtils.fromProto(createUfsFileOptions.getAcl())); } context.setOutputStream(ufs.create(request.getUfsPath(), createOptions)); context.setCreateOptions(createOptions); String ufsString = MetricsSystem.escape(ufsClient.getUfsMountPointUri()); String counterName = Metric.getMetricNameWithTags(WorkerMetrics.BYTES_WRITTEN_UFS, WorkerMetrics.TAG_UFS, ufsString); Counter counter = MetricsSystem.counter(counterName); context.setCounter(counter); String meterName = Metric.getMetricNameWithTags(WorkerMetrics.BYTES_WRITTEN_UFS_THROUGHPUT, WorkerMetrics.TAG_UFS, ufsString); context.setMeter(MetricsSystem.meter(meterName)); } }
@Test public void createParent() throws IOException { String testFile = PathUtils.concatPath(mUnderfsAddress, "createParent/testFile"); OutputStream o = mUfs.create(testFile, CreateOptions.defaults(mConfiguration) .setCreateParent(true)); o.close(); assertTrue(mUfs.exists(testFile)); }
@Override protected void completeRequest(UfsFileWriteRequestContext context) throws Exception { if (context == null) { return; } if (context.getOutputStream() == null) { createUfsFile(context); } Preconditions.checkState(context.getOutputStream() != null); context.getOutputStream().close(); CreateOptions createOptions = context.getCreateOptions(); if (createOptions != null) { try { // Set the owner/group of the file to the correct owner. context.getUfsResource().get() .setOwner(context.getRequest().getUfsPath(), createOptions.getOwner(), createOptions.getGroup()); } catch (IOException e) { LOG.warn("Failed to update ownership for ufs path: {} owner: {} group: {} error: {}", context.getRequest().getUfsPath(), createOptions.getOwner(), createOptions.getGroup(), e.toString()); } } context.setOutputStream(null); context.setCreateOptions(null); context.getUfsResource().close(); }
@Override public OutputStream create(String path, CreateOptions options) throws IOException { if (!options.isEnsureAtomic()) { return createDirect(path, options); } return new AtomicFileOutputStream(path, this, options); }
@Override public OutputStream create(String path, CreateOptions options) throws IOException { if (options.getCreateParent() && !mkdirs(getParentPath(path))) { throw new IOException(ExceptionMessage.PARENT_CREATION_FAILED.getMessage(path)); } return createObject(stripPrefixIfPresent(path)); }