/** * @param bucketPath the bucket path like "/bucket", "/mount/point/bucket" * @param objectKey the object key like "img/2017/9/1/s3.jpg" * @return the temporary directory used to hold parts of the object during multipart uploads */ public static String getMultipartTemporaryDirForObject(String bucketPath, String objectKey) { String multipartTemporaryDirSuffix = Configuration.get(PropertyKey.PROXY_S3_MULTIPART_TEMPORARY_DIR_SUFFIX); return bucketPath + AlluxioURI.SEPARATOR + objectKey + multipartTemporaryDirSuffix; }
/** * Constructs a {@link PermissionChecker} instance for Alluxio file system. * * @param inodeTree inode tree of the file system master */ public DefaultPermissionChecker(InodeTree inodeTree) { mInodeTree = Preconditions.checkNotNull(inodeTree, "inodeTree"); mPermissionCheckEnabled = Configuration.getBoolean(PropertyKey.SECURITY_AUTHORIZATION_PERMISSION_ENABLED); mFileSystemSuperGroup = Configuration.get(PropertyKey.SECURITY_AUTHORIZATION_PERMISSION_SUPERGROUP); mInodePermissionChecker = InodePermissionChecker.create(); }
/** * @return the instance of under file system for Alluxio root directory */ public static UnderFileSystem createForRoot() { String ufsRoot = Configuration.get(PropertyKey.MASTER_MOUNT_TABLE_ROOT_UFS); boolean readOnly = Configuration.getBoolean(PropertyKey.MASTER_MOUNT_TABLE_ROOT_READONLY); boolean shared = Configuration.getBoolean(PropertyKey.MASTER_MOUNT_TABLE_ROOT_SHARED); Map<String, String> ufsConf = Configuration.getNestedProperties(PropertyKey.MASTER_MOUNT_TABLE_ROOT_OPTION); return create(ufsRoot, UnderFileSystemConfiguration.defaults().setReadOnly(readOnly) .setShared(shared).setUserSpecifiedConf(ufsConf)); } }
@Override public String getUfsAddress() { return Configuration.get(PropertyKey.MASTER_MOUNT_TABLE_ROOT_UFS); }
/** * @return a primary selector based on zookeeper configuration */ public static PrimarySelector createZkPrimarySelector() { String zkAddress = Configuration.get(PropertyKey.ZOOKEEPER_ADDRESS); String zkElectionPath = Configuration.get(PropertyKey.ZOOKEEPER_ELECTION_PATH); String zkLeaderPath = Configuration.get(PropertyKey.ZOOKEEPER_LEADER_PATH); return new PrimarySelectorClient(zkAddress, zkElectionPath, zkLeaderPath); }
/** * @return the journal location */ private static URI getJournalLocation() { String journalDirectory = Configuration.get(PropertyKey.MASTER_JOURNAL_FOLDER); if (!journalDirectory.endsWith(AlluxioURI.SEPARATOR)) { journalDirectory += AlluxioURI.SEPARATOR; } try { return new URI(journalDirectory); } catch (URISyntaxException e) { throw new RuntimeException(e); } }
/** * Creates a new instance of {@link AppCallbackHandler}. */ public AppCallbackHandler() { if (Configuration.isSet(PropertyKey.SECURITY_LOGIN_USERNAME)) { mUserName = Configuration.get(PropertyKey.SECURITY_LOGIN_USERNAME); } else { mUserName = ""; } }
/** * Creates a new instance of {@link UfsSuperUserValidationTask} * for validating root under file system. */ public UfsSuperUserValidationTask() { mUfs = UnderFileSystem.Factory.createForRoot(); mPath = Configuration.get(PropertyKey.MASTER_MOUNT_TABLE_ROOT_UFS); }
private static void stopAlluxio() throws Exception { String stopScript = PathUtils.concatPath(Configuration.get(PropertyKey.HOME), "bin", "alluxio-stop.sh"); ProcessBuilder pb = new ProcessBuilder(stopScript, "all"); pb.start().waitFor(); }
/** * @param storageDir the root of a storage directory in tiered storage * * @return the worker data folder path after each storage directory, the final path will be like * "/mnt/ramdisk/alluxioworker" for storage dir "/mnt/ramdisk" by appending * {@link PropertyKey#WORKER_DATA_FOLDER). */ public static String getWorkerDataDirectory(String storageDir) { return concatPath( storageDir.trim(), Configuration.get(PropertyKey.WORKER_DATA_FOLDER)); }
/** * Creates a new instance of {@link UfsDirectoryValidationTask} * for validating root under file system. */ public UfsDirectoryValidationTask() { mUfs = UnderFileSystem.Factory.createForRoot(); mPath = Configuration.get(PropertyKey.MASTER_MOUNT_TABLE_ROOT_UFS); }
private LocalAlluxioMaster() throws IOException { mHostname = NetworkAddressUtils.getConnectHost(ServiceType.MASTER_RPC); mJournalFolder = Configuration.get(PropertyKey.MASTER_JOURNAL_FOLDER); }
private void startAlluxioFramework(Map<String, String> extraEnv) { String startScript = PathUtils.concatPath(Configuration.get(PropertyKey.HOME), "integration", "mesos", "bin", "alluxio-mesos-start.sh"); ProcessBuilder pb = new ProcessBuilder(startScript, mMesosAddress); Map<String, String> env = pb.environment(); env.putAll(extraEnv); try { pb.start().waitFor(); } catch (Exception e) { LOG.info("Failed to launch Alluxio on Mesos. Note that this test requires that " + "Mesos is currently running."); throw new RuntimeException(e); } }
private static void stopAlluxioFramework() throws Exception { String stopScript = PathUtils.concatPath(Configuration.get(PropertyKey.HOME), "integration", "mesos", "bin", "alluxio-mesos-stop.sh"); ProcessBuilder pb = new ProcessBuilder(stopScript); pb.start().waitFor(); // Wait for Mesos to unregister and shut down the Alluxio Framework. CommonUtils.sleepMs(5000); }
protected boolean shouldSkip() { String scheme = new AlluxioURI(Configuration.get(PropertyKey.MASTER_MOUNT_TABLE_ROOT_UFS)).getScheme(); if (scheme == null || !scheme.startsWith("hdfs")) { System.out.format("Root underFS is not HDFS. Skipping validation for HDFS properties.%n"); return true; } return false; }
/** * Finds all factory from the extensions directory. * * @param factories list of factories to add to */ private void scanExtensions(List<T> factories) { LOG.info("Loading extension jars from {}", Configuration.get(PropertyKey.EXTENSIONS_DIR)); scan(Arrays.asList(ExtensionUtils.listExtensions()), factories); }
/** * Starts Alluxio Master by executing the launch script. */ private static void startMaster() { String alluxioStartPath = PathUtils.concatPath(Configuration.get(PropertyKey.HOME), "bin", "alluxio-start.sh"); String startMasterCommand = String.format("%s master", alluxioStartPath); try { Runtime.getRuntime().exec(startMasterCommand).waitFor(); CommonUtils.sleepMs(LOG, 1000); } catch (Exception e) { LOG.error("Error when starting Master", e); } }
/** * Gets the local hostname to be used by the client. If this isn't configured, a non-loopback * local hostname will be looked up. * * @return the local hostname for the client */ public static String getClientHostName() { if (Configuration.isSet(PropertyKey.USER_HOSTNAME)) { return Configuration.get(PropertyKey.USER_HOSTNAME); } return getLocalHostName(); }
private void deleteObject(String bucket, String object) throws S3Exception { String bucketPath = parseBucketPath(AlluxioURI.SEPARATOR + bucket); // Delete the object. String objectPath = bucketPath + AlluxioURI.SEPARATOR + object; DeleteOptions options = DeleteOptions.defaults(); options.setAlluxioOnly(Configuration.get(PropertyKey.PROXY_S3_DELETE_TYPE) .equals(Constants.S3_DELETE_IN_ALLUXIO_ONLY)); try { mFileSystem.delete(new AlluxioURI(objectPath), options); } catch (Exception e) { throw toObjectS3Exception(e, objectPath); } }
private Upgrader(String master) { mMaster = master; mJournalV0 = (new alluxio.master.journalv0.MutableJournal.Factory( getJournalLocation(sJournalDirectoryV0))).create(master); mJournalV1 = new UfsJournal(getJournalLocation(Configuration.get(PropertyKey.MASTER_JOURNAL_FOLDER)), new NoopMaster(master), 0); mUfs = UnderFileSystem.Factory.create(sJournalDirectoryV0); mCheckpointV0 = URIUtils.appendPathOrDie(mJournalV0.getLocation(), "checkpoint.data"); mCompletedLogsV0 = URIUtils.appendPathOrDie(mJournalV0.getLocation(), "completed"); mCheckpointsV1 = URIUtils.appendPathOrDie(mJournalV1.getLocation(), "checkpoints"); mLogsV1 = URIUtils.appendPathOrDie(mJournalV1.getLocation(), "logs"); }