conf.put(PropertyKey.LOGGER_TYPE, "WORKER_LOGGER"); conf.put(PropertyKey.CONF_DIR, confDir.getAbsolutePath()); conf.put(PropertyKey.Template.WORKER_TIERED_STORE_LEVEL_DIRS_PATH.format(0), ramdisk.getAbsolutePath()); conf.put(PropertyKey.LOGS_DIR, logsDir.getAbsolutePath());
/** * Gets the property key if the property name matches the template. * * @param propertyName name of the property * @return the property key, or null if the property name does not match the template */ @Nullable private PropertyKey getPropertyKey(String propertyName) { Matcher matcher = match(propertyName); if (!matcher.matches()) { return null; } // if the template can extract a nested property, build the new property from the nested one String nestedKeyName = null; try { nestedKeyName = matcher.group(NESTED_GROUP); } catch (IllegalArgumentException e) { // ignore if group is not found } PropertyKey nestedProperty = null; if (nestedKeyName != null && isValid(nestedKeyName)) { nestedProperty = fromString(nestedKeyName); } return mPropertyCreator.apply(propertyName, nestedProperty); } }
/** * @param input string of property key * @return whether the input is a valid property name */ public static boolean isValid(String input) { // Check if input matches any default keys or aliases if (DEFAULT_KEYS_MAP.containsKey(input) || DEFAULT_ALIAS_MAP.containsKey(input)) { return true; } // Check if input matches any parameterized keys for (Template template : Template.values()) { if (template.matches(input)) { return true; } } return false; }
PropertyKey.Template.WORKER_TIERED_STORE_LEVEL_ALIAS.format(level); String alias = Configuration.get(tierAliasConf); PropertyKey.Template.WORKER_TIERED_STORE_LEVEL_DIRS_PATH.format(level); String[] dirPaths = Configuration.get(tierDirPathConf).split(","); PropertyKey.Template.WORKER_TIERED_STORE_LEVEL_DIRS_QUOTA.format(level); String rawDirQuota = Configuration.get(tierDirCapacityConf); if (rawDirQuota.isEmpty()) {
for (int level = 0; level < storageLevels; level++) { PropertyKey tierLevelDirPath = PropertyKey.Template.WORKER_TIERED_STORE_LEVEL_DIRS_PATH.format(level); String[] dirPaths = Configuration.get(tierLevelDirPath).split(","); String name = "Data path for tier " + level;
/** * Parses a string and return its corresponding {@link PropertyKey}, throwing exception if no such * a property can be found. * * @param input string of property key * @return corresponding property */ public static PropertyKey fromString(String input) { // First try to parse it as default key PropertyKey key = DEFAULT_KEYS_MAP.get(input); if (key != null) { return key; } // Try to match input with alias key = DEFAULT_ALIAS_MAP.get(input); if (key != null) { return key; } // Try different templates and see if any template matches for (Template template : Template.values()) { key = template.getPropertyKey(input); if (key != null) { return key; } } throw new IllegalArgumentException( ExceptionMessage.INVALID_CONFIGURATION_KEY.getMessage(input)); }
/** * Sets up corresponding directories for tests. */ protected void setupTest() throws IOException { UnderFileSystem ufs = UnderFileSystem.Factory.createForRoot(); String underfsAddress = Configuration.get(PropertyKey.MASTER_MOUNT_TABLE_ROOT_UFS); // Deletes the ufs dir for this test from to avoid permission problems UnderFileSystemUtils.deleteDirIfExists(ufs, underfsAddress); // Creates ufs dir. This must be called before starting UFS with UnderFileSystemCluster.get(). UnderFileSystemUtils.mkdirIfNotExists(ufs, underfsAddress); // Creates storage dirs for worker int numLevel = Configuration.getInt(PropertyKey.WORKER_TIERED_STORE_LEVELS); for (int level = 0; level < numLevel; level++) { PropertyKey tierLevelDirPath = PropertyKey.Template.WORKER_TIERED_STORE_LEVEL_DIRS_PATH.format(level); String[] dirPaths = Configuration.get(tierLevelDirPath).split(","); for (String dirPath : dirPaths) { FileUtils.createDir(dirPath); } } // Formats the journal Format.format(Format.Mode.MASTER); }
/** * Checks that tiered locality configuration is consistent. * * @throws IllegalStateException if invalid tiered locality configuration is encountered */ private void checkTieredLocality() { // Check that any custom tiers set by alluxio.locality.{custom_tier}=value are also defined in // the tier ordering defined by alluxio.locality.order. Set<String> tiers = Sets.newHashSet(getList(PropertyKey.LOCALITY_ORDER, ",")); Set<PropertyKey> predefinedKeys = new HashSet<>(PropertyKey.defaultKeys()); for (PropertyKey key : mProperties.keySet()) { if (predefinedKeys.contains(key)) { // Skip non-templated keys. continue; } Matcher matcher = Template.LOCALITY_TIER.match(key.toString()); if (matcher.matches() && matcher.group(1) != null) { String tierName = matcher.group(1); if (!tiers.contains(tierName)) { throw new IllegalStateException( String.format("Tier %s is configured by %s, but does not exist in the tier list %s " + "configured by %s", tierName, key, tiers, PropertyKey.LOCALITY_ORDER)); } } } } }
/** * Constructs a new {@link ImpersonationAuthenticator}. */ public ImpersonationAuthenticator() { mImpersonationGroups = new HashMap<>(); mImpersonationUsers = new HashMap<>(); for (PropertyKey key : Configuration.keySet()) { String value = Configuration.getOrDefault(key, null); // Process impersonation groups Matcher matcher = PropertyKey.Template.MASTER_IMPERSONATION_GROUPS_OPTION.match(key.getName()); if (matcher.matches()) { String connectionUser = matcher.group(1); if (connectionUser != null) { mImpersonationGroups.put(connectionUser, Sets.newHashSet(SPLITTER.split(value))); } } // Process impersonation users matcher = PropertyKey.Template.MASTER_IMPERSONATION_USERS_OPTION.match(key.getName()); if (matcher.matches()) { String connectionUser = matcher.group(1); if (connectionUser != null) { mImpersonationUsers.put(connectionUser, Sets.newHashSet(SPLITTER.split(value))); } } } }
/** * Constructs a new instance using the given {@link Configuration} object. The mapping cannot be * modified after creation. * * @param levelsProperty the property in the conf that specifies how many levels there are * @param template the format for the conf that identifies the alias for each level */ protected StorageTierAssoc(PropertyKey levelsProperty, PropertyKey.Template template) { int levels = Configuration.getInt(levelsProperty); ImmutableBiMap.Builder<String, Integer> builder = new ImmutableBiMap.Builder<>(); for (int i = 0; i < levels; i++) { String alias = Configuration.get(template.format(i)); builder.put(alias, i); } mAliasToOrdinal = builder.build(); }