/** * @param key property key * @return true if the key is contained in the given UFS configuration or global configuration */ public boolean containsKey(PropertyKey key) { return (mUfsConf != null && mUfsConf.containsKey(key.toString())) || Configuration.isSet(key); }
/** * Gets the property key if the property name matches the template. * * @param propertyName name of the property * @return the property key, or null if the property name does not match the template */ @Nullable private PropertyKey getPropertyKey(String propertyName) { Matcher matcher = match(propertyName); if (!matcher.matches()) { return null; } // if the template can extract a nested property, build the new property from the nested one String nestedKeyName = null; try { nestedKeyName = matcher.group(NESTED_GROUP); } catch (IllegalArgumentException e) { // ignore if group is not found } PropertyKey nestedProperty = null; if (nestedKeyName != null && isValid(nestedKeyName)) { nestedProperty = fromString(nestedKeyName); } return mPropertyCreator.apply(propertyName, nestedProperty); } }
/** * Registers the given key to the global key map. * * @param key th property * @return whether the property key is successfully registered */ @VisibleForTesting public static boolean register(PropertyKey key) { String name = key.getName(); String[] aliases = key.getAliases(); if (DEFAULT_KEYS_MAP.containsKey(name)) { if (DEFAULT_KEYS_MAP.get(name).isBuiltIn() || !key.isBuiltIn()) { return false; } } DEFAULT_KEYS_MAP.put(name, key); if (aliases != null) { for (String alias : aliases) { DEFAULT_ALIAS_MAP.put(alias, key); } } return true; }
/** * Checks if there is a value set for the given key. * * @param key the key to check * @return true if there is value for the key, false otherwise */ public boolean isSet(PropertyKey key) { if (mUserProps.containsKey(key)) { return mUserProps.get(key).isPresent(); } // In case key is not the reference to the original key return PropertyKey.fromString(key.toString()).getDefaultValue() != null; }
@Override public Map<String, String> getNestedProperties(PropertyKey prefixKey) { Map<String, String> ret = Maps.newHashMap(); for (Map.Entry<PropertyKey, String> entry: mProperties.entrySet()) { String key = entry.getKey().getName(); if (prefixKey.isNested(key)) { String suffixKey = key.substring(prefixKey.length() + 1); ret.put(suffixKey, entry.getValue()); } } return ret; }
if (!PropertyKey.isValid(propertyName)) { continue; PropertyKey propertyKey = PropertyKey.fromString(propertyName); PropertyKey.ConsistencyCheckLevel level = propertyKey.getConsistencyLevel(); if (level == PropertyKey.ConsistencyCheckLevel.IGNORE) { continue; Scope scope = propertyKey.getScope(); Set<String> targetNodes = ImmutableSet.of(); if (scope.contains(Scope.MASTER)) {
/** * Gets all global configuration properties filtered by the specified scope. * * @param scope the scope to filter by * @return the properties */ public static List<ConfigProperty> getConfiguration(Scope scope) { ConfigurationValueOptions useRawDisplayValue = ConfigurationValueOptions.defaults().useDisplayValue(true).useRawValue(true); return Configuration.keySet().stream() .filter(key -> key.getScope().contains(scope)) .filter(key -> key.isValid(key.getName())) .map(key -> new ConfigProperty() .setName(key.getName()) .setSource(Configuration.getSource(key).toString()).setValue( Configuration.isSet(key) ? Configuration.get(key, useRawDisplayValue) : null)) .collect(toList()); }
String name = property.getName(); if (PropertyKey.isValid(name) && property.getValue() != null) { PropertyKey key = PropertyKey.fromString(name); if (!key.getScope().contains(Scope.CLIENT)) { LOG.debug("Loading cluster default: {} ({}) -> {}", key, key.getScope(), value);
/** * Checks that tiered locality configuration is consistent. * * @throws IllegalStateException if invalid tiered locality configuration is encountered */ private void checkTieredLocality() { // Check that any custom tiers set by alluxio.locality.{custom_tier}=value are also defined in // the tier ordering defined by alluxio.locality.order. Set<String> tiers = Sets.newHashSet(getList(PropertyKey.LOCALITY_ORDER, ",")); Set<PropertyKey> predefinedKeys = new HashSet<>(PropertyKey.defaultKeys()); for (PropertyKey key : mProperties.keySet()) { if (predefinedKeys.contains(key)) { // Skip non-templated keys. continue; } Matcher matcher = Template.LOCALITY_TIER.match(key.toString()); if (matcher.matches() && matcher.group(1) != null) { String tierName = matcher.group(1); if (!tiers.contains(tierName)) { throw new IllegalStateException( String.format("Tier %s is configured by %s, but does not exist in the tier list %s " + "configured by %s", tierName, key, tiers, PropertyKey.LOCALITY_ORDER)); } } } } }
PropertyKey key = entry.getKey(); InconsistentProperty inconsistentProperty = new InconsistentProperty() .setName(key.getName()).setValues(entry.getValue()); Scope scope = key.getScope().equals(Scope.ALL) ? Scope.SERVER : key.getScope(); if (entry.getKey().getConsistencyLevel().equals(ConsistencyCheckLevel.ENFORCE)) { confErrors.putIfAbsent(scope, new ArrayList<>()); confErrors.get(scope).add(inconsistentProperty);
@Override public void validate() { if (!getBoolean(PropertyKey.CONF_VALIDATION_ENABLED)) { return; } for (PropertyKey key : keySet()) { Preconditions.checkState( getSource(key).getType() != Source.Type.SITE_PROPERTY || !key.isIgnoredSiteProperty(), "%s is not accepted in alluxio-site.properties, " + "and must be specified as a JVM property. " + "If no JVM property is present, Alluxio will use default value '%s'.", key.getName(), key.getDefaultValue()); } checkTimeouts(); checkWorkerPorts(); checkUserFileBufferBytes(); checkZkConfiguration(); checkTieredLocality(); }
/** * Unregisters the given key from the global key map. * * @param key the property to unregister */ @VisibleForTesting public static void unregister(PropertyKey key) { String name = key.getName(); DEFAULT_KEYS_MAP.remove(name); DEFAULT_ALIAS_MAP.remove(name); }
@Override public List<ConfigProperty> getConfiguration(GetConfigurationOptions options) { List<ConfigProperty> configInfoList = new ArrayList<>(); for (PropertyKey key : Configuration.keySet()) { if (key.isBuiltIn()) { String source = Configuration.getSource(key).toString(); String value = Configuration.getOrDefault(key, null, ConfigurationValueOptions.defaults().useDisplayValue(true) .useRawValue(options.isRawValue())); configInfoList .add(new ConfigProperty().setName(key.getName()).setValue(value).setSource(source)); } } return configInfoList; }
/** * Merges Hadoop {@link org.apache.hadoop.conf.Configuration} into the Alluxio configuration. * * @param source the {@link org.apache.hadoop.conf.Configuration} to merge * @param alluxioConfiguration the Alluxio configuration to merge to */ public static void mergeHadoopConfiguration(org.apache.hadoop.conf.Configuration source, AlluxioConfiguration alluxioConfiguration) { // Load Alluxio configuration if any and merge to the one in Alluxio file system // Push Alluxio configuration to the Job configuration Properties alluxioConfProperties = new Properties(); // Load any Alluxio configuration parameters existing in the Hadoop configuration. for (Map.Entry<String, String> entry : source) { String propertyName = entry.getKey(); if (PropertyKey.isValid(propertyName)) { alluxioConfProperties.put(propertyName, entry.getValue()); } } LOG.info("Loading Alluxio properties from Hadoop configuration: {}", alluxioConfProperties); // Merge the relevant Hadoop configuration into Alluxio's configuration. // TODO(jiri): support multiple client configurations (ALLUXIO-2034) alluxioConfiguration.merge(alluxioConfProperties, Source.RUNTIME); alluxioConfiguration.validate(); } }
@Override public String get(PropertyKey key, ConfigurationValueOptions options) { String value = mProperties.get(key); if (value == null) { // if value or default value is not set in configuration for the given key throw new RuntimeException(ExceptionMessage.UNDEFINED_CONFIGURATION_KEY.getMessage(key)); } if (!options.shouldUseRawValue()) { value = lookup(value); } if (options.shouldUseDisplayValue()) { PropertyKey.DisplayType displayType = key.getDisplayType(); switch (displayType) { case DEFAULT: break; case CREDENTIALS: value = "******"; break; default: throw new IllegalStateException(String.format("Invalid displayType %s for property %s", displayType.name(), key.getName())); } } return value; }
@Override public int getUriDefaultPort() { return Integer.parseInt(PropertyKey.MASTER_RPC_PORT.getDefaultValue()); } }
/** * Fills the configuration map. * * @param targetMap the map to fill * @param recordMap the map to get data from */ private void fillConfMap(Map<PropertyKey, Map<Optional<String>, List<String>>> targetMap, Map<Address, List<ConfigRecord>> recordMap) { for (Map.Entry<Address, List<ConfigRecord>> record : recordMap.entrySet()) { Address address = record.getKey(); String addressStr = String.format("%s:%s", address.getHost(), address.getRpcPort()); for (ConfigRecord conf : record.getValue()) { PropertyKey key = conf.getKey(); if (key.getConsistencyLevel() == ConsistencyCheckLevel.IGNORE) { continue; } Optional<String> value = conf.getValue(); targetMap.putIfAbsent(key, new HashMap<>()); Map<Optional<String>, List<String>> values = targetMap.get(key); values.putIfAbsent(value, new ArrayList<>()); values.get(value).add(addressStr); } } } }
/** * @param key the key to query * @return the value, or null if the key has no value set */ @Nullable public String get(PropertyKey key) { if (mUserProps.containsKey(key)) { return mUserProps.get(key).orElse(null); } // In case key is not the reference to the original key return PropertyKey.fromString(key.toString()).getDefaultValue(); }
@Override public Map<String, String> toMap(ConfigurationValueOptions opts) { Map<String, String> map = new HashMap<>(); // Cannot use Collectors.toMap because we support null keys. keySet().forEach(key -> map.put(key.getName(), getOrDefault(key, null, opts))); return map; }
/** * Gets the default port number on service. * * @return default port */ public int getDefaultPort() { return Integer.parseInt(mPortKey.getDefaultValue()); } }