private static String getSuffixedConf(Configuration conf, String key, String defaultVal, String[] suffixes) { String ret = conf.get(DFSUtil.addKeySuffixes(key, suffixes)); if (ret != null) { return ret; } return conf.get(key, defaultVal); }
String nameserviceId, String nnId, String... keys) { for (String key : keys) { String value = conf.get(addKeySuffixes(key, nameserviceId, nnId)); if (value != null) { conf.set(key, value); continue; value = conf.get(addKeySuffixes(key, nameserviceId)); if (value != null) { conf.set(key, value);
"Could not determine namenode ids in namespace '%s'. " + "Please configure " + DFSUtil.addKeySuffixes(DFSConfigKeys.DFS_HA_NAMENODES_KEY_PREFIX, nsId), nsId);
address = DFSUtilClient.getAddressesForNameserviceId( conf, nsId, null, confKey).get(namenodeId); bindHostActualKey = DFSUtil.addKeySuffixes(bindHostKey, nsId, namenodeId); } else { address = NetUtils.createSocketAddr(conf.get(confKey, defaultValue));
addKeySuffixes(PROXY_PROVIDER_KEY_PREFIX, nsId), InMemoryAliasMapFailoverProxyProvider.class, AbstractNNFailoverProxyProvider.class); } else { String key = addKeySuffixes(DFS_PROVIDED_ALIASMAP_INMEMORY_RPC_ADDRESS, nsId); String addr = conf.get(key); if (addr != null) {
continue; String key = addKeySuffixes(addressKey, nsId, nnId); String addr = conf.get(key); if (addr == null) {
private static String getSuffixedConf(Configuration conf, String key, String defaultVal, String[] suffixes) { String ret = conf.get(DFSUtil.addKeySuffixes(key, suffixes)); if (ret != null) { return ret; } return conf.get(key, defaultVal); }
private static String getSuffixedConf(Configuration conf, String key, String defaultVal, String[] suffixes) { String ret = conf.get(DFSUtil.addKeySuffixes(key, suffixes)); if (ret != null) { return ret; } return conf.get(key, defaultVal); }
public static void setFakeHttpAddresses(Configuration conf, final String logicalName) { conf.set(DFSUtil.addKeySuffixes( DFSConfigKeys.DFS_NAMENODE_HTTP_ADDRESS_KEY, logicalName, "nn1"), "127.0.0.1:12345"); conf.set(DFSUtil.addKeySuffixes( DFSConfigKeys.DFS_NAMENODE_HTTP_ADDRESS_KEY, logicalName, "nn2"), "127.0.0.1:12346"); }
private static void addNN(Configuration conf, String ns, String addr) { String key = DFSUtil.addKeySuffixes( DFSConfigKeys.DFS_NAMENODE_RPC_ADDRESS_KEY, ns); conf.set(key, addr); } }
private void setHAConf(Configuration conf, String nn1Addr, String nn2Addr) { conf.set(CommonConfigurationKeysPublic.FS_DEFAULT_NAME_KEY, "hdfs://" + NSID); conf.set(DFSConfigKeys.DFS_NAMESERVICES, NSID); conf.set(DFSConfigKeys.DFS_NAMESERVICE_ID, NSID); conf.set(DFSUtil.addKeySuffixes( DFSConfigKeys.DFS_HA_NAMENODES_KEY_PREFIX, NSID), "nn1,nn2"); conf.set(DFSConfigKeys.DFS_HA_NAMENODE_ID_KEY, "nn1"); conf.set(DFSUtil.addKeySuffixes( DFSConfigKeys.DFS_NAMENODE_RPC_ADDRESS_KEY, NSID, "nn1"), nn1Addr); conf.set(DFSUtil.addKeySuffixes( DFSConfigKeys.DFS_NAMENODE_RPC_ADDRESS_KEY, NSID, "nn2"), nn2Addr); }
private Configuration getHAConf(String nsId, String host1, String host2) { Configuration conf = new Configuration(); conf.set(DFSConfigKeys.DFS_NAMESERVICES, nsId); conf.set(DFSUtil.addKeySuffixes( DFSConfigKeys.DFS_HA_NAMENODES_KEY_PREFIX, nsId), "nn1,nn2"); conf.set(DFSConfigKeys.DFS_HA_NAMENODE_ID_KEY, "nn1"); conf.set(DFSUtil.addKeySuffixes( DFSConfigKeys.DFS_NAMENODE_RPC_ADDRESS_KEY, nsId, "nn1"), host1 + ":12345"); conf.set(DFSUtil.addKeySuffixes( DFSConfigKeys.DFS_NAMENODE_RPC_ADDRESS_KEY, nsId, "nn2"), host2 + ":12345"); return conf; }
@Test public void testInitializeSharedEditsConfiguresGenericConfKeys() throws IOException { Configuration conf = new Configuration(); conf.set(DFSConfigKeys.DFS_NAMESERVICES, "ns1"); conf.set(DFSUtil.addKeySuffixes(DFSConfigKeys.DFS_HA_NAMENODES_KEY_PREFIX, "ns1"), "nn1,nn2"); conf.set(DFSUtil.addKeySuffixes(DFSConfigKeys.DFS_NAMENODE_RPC_ADDRESS_KEY, "ns1", "nn1"), "localhost:1234"); assertNull(conf.get(DFSConfigKeys.DFS_NAMENODE_RPC_ADDRESS_KEY)); NameNode.initializeSharedEdits(conf); assertNotNull(conf.get(DFSConfigKeys.DFS_NAMENODE_RPC_ADDRESS_KEY)); } }
public static void setFederatedConfiguration(MiniDFSCluster cluster, Configuration conf) { Set<String> nameservices = new HashSet<String>(); for (NameNodeInfo info : cluster.getNameNodeInfos()) { assert info.nameserviceId != null; nameservices.add(info.nameserviceId); conf.set(DFSUtil.addKeySuffixes(DFS_NAMENODE_RPC_ADDRESS_KEY, info.nameserviceId), DFSUtil.createUri(HdfsConstants.HDFS_URI_SCHEME, info.nameNode.getNameNodeAddress()).toString()); conf.set(DFSUtil.addKeySuffixes(DFS_NAMENODE_SERVICE_RPC_ADDRESS_KEY, info.nameserviceId), DFSUtil.createUri(HdfsConstants.HDFS_URI_SCHEME, info.nameNode.getNameNodeAddress()).toString()); } conf.set(DFSConfigKeys.DFS_NAMESERVICES, Joiner.on(",") .join(nameservices)); }
private static Configuration createWebHDFSHAConfiguration(String logicalHostName, String nnaddr1, String nnaddr2) { HdfsConfiguration conf = new HdfsConfiguration(); conf.set(DFS_NAMESERVICES, "ns1"); conf.set(DFSUtil.addKeySuffixes(DFS_HA_NAMENODES_KEY_PREFIX, "ns1"),"nn1,nn2"); conf.set(DFSUtil.addKeySuffixes( DFS_NAMENODE_HTTP_ADDRESS_KEY, "ns1", "nn1"), nnaddr1); conf.set(DFSUtil.addKeySuffixes( DFS_NAMENODE_HTTP_ADDRESS_KEY, "ns1", "nn2"), nnaddr2); conf.set(DFS_CLIENT_FAILOVER_PROXY_PROVIDER_KEY_PREFIX + "." + logicalHostName, ConfiguredFailoverProxyProvider.class.getName()); return conf; }
/** * Initialize the address and port for this NameNode. In the * non-federated case, the nameservice and namenode ID may be * null. */ private static void initNameNodeAddress(Configuration conf, String nameserviceId, NNConf nnConf) { // Set NN-specific specific key String key = DFSUtil.addKeySuffixes( DFS_NAMENODE_HTTP_ADDRESS_KEY, nameserviceId, nnConf.getNnId()); conf.set(key, "127.0.0.1:" + nnConf.getHttpPort()); key = DFSUtil.addKeySuffixes( DFS_NAMENODE_RPC_ADDRESS_KEY, nameserviceId, nnConf.getNnId()); conf.set(key, "127.0.0.1:" + nnConf.getIpcPort()); }
/** Set a given key with value as address, for all the nameServiceIds. * @param conf configuration to set the addresses in * @param key configuration key * @param nameServiceIdCount Number of nameServices for which the key is set * @param portOffset starting port offset * @return list of addresses that are set in the configuration */ private String[] setupAddress(HdfsConfiguration conf, String key, int nameServiceIdCount, int portOffset) { String[] values = new String[nameServiceIdCount]; for (int i = 0; i < nameServiceIdCount; i++, portOffset++) { String nsID = getNameServiceId(i); String specificKey = DFSUtil.addKeySuffixes(key, nsID); values[i] = "nn" + i + ":" + portOffset; conf.set(specificKey, values[i]); } return values; }
private HdfsConfiguration getHAConf() { HdfsConfiguration conf = new HdfsConfiguration(); conf.set(DFSConfigKeys.DFS_NAMESERVICES, NSID); conf.set(DFSConfigKeys.DFS_NAMESERVICE_ID, NSID); conf.set(DFSUtil.addKeySuffixes( DFSConfigKeys.DFS_HA_NAMENODES_KEY_PREFIX, NSID), "nn1,nn2"); conf.set(DFSConfigKeys.DFS_HA_NAMENODE_ID_KEY, "nn1"); conf.set(DFSUtil.addKeySuffixes( DFSConfigKeys.DFS_NAMENODE_RPC_ADDRESS_KEY, NSID, "nn1"), HOST_A + ":12345"); conf.set(DFSUtil.addKeySuffixes( DFSConfigKeys.DFS_NAMENODE_RPC_ADDRESS_KEY, NSID, "nn2"), HOST_B + ":12345"); return conf; }
private Configuration setupAddress(String key) { HdfsConfiguration conf = new HdfsConfiguration(); conf.set(DFS_NAMESERVICES, "nn1"); conf.set(DFSUtil.addKeySuffixes(key, "nn1"), "localhost:9000"); return conf; }
/** * Test {@link DFSUtil#getNamenodeNameServiceId(Configuration)} to ensure * exception is thrown when multiple rpc addresses match the local node's * address */ @Test(expected = HadoopIllegalArgumentException.class) public void testGetNameServiceIdException() { HdfsConfiguration conf = new HdfsConfiguration(); conf.set(DFS_NAMESERVICES, "nn1,nn2"); conf.set(DFSUtil.addKeySuffixes(DFS_NAMENODE_RPC_ADDRESS_KEY, "nn1"), "localhost:9000"); conf.set(DFSUtil.addKeySuffixes(DFS_NAMENODE_RPC_ADDRESS_KEY, "nn2"), "localhost:9001"); DFSUtil.getNamenodeNameServiceId(conf); fail("Expected exception is not thrown"); }