/** Create an URI from scheme and address. */ public static URI createUri(String scheme, InetSocketAddress address) { return createUri(scheme, address.getHostName(), address.getPort()); }
/** Remove unnecessary path from HDFS URI. */ static URI trimUri(URI uri) { String path = uri.getPath(); if (HdfsConstants.HDFS_URI_SCHEME.equals(uri.getScheme()) && path != null && !path.isEmpty()) { uri = createUri(uri.getScheme(), uri.getHost(), uri.getPort()); } return uri; }
URI nsUri = createUri(HdfsConstants.HDFS_URI_SCHEME, nsId, -1); String addr = conf.get(DFSUtilClient.concatSuffixes(key, nsId)); if (addr != null) { URI uri = createUri(HdfsConstants.HDFS_URI_SCHEME, NetUtils.createSocketAddr(addr)); if (!uriFound) { String addr = conf.get(key); if (addr != null) { URI uri = createUri(HdfsConstants.HDFS_URI_SCHEME, NetUtils.createSocketAddr(addr)); if (!uriFound) { defaultUri = createUri(defaultUri.getScheme(), NetUtils.createSocketAddr(defaultUri.getHost(), defaultUri.getPort()));
protected URI getNamenodeUri(URI uri) { return DFSUtil.createUri(getUnderlyingProtocol(), getNamenodeAddr(uri)); }
protected URI getNamenodeUri(URI uri) { return DFSUtil.createUri(getUnderlyingProtocol(), getNamenodeAddr(uri)); }
@Override public Void run() throws Exception { InetSocketAddress serviceAddr = SecurityUtil .getTokenServiceAddr(token); DelegationTokenFetcher.cancelDelegationToken(connectionFactory, DFSUtil.createUri(getUnderlyingProtocol(), serviceAddr), (Token<DelegationTokenIdentifier>) token); return null; } });
@Override public Long run() throws Exception { InetSocketAddress serviceAddr = SecurityUtil .getTokenServiceAddr(token); return DelegationTokenFetcher.renewDelegationToken(connectionFactory, DFSUtil.createUri(getUnderlyingProtocol(), serviceAddr), (Token<DelegationTokenIdentifier>) token); } });
@Override public Long run() throws Exception { InetSocketAddress serviceAddr = SecurityUtil .getTokenServiceAddr(token); return DelegationTokenFetcher.renewDelegationToken(connectionFactory, DFSUtil.createUri(getUnderlyingProtocol(), serviceAddr), (Token<DelegationTokenIdentifier>) token); } });
@Override public Void run() throws Exception { InetSocketAddress serviceAddr = SecurityUtil .getTokenServiceAddr(token); DelegationTokenFetcher.cancelDelegationToken(connectionFactory, DFSUtil.createUri(getUnderlyingProtocol(), serviceAddr), (Token<DelegationTokenIdentifier>) token); return null; } });
String addr = conf.get(concatSuffixes(key, nsId)); if (addr != null) { URI uri = createUri(HdfsConstants.HDFS_URI_SCHEME, NetUtils.createSocketAddr(addr)); if (!uriFound) { String addr = conf.get(key); if (addr != null) { URI uri = createUri("hdfs", NetUtils.createSocketAddr(addr)); if (!uriFound) { uriFound = true; defaultUri = createUri(defaultUri.getScheme(), NetUtils.createSocketAddr(defaultUri.getHost(), defaultUri.getPort()));
String addr = conf.get(concatSuffixes(key, nsId)); if (addr != null) { URI uri = createUri(HdfsConstants.HDFS_URI_SCHEME, NetUtils.createSocketAddr(addr)); if (!uriFound) { String addr = conf.get(key); if (addr != null) { URI uri = createUri("hdfs", NetUtils.createSocketAddr(addr)); if (!uriFound) { uriFound = true; defaultUri = createUri(defaultUri.getScheme(), NetUtils.createSocketAddr(defaultUri.getHost(), defaultUri.getPort()));
public static void setFederatedConfiguration(MiniDFSCluster cluster, Configuration conf) { Set<String> nameservices = new HashSet<String>(); for (NameNodeInfo info : cluster.getNameNodeInfos()) { assert info.nameserviceId != null; nameservices.add(info.nameserviceId); conf.set(DFSUtil.addKeySuffixes(DFS_NAMENODE_RPC_ADDRESS_KEY, info.nameserviceId), DFSUtil.createUri(HdfsConstants.HDFS_URI_SCHEME, info.nameNode.getNameNodeAddress()).toString()); conf.set(DFSUtil.addKeySuffixes(DFS_NAMENODE_SERVICE_RPC_ADDRESS_KEY, info.nameserviceId), DFSUtil.createUri(HdfsConstants.HDFS_URI_SCHEME, info.nameNode.getNameNodeAddress()).toString()); } conf.set(DFSConfigKeys.DFS_NAMESERVICES, Joiner.on(",") .join(nameservices)); }
public static void setFederatedHAConfiguration(MiniDFSCluster cluster, Configuration conf) { Map<String, List<String>> nameservices = Maps.newHashMap(); for (NameNodeInfo info : cluster.getNameNodeInfos()) { Preconditions.checkState(info.nameserviceId != null); List<String> nns = nameservices.get(info.nameserviceId); if (nns == null) { nns = Lists.newArrayList(); nameservices.put(info.nameserviceId, nns); } nns.add(info.nnId); conf.set(DFSUtil.addKeySuffixes(DFS_NAMENODE_RPC_ADDRESS_KEY, info.nameserviceId, info.nnId), DFSUtil.createUri(HdfsConstants.HDFS_URI_SCHEME, info.nameNode.getNameNodeAddress()).toString()); conf.set(DFSUtil.addKeySuffixes(DFS_NAMENODE_SERVICE_RPC_ADDRESS_KEY, info.nameserviceId, info.nnId), DFSUtil.createUri(HdfsConstants.HDFS_URI_SCHEME, info.nameNode.getNameNodeAddress()).toString()); } for (Map.Entry<String, List<String>> entry : nameservices.entrySet()) { conf.set(DFSUtil.addKeySuffixes(DFS_HA_NAMENODES_KEY_PREFIX, entry.getKey()), Joiner.on(",").join(entry.getValue())); conf.set(DFS_CLIENT_FAILOVER_PROXY_PROVIDER_KEY_PREFIX + "." + entry .getKey(), ConfiguredFailoverProxyProvider.class.getName()); } conf.set(DFSConfigKeys.DFS_NAMESERVICES, Joiner.on(",") .join(nameservices.keySet())); }
UserGroupInformation.setConfiguration(clusterConf); uri = DFSUtil.createUri( "swebhdfs", cluster.getNameNode().getHttpsAddress()); validateLazyTokenFetch(clusterConf);
@SuppressWarnings("unchecked") // for any(Token.class) @Test public void testLazyTokenFetchForWebhdfs() throws Exception { MiniDFSCluster cluster = null; WebHdfsFileSystem fs = null; try { final Configuration clusterConf = new HdfsConfiguration(conf); SecurityUtil.setAuthenticationMethod(SIMPLE, clusterConf); clusterConf.setBoolean(DFSConfigKeys .DFS_NAMENODE_DELEGATION_TOKEN_ALWAYS_USE_KEY, true); // trick the NN into thinking security is enabled w/o it trying // to login from a keytab UserGroupInformation.setConfiguration(clusterConf); cluster = new MiniDFSCluster.Builder(clusterConf).numDataNodes(1).build(); cluster.waitActive(); SecurityUtil.setAuthenticationMethod(KERBEROS, clusterConf); UserGroupInformation.setConfiguration(clusterConf); uri = DFSUtil.createUri( "webhdfs", cluster.getNameNode().getHttpAddress()); validateLazyTokenFetch(clusterConf); } finally { IOUtils.cleanup(null, fs); if (cluster != null) { cluster.shutdown(); } } }