public InstrumentedWebHDFSFileSystem() { super(SCHEME, new WebHdfsFileSystem()); } }
Assert.assertNull(fs.getRenewToken()); Token<?> token = fs.getDelegationToken(null); fs.renewDelegationToken(token); fs.cancelDelegationToken(token); verify(fs, never()).getDelegationToken(); verify(fs, never()).replaceExpiredDelegationToken(); verify(fs, never()).setDelegationToken(any(Token.class)); Assert.assertNull(fs.getRenewToken()); reset(fs); fs.create(p, (short)1).close(); verify(fs, times(1)).getDelegationToken(); verify(fs, never()).replaceExpiredDelegationToken(); verify(fs, times(1)).getDelegationToken(anyString()); verify(fs, times(1)).setDelegationToken(any(Token.class)); token = fs.getRenewToken(); Assert.assertNotNull(token); Assert.assertEquals(testUser, getTokenOwner(token)); Assert.assertEquals(fs.getTokenKind(), token.getKind()); reset(fs); fs.getFileStatus(p); verify(fs, times(1)).getDelegationToken(); verify(fs, never()).replaceExpiredDelegationToken(); verify(fs, never()).getDelegationToken(anyString()); verify(fs, never()).setDelegationToken(any(Token.class)); Token<?> token2 = fs.getRenewToken(); Assert.assertNotNull(token2); Assert.assertEquals(fs.getTokenKind(), token.getKind());
) throws IOException { super.initialize(uri, conf); setConf(conf); this.nnAddrs = resolveNNAddr(); HAUtilClient.buildTokenServiceForLogicalUri(uri, getScheme()) : SecurityUtil.buildTokenService(getCanonicalUri()); this.workingDir = makeQualified(new Path(getHomeDirectoryString(ugi))); this.canRefreshDelegationToken = UserGroupInformation.isSecurityEnabled(); this.disallowFallbackToInsecureCluster = !conf.getBoolean( CommonConfigurationKeys.IPC_CLIENT_FALLBACK_TO_SIMPLE_AUTH_ALLOWED_KEY, CommonConfigurationKeys.IPC_CLIENT_FALLBACK_TO_SIMPLE_AUTH_ALLOWED_DEFAULT); this.initializeRestCsrf(conf); this.delegationToken = null;
@Test(timeout = 5000) public void testTokenForNonTokenOp() throws IOException { WebHdfsFileSystem fs = spyWebhdfsInSecureSetup(); Token<?> token = mock(Token.class); doReturn(token).when(fs).getDelegationToken(null); // should get/set/renew token fs.toUrl(GetOpParam.Op.OPEN, null); verify(fs).getDelegationToken(); verify(fs).getDelegationToken(null); verify(fs).setDelegationToken(token); reset(fs); // should return prior token fs.toUrl(GetOpParam.Op.OPEN, null); verify(fs).getDelegationToken(); verify(fs, never()).getDelegationToken(null); verify(fs, never()).setDelegationToken(token); }
protected synchronized Token<?> getDelegationToken() throws IOException { if (canRefreshDelegationToken && delegationToken == null) { Token<?> token = tokenSelector.selectToken( new Text(getCanonicalServiceName()), ugi.getTokens()); // ugi tokens are usually indicative of a task which can't // refetch tokens. even if ugi has credentials, don't attempt // to get another token to match hdfs/rpc behavior if (token != null) { LOG.debug("Using UGI token: " + token); canRefreshDelegationToken = false; } else { token = getDelegationToken(null); if (token != null) { LOG.debug("Fetched new token: " + token); } else { // security is disabled canRefreshDelegationToken = false; } } setDelegationToken(token); } return delegationToken; }
@Test public void testWebHdfsCustomDefaultPorts() throws IOException { URI uri = URI.create("webhdfs://localhost"); WebHdfsFileSystem fs = (WebHdfsFileSystem) FileSystem.get(uri, conf); assertEquals(123, fs.getDefaultPort()); assertEquals(uri, fs.getUri()); assertEquals("127.0.0.1:123", fs.getCanonicalServiceName()); }
final Path root = new Path("/"); final Path dir = new Path("/test/testUrl"); assertTrue(webhdfs.mkdirs(dir)); final Path file = new Path("/test/file"); final FSDataOutputStream out = webhdfs.create(file); out.write(1); out.close(); final URL url = webhdfs.toUrl(GetOpParam.Op.GETHOMEDIRECTORY, root); final HttpURLConnection conn = (HttpURLConnection) url.openConnection(); final Map<?, ?> m = WebHdfsTestUtil.connectAndGetJson( conn, HttpServletResponse.SC_OK); assertEquals(WebHdfsFileSystem.getHomeDirectoryString(ugi), m.get(Path.class.getSimpleName())); conn.disconnect(); final URL url = webhdfs.toUrl(GetOpParam.Op.GETHOMEDIRECTORY, root, new DoAsParam(ugi.getShortUserName() + "proxy")); final HttpURLConnection conn = (HttpURLConnection) url.openConnection(); final URL url = webhdfs.toUrl(PutOpParam.Op.SETOWNER, dir); final HttpURLConnection conn = (HttpURLConnection) url.openConnection(); conn.connect(); final URL url = webhdfs.toUrl(op, dir); final HttpURLConnection conn = (HttpURLConnection) url.openConnection(); conn.setRequestMethod(op.getType().toString()); assertFalse(webhdfs.setReplication(dir, (short)1));
/** Remove offset parameter before returning the resolved url. */ @Override protected URL getResolvedUrl(final HttpURLConnection connection ) throws MalformedURLException { return removeOffsetParam(connection.getURL()); } }
private void checkNoTokenForOperation(HttpOpParam.Op op) throws IOException { WebHdfsFileSystem fs = spyWebhdfsInSecureSetup(); doReturn(null).when(fs).getDelegationToken(null); fs.initialize(URI.create("webhdfs://127.0.0.1:0"), conf); // do not get a token! fs.toUrl(op, null); verify(fs, never()).getDelegationToken(); verify(fs, never()).getDelegationToken(null); verify(fs, never()).setDelegationToken((Token<?>)any(Token.class)); }
@VisibleForTesting synchronized boolean replaceExpiredDelegationToken() throws IOException { boolean replaced = false; if (canRefreshDelegationToken) { Token<?> token = getDelegationToken(null); LOG.debug("Replaced expired token: " + token); setDelegationToken(token); replaced = (token != null); } return replaced; }
/** * Returns a new {@link WebHdfsFileSystem}, with the given configuration. * * @param conf configuration * @return new WebHdfsFileSystem */ private static WebHdfsFileSystem createWebHdfsFileSystem(Configuration conf) { WebHdfsFileSystem fs = new WebHdfsFileSystem(); fs.setConf(conf); return fs; } }
/** * Resolve an HDFS URL into real INetSocketAddress. It works like a DNS * resolver when the URL points to an non-HA cluster. When the URL points to * an HA cluster with its logical name, the resolver further resolves the * logical name(i.e., the authority in the URL) into real namenode addresses. */ private InetSocketAddress[] resolveNNAddr() throws IOException { Configuration conf = getConf(); final String scheme = uri.getScheme(); ArrayList<InetSocketAddress> ret = new ArrayList<InetSocketAddress>(); if (!HAUtil.isLogicalUri(conf, uri)) { InetSocketAddress addr = NetUtils.createSocketAddr(uri.getAuthority(), getDefaultPort()); ret.add(addr); } else { Map<String, Map<String, InetSocketAddress>> addresses = DFSUtil .getHaNnWebHdfsAddresses(conf, scheme); // Extract the entry corresponding to the logical name. Map<String, InetSocketAddress> addrs = addresses.get(uri.getHost()); for (InetSocketAddress addr : addrs.values()) { ret.add(addr); } } InetSocketAddress[] r = new InetSocketAddress[ret.size()]; return ret.toArray(r); }
final Map<?, ?> m; try { m = jsonParse(conn, true); } catch(Exception e) { throw new IOException("Unexpected HTTP response: code=" + code + " != " re = ((RemoteException)re).unwrapRemoteException(InvalidToken.class); throw unwrapException? toIOException(re): re;
/** * Expect read timeout on a URL that requires auth, because the bogus server * never sends a reply. */ @Test(timeout=TEST_TIMEOUT) public void testAuthUrlReadTimeout() throws Exception { try { fs.getDelegationToken("renewer"); fail("expected timeout"); } catch (SocketTimeoutException e) { GenericTestUtils.assertExceptionContains( fs.getUri().getAuthority() + ": Read timed out", e); } }
Path responsePath = webhdfs.getHomeDirectory(); WebHdfsTestUtil.LOG.info("responsePath=" + responsePath); Assert.assertEquals(webhdfs.getUri() + "/user/" + PROXY_USER, responsePath.toString()); FSDataOutputStream out = webhdfs.create(f); out.write("Hello, webhdfs user!".getBytes()); out.close(); final FileStatus status = webhdfs.getFileStatus(f); WebHdfsTestUtil.LOG.info("status.getOwner()=" + status.getOwner()); Assert.assertEquals(PROXY_USER, status.getOwner()); final FSDataOutputStream out = webhdfs.append(f); out.write("\nHello again!".getBytes()); out.close(); final FileStatus status = webhdfs.getFileStatus(f); WebHdfsTestUtil.LOG.info("status.getOwner()=" + status.getOwner()); WebHdfsTestUtil.LOG.info("status.getLen() =" + status.getLen());
@Override public Path getHomeDirectory() { return makeQualified(new Path(getHomeDirectoryString(ugi))); }
@Test public void testSecureHAToken() throws IOException, InterruptedException { Configuration conf = DFSTestUtil.newHAConfiguration(LOGICAL_NAME); conf.setBoolean(DFSConfigKeys .DFS_NAMENODE_DELEGATION_TOKEN_ALWAYS_USE_KEY, true); MiniDFSCluster cluster = null; WebHdfsFileSystem fs = null; try { cluster = new MiniDFSCluster.Builder(conf).nnTopology(topo) .numDataNodes(0).build(); HATestUtil.setFailoverConfigurations(cluster, conf, LOGICAL_NAME); cluster.waitActive(); fs = spy((WebHdfsFileSystem) FileSystem.get(WEBHDFS_URI, conf)); FileSystemTestHelper.addFileSystemForTesting(WEBHDFS_URI, conf, fs); cluster.transitionToActive(0); Token<?> token = fs.getDelegationToken(null); cluster.shutdownNameNode(0); cluster.transitionToActive(1); token.renew(conf); token.cancel(conf); verify(fs).renewDelegationToken(token); verify(fs).cancelDelegationToken(token); } finally { IOUtils.cleanup(null, fs); if (cluster != null) { cluster.shutdown(); } } }
@Override public URI getKeyProviderUri() throws IOException { String keyProviderUri = null; try { keyProviderUri = getServerDefaults().getKeyProviderUri(); } catch (UnsupportedOperationException e) { // This means server doesn't support GETSERVERDEFAULTS call. // Do nothing, let keyProviderUri = null. } catch (RemoteException e) { if (e.getClassName() != null && e.getClassName().equals("java.lang.IllegalArgumentException")) { // See HDFS-13100. // This means server doesn't support GETSERVERDEFAULTS call. // Do nothing, let keyProviderUri = null. } else { throw e; } } return HdfsKMSUtil.getKeyProviderUri(ugi, getUri(), keyProviderUri, getConf()); }
private FileStatus makeQualified(HdfsFileStatus f, Path parent) { return new FileStatus(f.getLen(), f.isDir(), f.getReplication(), f.getBlockSize(), f.getModificationTime(), f.getAccessTime(), f.getPermission(), f.getOwner(), f.getGroup(), f.isSymlink() ? new Path(f.getSymlink()) : null, f.getFullPath(parent).makeQualified(getUri(), getWorkingDirectory())); }
createFile(file); final HttpOpParam.Op op = GetOpParam.Op.GETHOMEDIRECTORY; final URL url = webhdfs.toUrl(op, file); final HttpURLConnection conn = (HttpURLConnection) url.openConnection(); conn.setRequestMethod(op.getType().toString()); WebHdfsFileSystem.jsonParse(spyConn, false); Assert.assertTrue(closedInputStream); } catch(IOException ioe) {