@Test public void testObtainTokens() throws Exception { Credentials credentials = new Credentials(); FileSystem fs = mock(FileSystem.class); TokenCache.obtainTokensForNamenodesInternal(fs, credentials, conf, renewer); verify(fs).addDelegationTokens(eq(renewer), eq(credentials)); }
@Test public void testSingleTokenFetch() throws Exception { Configuration conf = new Configuration(); conf.set(YarnConfiguration.RM_PRINCIPAL, "mapred/host@REALM"); String renewer = Master.getMasterPrincipal(conf); Credentials credentials = new Credentials(); final MockFileSystem fs = new MockFileSystem(); final MockFileSystem mockFs = (MockFileSystem) fs.getRawFileSystem(); when(mockFs.getCanonicalServiceName()).thenReturn("host:0"); when(mockFs.getUri()).thenReturn(new URI("mockfs://host:0")); Path mockPath = mock(Path.class); when(mockPath.getFileSystem(conf)).thenReturn(mockFs); Path[] paths = new Path[]{ mockPath, mockPath }; when(mockFs.addDelegationTokens("me", credentials)).thenReturn(null); TokenCache.obtainTokensForNamenodesInternal(credentials, paths, conf); verify(mockFs, times(1)).addDelegationTokens(renewer, credentials); }
@SuppressWarnings("deprecation") @Test public void testGetTokensForNamenodes() throws IOException, URISyntaxException { Path TEST_ROOT_DIR = new Path(System.getProperty("test.build.data", "test/build/data")); // ick, but need fq path minus file:/ String binaryTokenFile = FileSystem.getLocal(conf) .makeQualified(new Path(TEST_ROOT_DIR, "tokenFile")).toUri() .getPath(); MockFileSystem fs1 = createFileSystemForServiceName("service1"); Credentials creds = new Credentials(); Token<?> token1 = fs1.getDelegationToken(renewer); creds.addToken(token1.getService(), token1); // wait to set, else the obtain tokens call above will fail with FNF conf.set(MRJobConfig.MAPREDUCE_JOB_CREDENTIALS_BINARY, binaryTokenFile); creds.writeTokenStorageFile(new Path(binaryTokenFile), conf); TokenCache.obtainTokensForNamenodesInternal(fs1, creds, conf, renewer); String fs_addr = fs1.getCanonicalServiceName(); Token<?> nnt = TokenCache.getDelegationToken(creds, fs_addr); assertNotNull("Token for nn is null", nnt); } }
TokenCache.obtainTokensForNamenodesInternal(fs1, creds, conf, renewer); checkToken(creds, newerToken1, token2); TokenCache.obtainTokensForNamenodesInternal(fs2, creds, conf, renewer); checkToken(creds, newerToken1, token2); TokenCache.obtainTokensForNamenodesInternal(fs3, creds, conf, renewer); Token<?> token3 = creds.getToken(new Text(fs3.getCanonicalServiceName())); assertTrue(token3 != null); TokenCache.obtainTokensForNamenodesInternal(fs1, creds, conf, renewer); TokenCache.obtainTokensForNamenodesInternal(fs2, creds, conf, renewer); TokenCache.obtainTokensForNamenodesInternal(fs3, creds, conf, renewer); checkToken(creds, newerToken1, token2, token3);
static void obtainTokensForNamenodesInternal(Credentials credentials, Path[] ps, Configuration conf) throws IOException { for(Path p: ps) { FileSystem fs = FileSystem.get(p.toUri(), conf); obtainTokensForNamenodesInternal(fs, credentials, conf); } }
static void obtainTokensForNamenodesInternal(Credentials credentials, Path[] ps, Configuration conf) throws IOException { Set<FileSystem> fsSet = new HashSet<FileSystem>(); for(Path p: ps) { fsSet.add(p.getFileSystem(conf)); } for (FileSystem fs : fsSet) { obtainTokensForNamenodesInternal(fs, credentials, conf); } }
static void obtainTokensForNamenodesInternal(Credentials credentials, Path[] ps, Configuration conf) throws IOException { Set<FileSystem> fsSet = new HashSet<FileSystem>(); for(Path p: ps) { fsSet.add(p.getFileSystem(conf)); } for (FileSystem fs : fsSet) { obtainTokensForNamenodesInternal(fs, credentials, conf); } }
static void obtainTokensForNamenodesInternal(Credentials credentials, Path[] ps, Configuration conf) throws IOException { Set<FileSystem> fsSet = new HashSet<FileSystem>(); for(Path p: ps) { fsSet.add(p.getFileSystem(conf)); } for (FileSystem fs : fsSet) { obtainTokensForNamenodesInternal(fs, credentials, conf); } }
static void obtainTokensForNamenodesInternal(Credentials credentials, Path[] ps, Configuration conf) throws IOException { Set<FileSystem> fsSet = new HashSet<FileSystem>(); for(Path p: ps) { fsSet.add(p.getFileSystem(conf)); } for (FileSystem fs : fsSet) { obtainTokensForNamenodesInternal(fs, credentials, conf); } }
/** * Convenience method to obtain delegation tokens from namenodes * corresponding to the paths passed. * @param credentials * @param ps array of paths * @param conf configuration * @throws IOException */ public static void obtainTokensForNamenodes(Credentials credentials, Path[] ps, Configuration conf) throws IOException { if (!UserGroupInformation.isSecurityEnabled()) { return; } obtainTokensForNamenodesInternal(credentials, ps, conf); }
/** * Convenience method to obtain delegation tokens from namenodes * corresponding to the paths passed. * @param credentials * @param ps array of paths * @param conf configuration * @throws IOException */ public static void obtainTokensForNamenodes(Credentials credentials, Path[] ps, Configuration conf) throws IOException { if (!UserGroupInformation.isSecurityEnabled()) { return; } obtainTokensForNamenodesInternal(credentials, ps, conf); }
/** * Convenience method to obtain delegation tokens from namenodes * corresponding to the paths passed. * @param credentials * @param ps array of paths * @param conf configuration * @throws IOException */ public static void obtainTokensForNamenodes(Credentials credentials, Path[] ps, Configuration conf) throws IOException { if (!UserGroupInformation.isSecurityEnabled()) { return; } obtainTokensForNamenodesInternal(credentials, ps, conf); }
/** * Convenience method to obtain delegation tokens from namenodes * corresponding to the paths passed. * @param credentials * @param ps array of paths * @param conf configuration * @throws IOException */ public static void obtainTokensForNamenodes(Credentials credentials, Path[] ps, Configuration conf) throws IOException { if (!UserGroupInformation.isSecurityEnabled()) { return; } obtainTokensForNamenodesInternal(credentials, ps, conf); }
/** * Convenience method to obtain delegation tokens from namenodes * corresponding to the paths passed. * @param credentials * @param ps array of paths * @param conf configuration * @throws IOException */ public static void obtainTokensForNamenodes(Credentials credentials, Path[] ps, Configuration conf) throws IOException { if (!UserGroupInformation.isSecurityEnabled()) { return; } obtainTokensForNamenodesInternal(credentials, ps, conf); }
private void populateTokens(Job job) { // Credentials in the job will not have delegation tokens // because security is disabled. Fetch delegation tokens // and populate the credential in the job. try { Credentials ts = job.getCredentials(); Path p1 = new Path("file1"); p1 = p1.getFileSystem(job.getConfiguration()).makeQualified(p1); Credentials cred = new Credentials(); TokenCache.obtainTokensForNamenodesInternal(cred, new Path[] { p1 }, job.getConfiguration()); for (Token<? extends TokenIdentifier> t : cred.getAllTokens()) { ts.addToken(new Text("Hdfs"), t); } } catch (IOException e) { Assert.fail("Exception " + e); } } }
private void setupBinaryTokenFile(Job job) { // Credentials in the job will not have delegation tokens // because security is disabled. Fetch delegation tokens // and store in binary token file. try { Credentials cred1 = new Credentials(); Credentials cred2 = new Credentials(); TokenCache.obtainTokensForNamenodesInternal(cred1, new Path[] { p1 }, job.getConfiguration()); for (Token<? extends TokenIdentifier> t : cred1.getAllTokens()) { cred2.addToken(new Text("Hdfs"), t); } DataOutputStream os = new DataOutputStream(new FileOutputStream( binaryTokenFileName.toString())); cred2.writeTokenStorageToStream(os); os.close(); job.getConfiguration().set("mapreduce.job.credentials.binary", binaryTokenFileName.toString()); } catch (IOException e) { Assert.fail("Exception " + e); } } }
private void populateTokens(JobConf job) { // Credentials in the job will not have delegation tokens // because security is disabled. Fetch delegation tokens // and populate the credential in the job. try { Credentials ts = job.getCredentials(); Path p1 = new Path("file1"); p1 = p1.getFileSystem(job).makeQualified(p1); Credentials cred = new Credentials(); TokenCache.obtainTokensForNamenodesInternal(cred, new Path[] { p1 }, job); for (Token<? extends TokenIdentifier> t : cred.getAllTokens()) { ts.addToken(new Text("Hdfs"), t); } } catch (IOException e) { Assert.fail("Exception " + e); } }
@Test public void testGetTokensForNamenodes() throws IOException { Credentials credentials = new Credentials(); TokenCache.obtainTokensForNamenodesInternal(credentials, new Path[] { p1, p2 }, jConf); // this token is keyed by hostname:port key. String fs_addr = SecurityUtil.buildDTServiceName(p1.toUri(), NameNode.DEFAULT_PORT); Token<DelegationTokenIdentifier> nnt = TokenCache.getDelegationToken( credentials, fs_addr); System.out.println("dt for " + p1 + "(" + fs_addr + ")" + " = " + nnt); assertNotNull("Token for nn is null", nnt); // verify the size Collection<Token<? extends TokenIdentifier>> tns = credentials.getAllTokens(); assertEquals("number of tokens is not 1", 1, tns.size()); boolean found = false; for(Token<? extends TokenIdentifier> t: tns) { if(t.getKind().equals(DelegationTokenIdentifier.HDFS_DELEGATION_KIND) && t.getService().equals(new Text(fs_addr))) { found = true; } assertTrue("didn't find token for " + p1 ,found); } }
Path p = new Path(uri.toString()); System.out.println("Path for hftp="+ p + "; fs_addr="+fs_addr + "; rn=" + renewer); TokenCache.obtainTokensForNamenodesInternal(hfs, credentials, jConf);
private static void createBinaryTokenFile(Configuration conf) { // Fetch delegation tokens and store in binary token file. try { Credentials cred1 = new Credentials(); Credentials cred2 = new Credentials(); TokenCache.obtainTokensForNamenodesInternal(cred1, new Path[] { p1 }, conf); for (Token<? extends TokenIdentifier> t : cred1.getAllTokens()) { cred2.addToken(new Text(DELEGATION_TOKEN_KEY), t); } DataOutputStream os = new DataOutputStream(new FileOutputStream( binaryTokenFileName.toString())); try { cred2.writeTokenStorageToStream(os); } finally { os.close(); } } catch (IOException e) { Assert.fail("Exception " + e); } }