public SecureCmdDoAs(HiveConf conf) throws HiveException, IOException{ // Get delegation token for user from filesystem and write the token along with // metastore tokens into a file String uname = UserGroupInformation.getLoginUser().getShortUserName(); FileSystem fs = FileSystem.get(conf); Credentials cred = new Credentials(); ShimLoader.getHadoopShims().addDelegationTokens(fs, cred, uname); // ask default fs first for (String uri : conf.getStringCollection("mapreduce.job.hdfs-servers")) { try { ShimLoader.getHadoopShims().addDelegationTokens( FileSystem.get(new URI(uri), conf), cred, uname); } catch (URISyntaxException e) { LOG.warn("Invalid URI in mapreduce.job.hdfs-servers:["+uri+"], ignoring.", e); } } tokenFile = File.createTempFile("hive_hadoop_delegation_token", null); tokenPath = new Path(tokenFile.toURI()); //write credential with token to file cred.writeTokenStorageFile(tokenPath, conf); }
private static void getHdfsToken(Configuration conf, Credentials cred) throws IOException { FileSystem fs = FileSystem.get(conf); LOG.info("Getting DFS token from " + fs.getUri()); Token<?> fsToken = fs.getDelegationToken(getMRTokenRenewerInternal(new JobConf()).toString()); if (fsToken == null) { LOG.error("Failed to fetch DFS token for "); throw new IOException("Failed to fetch DFS token."); } LOG.info("Created DFS token: " + fsToken.toString()); LOG.info("Token kind: " + fsToken.getKind()); LOG.info("Token id: " + Arrays.toString(fsToken.getIdentifier())); LOG.info("Token service: " + fsToken.getService()); cred.addToken(fsToken.getService(), fsToken); }
@SuppressWarnings("unchecked") protected byte[] getHadoopCredentials(Map<String, Object> conf, Configuration hbaseConf, final String topologySubmitterUser) { try { if(UserGroupInformation.isSecurityEnabled()) { UserProvider provider = UserProvider.instantiate(hbaseConf); provider.login(HBASE_KEYTAB_FILE_KEY, HBASE_PRINCIPAL_KEY, InetAddress.getLocalHost().getCanonicalHostName()); LOG.info("Logged into Hbase as principal = " + hbaseConf.get(HBASE_PRINCIPAL_KEY)); UserGroupInformation ugi = UserGroupInformation.getCurrentUser(); final UserGroupInformation proxyUser = UserGroupInformation.createProxyUser(topologySubmitterUser, ugi); Credentials credential = proxyUser.getCredentials(); for (Token<? extends TokenIdentifier> tokenForLog : credential.getAllTokens()) { LOG.debug("Obtained token info in credential: {} / {}", tokenForLog.toString(), tokenForLog.decodeIdentifier().getUser()); credential.write(out); out.flush(); out.close();
public void read(ObjectInputStream in) throws IOException { this.credentials = new Credentials(); credentials.readFields(in); } }
/** * Read a collection {@link Token}s from a given file. * * @param tokenFilePath the token file path * @param configuration a {@link Configuration} object carrying Hadoop configuration properties * @return a collection of {@link Token}s * @throws IOException */ public static Collection<Token<? extends TokenIdentifier>> readTokensFromFile(Path tokenFilePath, Configuration configuration) throws IOException { return Credentials.readTokenStorageFile(tokenFilePath, configuration).getAllTokens(); }
public Object run() throws IOException { Credentials cred = new Credentials(); for(Token<?> fsToken : fsTokens) { cred.addToken(fsToken.getService(), fsToken); } cred.addToken(msToken.getService(), msToken); cred.writeTokenStorageFile(tokenPath, conf); return null; } });
FileSystem fs = FileSystem.get(jstormClientContext.conf); addToLocalResources(fs, jstormClientContext.appMasterJar, JOYConstants.appMasterJarPath, appId.toString(), localResources, null); for (String c : jstormClientContext.conf.getStrings( YarnConfiguration.YARN_APPLICATION_CLASSPATH, YarnConfiguration.DEFAULT_YARN_CROSS_PLATFORM_APPLICATION_CLASSPATH)) { if (jstormClientContext.conf.getBoolean(YarnConfiguration.IS_MINI_YARN_CLUSTER, false)) { classPathEnv.append(JOYConstants.COLON); classPathEnv.append(System.getProperty(JOYConstants.JAVA_CLASS_PATH)); if (UserGroupInformation.isSecurityEnabled()) { Credentials credentials = new Credentials(); String tokenRenewer = jstormClientContext.conf.get(YarnConfiguration.RM_PRINCIPAL); if (tokenRenewer == null || tokenRenewer.length() == 0) { throw new IOException( credentials.writeTokenStorageToStream(dob); ByteBuffer fsTokens = ByteBuffer.wrap(dob.getData(), 0, dob.getLength()); amContainer.setTokens(fsTokens);
private void testBinaryCredentials(boolean hasScheme) throws Exception { Path TEST_ROOT_DIR = new Path(System.getProperty("test.build.data","test/build/data")); ? FileSystem.getLocal(conf).makeQualified( new Path(TEST_ROOT_DIR, "tokenFile")).toString() : FileSystem.getLocal(conf).makeQualified( new Path(TEST_ROOT_DIR, "tokenFile")).toUri().getPath(); Credentials creds = new Credentials(); Token<?> token1 = fs1.getDelegationToken(renewer); Token<?> token2 = fs2.getDelegationToken(renewer); creds.addToken(token1.getService(), token1); creds.addToken(token2.getService(), token2); conf.set(MRJobConfig.MAPREDUCE_JOB_CREDENTIALS_BINARY, binaryTokenFile); creds.writeTokenStorageFile(new Path(binaryTokenFile), conf); creds = new Credentials(); Token<?> newerToken1 = fs1.getDelegationToken(renewer); assertNotSame(newerToken1, token1); creds.addToken(newerToken1.getService(), newerToken1); checkToken(creds, newerToken1); Token<?> token3 = creds.getToken(new Text(fs3.getCanonicalServiceName())); assertTrue(token3 != null); checkToken(creds, newerToken1, token2, token3);
throws IOException, ClassNotFoundException { Job job = null; boolean isDryRun = conf.getBoolean(DRY_RUN_CONF_KEY, false); try (Connection connection = ConnectionFactory.createConnection(conf)) { try (Admin admin = connection.getAdmin()) { String actualSeparator = conf.get(SEPARATOR_CONF_KEY); if (actualSeparator != null) { conf.set(SEPARATOR_CONF_KEY, Bytes.toString(Base64.getEncoder().encode(actualSeparator.getBytes()))); Path inputDir = new Path(args[1]); String jobName = conf.get(JOB_NAME_CONF_KEY,NAME + "_" + tableName.getNameAsString()); job = Job.getInstance(conf, jobName); if(StringUtils.isNotEmpty(conf.get(CREDENTIALS_LOCATION))) { String fileLoc = conf.get(CREDENTIALS_LOCATION); Credentials cred = Credentials.readTokenStorageFile(new File(fileLoc), conf); job.getCredentials().addAll(cred); Path outputDir = new Path(hfileOutPath); FileOutputFormat.setOutputPath(job, outputDir); HFileOutputFormat2.configureIncrementalLoad(job, table.getDescriptor(),
@Override public Path createDelegationTokenFile(Configuration conf) throws IOException { //get delegation token for user String uname = UserGroupInformation.getLoginUser().getShortUserName(); FileSystem fs = FileSystem.get(conf); Token<?> fsToken = fs.getDelegationToken(uname); File t = File.createTempFile("hive_hadoop_delegation_token", null); Path tokenPath = new Path(t.toURI()); //write credential with token to file Credentials cred = new Credentials(); cred.addToken(fsToken.getService(), fsToken); cred.writeTokenStorageFile(tokenPath, conf); return tokenPath; }
FileSystem fs = FileSystem.get(conf); createMaaSDirectory(fs, appId.toString()); Path ajPath = addToLocalResources(fs, appMasterJar, appMasterJarPath, appId.toString(), localResources, null); for (String c : conf.getStrings( YarnConfiguration.YARN_APPLICATION_CLASSPATH, YarnConfiguration.DEFAULT_YARN_CROSS_PLATFORM_APPLICATION_CLASSPATH)) { if (conf.getBoolean(YarnConfiguration.IS_MINI_YARN_CLUSTER, false)) { classPathEnv.append(':'); classPathEnv.append(System.getProperty("java.class.path")); ,ApplicationMaster.AMOptions.APP_JAR_PATH.of(ajPath.toString()) if (UserGroupInformation.isSecurityEnabled()) { Credentials credentials = new Credentials(); String tokenRenewer = conf.get(YarnConfiguration.RM_PRINCIPAL); if (tokenRenewer == null || tokenRenewer.length() == 0) { throw new IOException( credentials.writeTokenStorageToStream(dob); ByteBuffer fsTokens = ByteBuffer.wrap(dob.getData(), 0, dob.getLength()); amContainer.setTokens(fsTokens);
@SuppressWarnings("deprecation") @Test public void testGetTokensForNamenodes() throws IOException, URISyntaxException { Path TEST_ROOT_DIR = new Path(System.getProperty("test.build.data", "test/build/data")); // ick, but need fq path minus file:/ String binaryTokenFile = FileSystem.getLocal(conf) .makeQualified(new Path(TEST_ROOT_DIR, "tokenFile")).toUri() .getPath(); MockFileSystem fs1 = createFileSystemForServiceName("service1"); Credentials creds = new Credentials(); Token<?> token1 = fs1.getDelegationToken(renewer); creds.addToken(token1.getService(), token1); // wait to set, else the obtain tokens call above will fail with FNF conf.set(MRJobConfig.MAPREDUCE_JOB_CREDENTIALS_BINARY, binaryTokenFile); creds.writeTokenStorageFile(new Path(binaryTokenFile), conf); TokenCache.obtainTokensForNamenodesInternal(fs1, creds, conf, renewer); String fs_addr = fs1.getCanonicalServiceName(); Token<?> nnt = TokenCache.getDelegationToken(creds, fs_addr); assertNotNull("Token for nn is null", nnt); } }
@Test public void testUserProvider() throws Exception { Configuration conf = new Configuration(); final String ourUrl = UserProvider.SCHEME_NAME + ":///"; conf.set(CredentialProviderFactory.CREDENTIAL_PROVIDER_PATH, ourUrl); checkSpecificProvider(conf, ourUrl); // see if the credentials are actually in the UGI Credentials credentials = UserGroupInformation.getCurrentUser().getCredentials(); assertArrayEquals(new byte[]{'1', '2', '3'}, credentials.getSecretKey(new Text("pass2"))); }
private void populateTokens(JobConf job) { // Credentials in the job will not have delegation tokens // because security is disabled. Fetch delegation tokens // and populate the credential in the job. try { Credentials ts = job.getCredentials(); Path p1 = new Path("file1"); p1 = p1.getFileSystem(job).makeQualified(p1); Credentials cred = new Credentials(); TokenCache.obtainTokensForNamenodesInternal(cred, new Path[] { p1 }, job); for (Token<? extends TokenIdentifier> t : cred.getAllTokens()) { ts.addToken(new Text("Hdfs"), t); } } catch (IOException e) { Assert.fail("Exception " + e); } }
FileSystem fs = FileSystem.get(conf); ignite = getIgnite(props, fs); else ignite = new Path(props.ignitePath()); Collections.singletonList( Environment.JAVA_HOME.$() + "/bin/java -Xmx512m " + ApplicationMaster.class.getName() + IgniteYarnUtils.SPACE + ignite.toUri() + IgniteYarnUtils.YARN_LOG_OUT if (UserGroupInformation.isSecurityEnabled()) { Credentials creds = new Credentials(); final Token<?> tokens[] = fs.addDelegationTokens(tokRenewer, creds);
conf.set(JAR_ARGS_NAME, TempletonUtils.encodeArray(args)); String memoryMb = appConf.mapperMemoryMb(); if(memoryMb != null && memoryMb.length() != 0) { conf.set(AppConfig.HADOOP_MAP_MEMORY_MB, memoryMb); conf.set(AppConfig.HADOOP_MR_AM_MEMORY_MB, amMemoryMB); String user = UserGroupInformation.getCurrentUser().getShortUserName(); conf.set("user.name", user); job = new Job(conf); job.setNumReduceTasks(0); JobClient jc = new JobClient(new JobConf(job.getConfiguration())); if(UserGroupInformation.isSecurityEnabled()) { Token<DelegationTokenIdentifier> mrdt = jc.getDelegationToken(new Text("mr token")); job.getCredentials().addToken(new Text("mr token"), mrdt);
@Test public void testSingleTokenFetch() throws Exception { Configuration conf = new Configuration(); conf.set(YarnConfiguration.RM_PRINCIPAL, "mapred/host@REALM"); String renewer = Master.getMasterPrincipal(conf); Credentials credentials = new Credentials(); final MockFileSystem fs = new MockFileSystem(); final MockFileSystem mockFs = (MockFileSystem) fs.getRawFileSystem(); when(mockFs.getCanonicalServiceName()).thenReturn("host:0"); when(mockFs.getUri()).thenReturn(new URI("mockfs://host:0")); Path mockPath = mock(Path.class); when(mockPath.getFileSystem(conf)).thenReturn(mockFs); Path[] paths = new Path[]{ mockPath, mockPath }; when(mockFs.addDelegationTokens("me", credentials)).thenReturn(null); TokenCache.obtainTokensForNamenodesInternal(credentials, paths, conf); verify(mockFs, times(1)).addDelegationTokens(renewer, credentials); }
this.configuration.set("mapreduce.output.basename", "tmp"); + "_0"); this.configuration.set("mapred.task.id", taskAttemptID.toString()); this.configuration.setInt("mapred.task.partition", taskNumber + 1); this.configuration.set("mapreduce.task.attempt.id", taskAttemptID.toString()); this.context.getCredentials().addAll(this.credentials); Credentials currentUserCreds = getCredentialsFromUGI(UserGroupInformation.getCurrentUser()); if (currentUserCreds != null) { this.context.getCredentials().addAll(currentUserCreds); this.configuration.set("mapreduce.task.output.dir", ((FileOutputCommitter) this.outputCommitter).getWorkPath().toString());
private void setupBinaryTokenFile(Job job) { // Credentials in the job will not have delegation tokens // because security is disabled. Fetch delegation tokens // and store in binary token file. try { Credentials cred1 = new Credentials(); Credentials cred2 = new Credentials(); TokenCache.obtainTokensForNamenodesInternal(cred1, new Path[] { p1 }, job.getConfiguration()); for (Token<? extends TokenIdentifier> t : cred1.getAllTokens()) { cred2.addToken(new Text("Hdfs"), t); } DataOutputStream os = new DataOutputStream(new FileOutputStream( binaryTokenFileName.toString())); cred2.writeTokenStorageToStream(os); os.close(); job.getConfiguration().set("mapreduce.job.credentials.binary", binaryTokenFileName.toString()); } catch (IOException e) { Assert.fail("Exception " + e); } } }
boolean harRequested) throws IOException, MetaException, TException, Exception { if (UserGroupInformation.isSecurityEnabled()) { UserGroupInformation ugi = UserGroupInformation.getCurrentUser(); new Text(), ugi.getTokens()); LOG.info("Security::handleSecurity(): Checking for pre-existing metastore token... " + (hiveToken == null? "Not found. Creating a new one." : "Found. Using existing token.")); credentials.addToken( new Text("hcat jt token"), HCatUtil.getJobTrackerDelegationToken(conf, ugi.getUserName()) ); credentials.addToken(new Text(ugi.getUserName() + "_" + tokenSignature), hiveToken); conf.set(HCatConstants.HCAT_KEY_TOKEN_SIGNATURE, tokenSignature);