private static void getOtherNamenodesToken(List<String> otherNamenodes, Configuration conf, Credentials cred) throws IOException { LOG.info(OTHER_NAMENODES + ": " + otherNamenodes); Path[] ps = new Path[otherNamenodes.size()]; for (int i = 0; i < ps.length; i++) { ps[i] = new Path(otherNamenodes.get(i).trim()); } TokenCache.obtainTokensForNamenodes(cred, ps, conf); LOG.info("Successfully fetched tokens for: " + otherNamenodes); }
TokenCache.obtainTokensForNamenodes(job.getCredentials(), new Path[] { inputRoot }, srcConf); Configuration destConf = HBaseConfiguration.createClusterConf(conf, null, CONF_DEST_PREFIX); TokenCache.obtainTokensForNamenodes(job.getCredentials(), new Path[] { outputRoot }, destConf);
public static void setTokensFor(ContainerLaunchContext amContainer, List<Path> paths, Configuration conf) throws IOException { Credentials credentials = new Credentials(); // for HDFS TokenCache.obtainTokensForNamenodes(credentials, paths.toArray(new Path[0]), conf); // for HBase obtainTokenForHBase(credentials, conf); // for user UserGroupInformation currUsr = UserGroupInformation.getCurrentUser(); Collection<Token<? extends TokenIdentifier>> usrTok = currUsr.getTokens(); for (Token<? extends TokenIdentifier> token : usrTok) { final Text id = new Text(token.getIdentifier()); LOG.info("Adding user token " + id + " with " + token); credentials.addToken(id, token); } try (DataOutputBuffer dob = new DataOutputBuffer()) { credentials.writeTokenStorageToStream(dob); if (LOG.isDebugEnabled()) { LOG.debug("Wrote tokens. Credentials buffer length: " + dob.getLength()); } ByteBuffer securityTokens = ByteBuffer.wrap(dob.getData(), 0, dob.getLength()); amContainer.setTokens(securityTokens); } }
@Override public Void run() throws Exception { TokenCache.obtainTokensForNamenodes(credentials, paths, config); return null; } }
@Override public Void run() throws Exception { TokenCache.obtainTokensForNamenodes(credentials, paths, config); return null; } }
private static void getOtherNamenodesToken(List<String> otherNamenodes, Configuration conf, Credentials cred) throws IOException { LOG.info(OTHER_NAMENODES + ": " + otherNamenodes); Path[] ps = new Path[otherNamenodes.size()]; for (int i = 0; i < ps.length; i++) { ps[i] = new Path(otherNamenodes.get(i).trim()); } TokenCache.obtainTokensForNamenodes(cred, ps, conf); LOG.info("Successfully fetched tokens for: " + otherNamenodes); }
private static void getOtherNamenodesToken(List<String> otherNamenodes, Configuration conf, Credentials cred) throws IOException { LOG.info(OTHER_NAMENODES + ": " + otherNamenodes); Path[] ps = new Path[otherNamenodes.size()]; for (int i = 0; i < ps.length; i++) { ps[i] = new Path(otherNamenodes.get(i).trim()); } TokenCache.obtainTokensForNamenodes(cred, ps, conf); LOG.info("Successfully fetched tokens for: " + otherNamenodes); }
private void populateTokenCache(Configuration conf, Credentials credentials) throws IOException{ readTokensFromFiles(conf, credentials); // add the delegation tokens from configuration String [] nameNodes = conf.getStrings(MRJobConfig.JOB_NAMENODES); LOG.debug("adding the following namenodes' delegation tokens:" + Arrays.toString(nameNodes)); if(nameNodes != null) { Path [] ps = new Path[nameNodes.length]; for(int i=0; i< nameNodes.length; i++) { ps[i] = new Path(nameNodes[i]); } TokenCache.obtainTokensForNamenodes(credentials, ps, conf); } }
private void populateTokenCache(Configuration conf, Credentials credentials) throws IOException{ readTokensFromFiles(conf, credentials); // add the delegation tokens from configuration String [] nameNodes = conf.getStrings(MRJobConfig.JOB_NAMENODES); LOG.debug("adding the following namenodes' delegation tokens:" + Arrays.toString(nameNodes)); if(nameNodes != null) { Path [] ps = new Path[nameNodes.length]; for(int i=0; i< nameNodes.length; i++) { ps[i] = new Path(nameNodes[i]); } TokenCache.obtainTokensForNamenodes(credentials, ps, conf); } }
public static void populateTokenCache(Configuration conf, Credentials credentials) throws IOException { readTokensFromFiles(conf, credentials); // add the delegation tokens from configuration String[] nameNodes = conf.getStrings(MRJobConfig.JOB_NAMENODES); LOG.debug("adding the following namenodes' delegation tokens:" + Arrays.toString(nameNodes)); if (nameNodes != null) { Path[] ps = new Path[nameNodes.length]; for (int i = 0; i < nameNodes.length; i++) { ps[i] = new Path(nameNodes[i]); } TokenCache.obtainTokensForNamenodes(credentials, ps, conf); } }
private void populateTokenCache(Configuration conf, Credentials credentials) throws IOException{ readTokensFromFiles(conf, credentials); // add the delegation tokens from configuration String [] nameNodes = conf.getStrings(MRJobConfig.JOB_NAMENODES); LOG.debug("adding the following namenodes' delegation tokens:" + Arrays.toString(nameNodes)); if(nameNodes != null) { Path [] ps = new Path[nameNodes.length]; for(int i=0; i< nameNodes.length; i++) { ps[i] = new Path(nameNodes[i]); } TokenCache.obtainTokensForNamenodes(credentials, ps, conf); } }
private Path prepareEnsureStagingDir( TezConfiguration workingConf ) throws IOException { String stepStagingPath = createStepStagingPath(); workingConf.set( TezConfiguration.TEZ_AM_STAGING_DIR, stepStagingPath ); Path stagingDir = new Path( stepStagingPath ); FileSystem fileSystem = FileSystem.get( workingConf ); stagingDir = fileSystem.makeQualified( stagingDir ); TokenCache.obtainTokensForNamenodes( new Credentials(), new Path[]{stagingDir}, workingConf ); TezClientUtils.ensureStagingDirExists( workingConf, stagingDir ); if( fileSystem.getScheme().startsWith( "file:/" ) ) new File( stagingDir.toUri() ).mkdirs(); return stagingDir; }
/** @inheritDoc */ @Override public void checkOutputSpecs(JobContext context) throws IOException { Configuration conf = context.getConfiguration(); Path workingPath = getCommitDirectory(conf); if (getCommitDirectory(conf) == null) { throw new IllegalStateException("Commit directory not configured"); } // get delegation token for outDir's file system TokenCache.obtainTokensForNamenodes(context.getCredentials(), new Path[] { workingPath }, conf); } }
private Path prepareEnsureStagingDir( TezConfiguration workingConf ) throws IOException { String stepStagingPath = createStepStagingPath(); workingConf.set( TezConfiguration.TEZ_AM_STAGING_DIR, stepStagingPath ); Path stagingDir = new Path( stepStagingPath ); FileSystem fileSystem = FileSystem.get( workingConf ); stagingDir = fileSystem.makeQualified( stagingDir ); TokenCache.obtainTokensForNamenodes( new Credentials(), new Path[]{stagingDir}, workingConf ); TezClientUtils.ensureStagingDirExists( workingConf, stagingDir ); if( fileSystem.getScheme().startsWith( "file:/" ) ) new File( stagingDir.toUri() ).mkdirs(); return stagingDir; }
/** @inheritDoc */ @Override public void checkOutputSpecs(JobContext context) throws IOException { Configuration conf = context.getConfiguration(); Path workingPath = getCommitDirectory(conf); if (getCommitDirectory(conf) == null) { throw new IllegalStateException("Commit directory not configured"); } // get delegation token for outDir's file system TokenCache.obtainTokensForNamenodes(context.getCredentials(), new Path[] { workingPath }, conf); } }
public void checkOutputSpecs(JobContext job ) throws FileAlreadyExistsException, IOException{ // Ensure that the output directory is set and not already there Path outDir = getOutputPath(job); if (outDir == null) { throw new InvalidJobConfException("Output directory not set."); } // get delegation token for outDir's file system TokenCache.obtainTokensForNamenodes(job.getCredentials(), new Path[] { outDir }, job.getConfiguration()); if (outDir.getFileSystem(job.getConfiguration()).exists(outDir)) { throw new FileAlreadyExistsException("Output directory " + outDir + " already exists"); } }
public void checkOutputSpecs(JobContext job ) throws FileAlreadyExistsException, IOException{ // Ensure that the output directory is set and not already there Path outDir = getOutputPath(job); if (outDir == null) { throw new InvalidJobConfException("Output directory not set."); } // get delegation token for outDir's file system TokenCache.obtainTokensForNamenodes(job.getCredentials(), new Path[] { outDir }, job.getConfiguration()); if (outDir.getFileSystem(job.getConfiguration()).exists(outDir)) { throw new FileAlreadyExistsException("Output directory " + outDir + " already exists"); } }
public void checkOutputSpecs(JobContext job ) throws FileAlreadyExistsException, IOException{ // Ensure that the output directory is set and not already there Path outDir = getOutputPath(job); if (outDir == null) { throw new InvalidJobConfException("Output directory not set."); } // get delegation token for outDir's file system TokenCache.obtainTokensForNamenodes(job.getCredentials(), new Path[] { outDir }, job.getConfiguration()); if (outDir.getFileSystem(job.getConfiguration()).exists(outDir)) { throw new FileAlreadyExistsException("Output directory " + outDir + " already exists"); } }
public void checkOutputSpecs(JobContext job ) throws FileAlreadyExistsException, IOException{ // Ensure that the output directory is set and not already there Path outDir = getOutputPath(job); if (outDir == null) { throw new InvalidJobConfException("Output directory not set."); } // get delegation token for outDir's file system TokenCache.obtainTokensForNamenodes(job.getCredentials(), new Path[] { outDir }, job.getConfiguration()); if (outDir.getFileSystem(job.getConfiguration()).exists(outDir)) { throw new FileAlreadyExistsException("Output directory " + outDir + " already exists"); } }
public void checkOutputSpecs(JobContext job) throws IOException { // Ensure that the output directory is set and not already there Path outDir = getOutputPath(job); if (outDir == null) { throw new InvalidJobConfException("Output directory not set."); } // get delegation token for outDir's file system TokenCache.obtainTokensForNamenodes( job.getCredentials(), new Path[]{outDir}, job.getConfiguration() ); }