@VisibleForTesting void addDelegationTokens(Collection<Token<? extends TokenIdentifier>> tokens) throws IOException { for (Token<? extends TokenIdentifier> token : tokens) { if (!UserGroupInformation.getCurrentUser().addToken(token)) { LOGGER.error(String.format("Failed to add token %s to user %s", token.toString(), UserGroupInformation.getLoginUser().getShortUserName())); } } } }
private static void getHdfsToken(Configuration conf, Credentials cred) throws IOException { FileSystem fs = FileSystem.get(conf); LOG.info("Getting DFS token from " + fs.getUri()); Token<?> fsToken = fs.getDelegationToken(getMRTokenRenewerInternal(new JobConf()).toString()); if (fsToken == null) { LOG.error("Failed to fetch DFS token for "); throw new IOException("Failed to fetch DFS token."); } LOG.info("Created DFS token: " + fsToken.toString()); LOG.info("Token kind: " + fsToken.getKind()); LOG.info("Token id: " + Arrays.toString(fsToken.getIdentifier())); LOG.info("Token service: " + fsToken.getService()); cred.addToken(fsToken.getService(), fsToken); }
private static void getJtToken(Credentials cred) throws IOException { try { JobConf jobConf = new JobConf(); JobClient jobClient = new JobClient(jobConf); LOG.info("Pre-fetching JT token from JobTracker"); Token<DelegationTokenIdentifier> mrdt = jobClient.getDelegationToken(getMRTokenRenewerInternal(jobConf)); if (mrdt == null) { LOG.error("Failed to fetch JT token"); throw new IOException("Failed to fetch JT token."); } LOG.info("Created JT token: " + mrdt.toString()); LOG.info("Token kind: " + mrdt.getKind()); LOG.info("Token id: " + Arrays.toString(mrdt.getIdentifier())); LOG.info("Token service: " + mrdt.getService()); cred.addToken(mrdt.getService(), mrdt); } catch (InterruptedException ie) { throw new IOException(ie); } }
boolean foundHBaseAuthToken = false; for (Token<? extends TokenIdentifier> token : ugi.getTokens()) { LOG.debug("Token in UGI (delegation token): {} / {}", token.toString(), token.decodeIdentifier().getUser());
tokenForLog.toString(), tokenForLog.decodeIdentifier().getUser());
private static void getJhToken(Configuration conf, Credentials cred) throws IOException { YarnRPC rpc = YarnRPC.create(conf); final String serviceAddr = conf.get(JHAdminConfig.MR_HISTORY_ADDRESS); LOG.debug("Connecting to HistoryServer at: " + serviceAddr); HSClientProtocol hsProxy = (HSClientProtocol) rpc.getProxy(HSClientProtocol.class, NetUtils.createSocketAddr(serviceAddr), conf); LOG.info("Pre-fetching JH token from job history server"); Token<?> jhToken = null; try { jhToken = getDelegationTokenFromHS(hsProxy, conf); } catch (Exception exc) { throw new IOException("Failed to fetch JH token.", exc); } if (jhToken == null) { LOG.error("getDelegationTokenFromHS() returned null"); throw new IOException("Unable to fetch JH token."); } LOG.info("Created JH token: " + jhToken.toString()); LOG.info("Token kind: " + jhToken.getKind()); LOG.info("Token id: " + Arrays.toString(jhToken.getIdentifier())); LOG.info("Token service: " + jhToken.getService()); cred.addToken(jhToken.getService(), jhToken); }
TOKEN : { < #DIGIT : [ "0"-"9" ] > | < #ALPHABET: ["a" - "z"] > | < #CAPSALPHABET: ["A" - "Z"] > | < WORD: ( <DIGIT> | <ALPHABET> | <CAPSALPHABET>)+ > } String comment() : { Token token; } { token=( <WORD> )+ { return token.toString(); } }
@VisibleForTesting void addDelegationTokens(Collection<Token<? extends TokenIdentifier>> tokens) throws IOException { for (Token<? extends TokenIdentifier> token : tokens) { if (!UserGroupInformation.getCurrentUser().addToken(token)) { LOGGER.error(String.format("Failed to add token %s to user %s", token.toString(), UserGroupInformation.getLoginUser().getShortUserName())); } } } }
private static void getHdfsToken(Configuration conf, Credentials cred) throws IOException { FileSystem fs = FileSystem.get(conf); LOG.info("Getting DFS token from " + fs.getUri()); Token<?> fsToken = fs.getDelegationToken(getMRTokenRenewerInternal(new JobConf()).toString()); if (fsToken == null) { LOG.error("Failed to fetch DFS token for "); throw new IOException("Failed to fetch DFS token."); } LOG.info("Created DFS token: " + fsToken.toString()); LOG.info("Token kind: " + fsToken.getKind()); LOG.info("Token id: " + Arrays.toString(fsToken.getIdentifier())); LOG.info("Token service: " + fsToken.getService()); cred.addToken(fsToken.getService(), fsToken); }
private static void getHdfsToken(Configuration conf, Credentials cred) throws IOException { FileSystem fs = FileSystem.get(conf); LOG.info("Getting DFS token from " + fs.getUri()); Token<?> fsToken = fs.getDelegationToken(getMRTokenRenewerInternal(new JobConf()).toString()); if (fsToken == null) { LOG.error("Failed to fetch DFS token for "); throw new IOException("Failed to fetch DFS token."); } LOG.info("Created DFS token: " + fsToken.toString()); LOG.info("Token kind: " + fsToken.getKind()); LOG.info("Token id: " + Arrays.toString(fsToken.getIdentifier())); LOG.info("Token service: " + fsToken.getService()); cred.addToken(fsToken.getService(), fsToken); }
/** * Create a string for people to look at * @param token token to convert to a string form * @return a printable view of the token */ public static String tokenToString(Token<? extends TokenIdentifier> token) { DateFormat df = DateFormat.getDateTimeInstance( DateFormat.SHORT, DateFormat.SHORT); StringBuilder buffer = new StringBuilder(128); buffer.append(token.toString()); try { TokenIdentifier ti = token.decodeIdentifier(); buffer.append("; ").append(ti); if (ti instanceof AbstractDelegationTokenIdentifier) { // details in human readable form, and compensate for information HDFS DT omits AbstractDelegationTokenIdentifier dt = (AbstractDelegationTokenIdentifier) ti; buffer.append("; Renewer: ").append(dt.getRenewer()); buffer.append("; Issued: ") .append(df.format(new Date(dt.getIssueDate()))); buffer.append("; Max Date: ") .append(df.format(new Date(dt.getMaxDate()))); } } catch (IOException e) { //marshall problem; not ours LOG.debug("Failed to decode {}: {}", token, e, e); } return buffer.toString(); }
/** * Create a string for people to look at * @param token token to convert to a string form * @return a printable view of the token */ public static String tokenToString(Token<? extends TokenIdentifier> token) { DateFormat df = DateFormat.getDateTimeInstance( DateFormat.SHORT, DateFormat.SHORT); StringBuilder buffer = new StringBuilder(128); buffer.append(token.toString()); try { TokenIdentifier ti = token.decodeIdentifier(); buffer.append("; ").append(ti); if (ti instanceof AbstractDelegationTokenIdentifier) { // details in human readable form, and compensate for information HDFS DT omits AbstractDelegationTokenIdentifier dt = (AbstractDelegationTokenIdentifier) ti; buffer.append("; Renewer: ").append(dt.getRenewer()); buffer.append("; Issued: ") .append(df.format(new Date(dt.getIssueDate()))); buffer.append("; Max Date: ") .append(df.format(new Date(dt.getMaxDate()))); } } catch (IOException e) { //marshall problem; not ours LOG.debug("Failed to decode {}: {}", token, e, e); } return buffer.toString(); }
private static void getJtToken(Credentials cred) throws IOException { try { JobConf jobConf = new JobConf(); JobClient jobClient = new JobClient(jobConf); LOG.info("Pre-fetching JT token from JobTracker"); Token<DelegationTokenIdentifier> mrdt = jobClient.getDelegationToken(getMRTokenRenewerInternal(jobConf)); if (mrdt == null) { LOG.error("Failed to fetch JT token"); throw new IOException("Failed to fetch JT token."); } LOG.info("Created JT token: " + mrdt.toString()); LOG.info("Token kind: " + mrdt.getKind()); LOG.info("Token id: " + Arrays.toString(mrdt.getIdentifier())); LOG.info("Token service: " + mrdt.getService()); cred.addToken(mrdt.getService(), mrdt); } catch (InterruptedException ie) { throw new IOException(ie); } }
private static void getJtToken(Credentials cred) throws IOException { try { JobConf jobConf = new JobConf(); JobClient jobClient = new JobClient(jobConf); LOG.info("Pre-fetching JT token from JobTracker"); Token<DelegationTokenIdentifier> mrdt = jobClient.getDelegationToken(getMRTokenRenewerInternal(jobConf)); if (mrdt == null) { LOG.error("Failed to fetch JT token"); throw new IOException("Failed to fetch JT token."); } LOG.info("Created JT token: " + mrdt.toString()); LOG.info("Token kind: " + mrdt.getKind()); LOG.info("Token id: " + Arrays.toString(mrdt.getIdentifier())); LOG.info("Token service: " + mrdt.getService()); cred.addToken(mrdt.getService(), mrdt); } catch (InterruptedException ie) { throw new IOException(ie); } }
/** * Convert the Token ByteBuffer to the appropriate Credentials object. * * @param tokens the Tokens from the ContainerLaunchContext. * @return the Credentials object populated from the Tokens. */ public static Credentials getCredentialsFromTokensByteBuffer( ByteBuffer tokens) throws IOException { Credentials credentials = new Credentials(); DataInputByteBuffer dibb = new DataInputByteBuffer(); tokens.rewind(); dibb.reset(tokens); credentials.readTokenStorageStream(dibb); tokens.rewind(); if (LOG.isDebugEnabled()) { for (Token token : credentials.getAllTokens()) { LOG.debug("Token read from token storage: " + token.toString()); } } return credentials; }
private static void getJhToken(Configuration conf, Credentials cred) throws IOException { YarnRPC rpc = YarnRPC.create(conf); final String serviceAddr = conf.get(JHAdminConfig.MR_HISTORY_ADDRESS); LOG.debug("Connecting to HistoryServer at: " + serviceAddr); HSClientProtocol hsProxy = (HSClientProtocol) rpc.getProxy(HSClientProtocol.class, NetUtils.createSocketAddr(serviceAddr), conf); LOG.info("Pre-fetching JH token from job history server"); Token<?> jhToken = null; try { jhToken = getDelegationTokenFromHS(hsProxy, conf); } catch (Exception exc) { throw new IOException("Failed to fetch JH token.", exc); } if (jhToken == null) { LOG.error("getDelegationTokenFromHS() returned null"); throw new IOException("Unable to fetch JH token."); } LOG.info("Created JH token: " + jhToken.toString()); LOG.info("Token kind: " + jhToken.getKind()); LOG.info("Token id: " + Arrays.toString(jhToken.getIdentifier())); LOG.info("Token service: " + jhToken.getService()); cred.addToken(jhToken.getService(), jhToken); }
private static void getJhToken(Configuration conf, Credentials cred) throws IOException { YarnRPC rpc = YarnRPC.create(conf); final String serviceAddr = conf.get(JHAdminConfig.MR_HISTORY_ADDRESS); LOG.debug("Connecting to HistoryServer at: " + serviceAddr); HSClientProtocol hsProxy = (HSClientProtocol) rpc.getProxy(HSClientProtocol.class, NetUtils.createSocketAddr(serviceAddr), conf); LOG.info("Pre-fetching JH token from job history server"); Token<?> jhToken = null; try { jhToken = getDelegationTokenFromHS(hsProxy, conf); } catch (Exception exc) { throw new IOException("Failed to fetch JH token.", exc); } if (jhToken == null) { LOG.error("getDelegationTokenFromHS() returned null"); throw new IOException("Unable to fetch JH token."); } LOG.info("Created JH token: " + jhToken.toString()); LOG.info("Token kind: " + jhToken.getKind()); LOG.info("Token id: " + Arrays.toString(jhToken.getIdentifier())); LOG.info("Token service: " + jhToken.getService()); cred.addToken(jhToken.getService(), jhToken); }
private Credentials parseCredentials(ContainerLaunchContext launchContext) throws IOException { Credentials credentials = new Credentials(); // //////////// Parse credentials ByteBuffer tokens = launchContext.getTokens(); if (tokens != null) { DataInputByteBuffer buf = new DataInputByteBuffer(); tokens.rewind(); buf.reset(tokens); credentials.readTokenStorageStream(buf); if (LOG.isDebugEnabled()) { for (Token<? extends TokenIdentifier> tk : credentials.getAllTokens()) { LOG.debug(tk.getService() + " = " + tk.toString()); } } } // //////////// End of parsing credentials return credentials; }
private Credentials parseCredentials(ContainerLaunchContext launchContext) throws IOException { Credentials credentials = new Credentials(); // //////////// Parse credentials ByteBuffer tokens = launchContext.getTokens(); if (tokens != null) { DataInputByteBuffer buf = new DataInputByteBuffer(); tokens.rewind(); buf.reset(tokens); credentials.readTokenStorageStream(buf); if (LOG.isDebugEnabled()) { for (Token<? extends TokenIdentifier> tk : credentials.getAllTokens()) { LOG.debug(tk.getService() + " = " + tk.toString()); } } } // //////////// End of parsing credentials return credentials; }
/** * Parses the container launch context and returns a Credential instance that * contains all the tokens from the launch context. * @param launchContext * @return the credential instance * @throws IOException */ public static Credentials parseCredentials( ContainerLaunchContext launchContext) throws IOException { Credentials credentials = new Credentials(); ByteBuffer tokens = launchContext.getTokens(); if (tokens != null) { DataInputByteBuffer buf = new DataInputByteBuffer(); tokens.rewind(); buf.reset(tokens); credentials.readTokenStorageStream(buf); if (LOG.isDebugEnabled()) { for (Token<? extends TokenIdentifier> tk : credentials .getAllTokens()) { LOG.debug(tk.getService() + " = " + tk.toString()); } } } return credentials; } }