/** * Indicates whether the current user has an HDFS delegation token. */ public static boolean hasHDFSDelegationToken() throws Exception { UserGroupInformation loginUser = UserGroupInformation.getCurrentUser(); Collection<Token<? extends TokenIdentifier>> usrTok = loginUser.getTokens(); for (Token<? extends TokenIdentifier> token : usrTok) { if (token.getKind().equals(HDFS_DELEGATION_TOKEN_KIND)) { return true; } } return false; }
@Override public Token<AuthenticationTokenIdentifier> selectToken(Text serviceName, Collection<Token<? extends TokenIdentifier>> tokens) { if (serviceName != null) { for (Token ident : tokens) { if (serviceName.equals(ident.getService()) && AuthenticationTokenIdentifier.AUTH_TOKEN_TYPE.equals(ident.getKind())) { if (LOG.isDebugEnabled()) { LOG.debug("Returning token "+ident); } return (Token<AuthenticationTokenIdentifier>)ident; } } } LOG.debug("No matching token found"); return null; } }
@SuppressWarnings("unchecked") @Override public Token<TokenIdent> selectToken(Text service, Collection<Token<? extends TokenIdentifier>> tokens) { if (service == null) { return null; } for (Token<? extends TokenIdentifier> token : tokens) { if (kindName.equals(token.getKind()) && service.equals(token.getService())) { return (Token<TokenIdent>) token; } } return null; } }
@Override public Token<LlapTokenIdentifier> selectToken(Text service, Collection<Token<? extends TokenIdentifier>> tokens) { if (service == null) return null; if (LOG.isDebugEnabled()) { LOG.debug("Looking for a token with service " + service); } for (Token<? extends TokenIdentifier> token : tokens) { if (LOG.isDebugEnabled()) { LOG.debug("Token = " + token.getKind() + "; service = " + token.getService()); } if (LlapTokenIdentifier.KIND_NAME.equals(token.getKind()) && service.equals(token.getService())) { @SuppressWarnings("unchecked") Token<LlapTokenIdentifier> result = (Token<LlapTokenIdentifier>)token; return result; } } return null; } }
/** * Returns the Token of the specified kind associated with this user, * or null if the Token is not present. * * @param kind the kind of token * @param service service on which the token is supposed to be used * @return the token of the specified kind. */ public Token<?> getToken(String kind, String service) throws IOException { for (Token<?> token : ugi.getTokens()) { if (token.getKind().toString().equals(kind) && (service != null && token.getService().toString().equals(service))) { return token; } } return null; }
protected static Token<?> selectDelegationToken(Credentials creds, Text service) { Token<?> token = creds.getToken(service); LOG.debug("selected by alias={} token={}", service, token); if (token != null && TOKEN_KIND.equals(token.getKind())) { return token; } token = TokenSelector.INSTANCE.selectToken(service, creds.getAllTokens()); LOG.debug("selected by service={} token={}", service, token); return token; }
/** * Get the token identifier object, or null if it could not be constructed * (because the class could not be loaded, for example). * @return the token identifier, or null * @throws IOException */ @SuppressWarnings("unchecked") public T decodeIdentifier() throws IOException { Class<? extends TokenIdentifier> cls = getClassForIdentifier(getKind()); if (cls == null) { return null; } TokenIdentifier tokenIdentifier = ReflectionUtils.newInstance(cls, null); ByteArrayInputStream buf = new ByteArrayInputStream(identifier); DataInputStream in = new DataInputStream(buf); tokenIdentifier.readFields(in); in.close(); return (T) tokenIdentifier; }
private static void getHdfsToken(Configuration conf, Credentials cred) throws IOException { FileSystem fs = FileSystem.get(conf); LOG.info("Getting DFS token from " + fs.getUri()); Token<?> fsToken = fs.getDelegationToken(getMRTokenRenewerInternal(new JobConf()).toString()); if (fsToken == null) { LOG.error("Failed to fetch DFS token for "); throw new IOException("Failed to fetch DFS token."); } LOG.info("Created DFS token: " + fsToken.toString()); LOG.info("Token kind: " + fsToken.getKind()); LOG.info("Token id: " + Arrays.toString(fsToken.getIdentifier())); LOG.info("Token service: " + fsToken.getService()); cred.addToken(fsToken.getService(), fsToken); }
/** * Construct a TokenProto from this Token instance. * @return a new TokenProto object holding copies of data in this instance */ public TokenProto toTokenProto() { return TokenProto.newBuilder(). setIdentifier(ByteString.copyFrom(this.getIdentifier())). setPassword(ByteString.copyFrom(this.getPassword())). setKindBytes(ByteString.copyFrom( this.getKind().getBytes(), 0, this.getKind().getLength())). setServiceBytes(ByteString.copyFrom( this.getService().getBytes(), 0, this.getService().getLength())). build(); }
private static void getJtToken(Credentials cred) throws IOException { try { JobConf jobConf = new JobConf(); JobClient jobClient = new JobClient(jobConf); LOG.info("Pre-fetching JT token from JobTracker"); Token<DelegationTokenIdentifier> mrdt = jobClient.getDelegationToken(getMRTokenRenewerInternal(jobConf)); if (mrdt == null) { LOG.error("Failed to fetch JT token"); throw new IOException("Failed to fetch JT token."); } LOG.info("Created JT token: " + mrdt.toString()); LOG.info("Token kind: " + mrdt.getKind()); LOG.info("Token id: " + Arrays.toString(mrdt.getIdentifier())); LOG.info("Token service: " + mrdt.getService()); cred.addToken(mrdt.getService(), mrdt); } catch (InterruptedException ie) { throw new IOException(ie); } }
private ByteBuffer getSecurityTokens() throws IOException { Credentials credentials = UserGroupInformation.getCurrentUser().getCredentials(); Closer closer = Closer.create(); try { DataOutputBuffer dataOutputBuffer = closer.register(new DataOutputBuffer()); credentials.writeTokenStorageToStream(dataOutputBuffer); // Remove the AM->RM token so that containers cannot access it Iterator<Token<?>> tokenIterator = credentials.getAllTokens().iterator(); while (tokenIterator.hasNext()) { Token<?> token = tokenIterator.next(); if (token.getKind().equals(AMRMTokenIdentifier.KIND_NAME)) { tokenIterator.remove(); } } return ByteBuffer.wrap(dataOutputBuffer.getData(), 0, dataOutputBuffer.getLength()); } catch (Throwable t) { throw closer.rethrow(t); } finally { closer.close(); } }
if (token.getKind().toString().equals(TOKEN_KIND_HBASE_AUTH_TOKEN)) {
private static void getJhToken(Configuration conf, Credentials cred) throws IOException { YarnRPC rpc = YarnRPC.create(conf); final String serviceAddr = conf.get(JHAdminConfig.MR_HISTORY_ADDRESS); LOG.debug("Connecting to HistoryServer at: " + serviceAddr); HSClientProtocol hsProxy = (HSClientProtocol) rpc.getProxy(HSClientProtocol.class, NetUtils.createSocketAddr(serviceAddr), conf); LOG.info("Pre-fetching JH token from job history server"); Token<?> jhToken = null; try { jhToken = getDelegationTokenFromHS(hsProxy, conf); } catch (Exception exc) { throw new IOException("Failed to fetch JH token.", exc); } if (jhToken == null) { LOG.error("getDelegationTokenFromHS() returned null"); throw new IOException("Unable to fetch JH token."); } LOG.info("Created JH token: " + jhToken.toString()); LOG.info("Token kind: " + jhToken.getKind()); LOG.info("Token id: " + Arrays.toString(jhToken.getIdentifier())); LOG.info("Token service: " + jhToken.getService()); cred.addToken(jhToken.getService(), jhToken); }
LOG.debug("Obtained token " + token.getKind().toString() + " for user " + user.getName() + " on cluster " + clusterId.toString());
/** * Dump all tokens of a UGI. * @param ugi UGI to examine */ public void dumpTokens(UserGroupInformation ugi) { Collection<Token<? extends TokenIdentifier>> tokens = ugi.getCredentials().getAllTokens(); title("Token Count: %d", tokens.size()); for (Token<? extends TokenIdentifier> token : tokens) { println("Token %s", token.getKind()); } endln(); }
public static void cancelTokens(State state) throws IOException, InterruptedException, TException { Preconditions.checkArgument(state.contains(ConfigurationKeys.SUPER_USER_KEY_TAB_LOCATION), "Missing required property " + ConfigurationKeys.SUPER_USER_KEY_TAB_LOCATION); Preconditions.checkArgument(state.contains(ComplianceConfigurationKeys.GOBBLIN_COMPLIANCE_SUPER_USER), "Missing required property " + ComplianceConfigurationKeys.GOBBLIN_COMPLIANCE_SUPER_USER); Preconditions.checkArgument(state.contains(ConfigurationKeys.KERBEROS_REALM), "Missing required property " + ConfigurationKeys.KERBEROS_REALM); String superUser = state.getProp(ComplianceConfigurationKeys.GOBBLIN_COMPLIANCE_SUPER_USER); String keytabLocation = state.getProp(ConfigurationKeys.SUPER_USER_KEY_TAB_LOCATION); String realm = state.getProp(ConfigurationKeys.KERBEROS_REALM); UserGroupInformation.loginUserFromKeytab(HostUtils.getPrincipalUsingHostname(superUser, realm), keytabLocation); UserGroupInformation currentUser = UserGroupInformation.getCurrentUser(); UserGroupInformation realUser = currentUser.getRealUser(); Credentials credentials = realUser.getCredentials(); for (Token<?> token : credentials.getAllTokens()) { if (token.getKind().equals(DelegationTokenIdentifier.HIVE_DELEGATION_KIND)) { log.info("Cancelling hive token"); HiveMetaStoreClient hiveClient = new HiveMetaStoreClient(new HiveConf()); hiveClient.cancelDelegationToken(token.encodeToUrlString()); } } }
private static ExportProtos.ExportRequest getConfiguredRequest(Configuration conf, Path dir, final Scan scan, final Token<?> userToken) throws IOException { boolean compressed = conf.getBoolean(FileOutputFormat.COMPRESS, false); String compressionType = conf.get(FileOutputFormat.COMPRESS_TYPE, DEFAULT_TYPE.toString()); String compressionCodec = conf.get(FileOutputFormat.COMPRESS_CODEC, DEFAULT_CODEC.getName()); DelegationToken protoToken = null; if (userToken != null) { protoToken = DelegationToken.newBuilder() .setIdentifier(ByteStringer.wrap(userToken.getIdentifier())) .setPassword(ByteStringer.wrap(userToken.getPassword())) .setKind(userToken.getKind().toString()) .setService(userToken.getService().toString()).build(); } LOG.info("compressed=" + compressed + ", compression type=" + compressionType + ", compression codec=" + compressionCodec + ", userToken=" + userToken); ExportProtos.ExportRequest.Builder builder = ExportProtos.ExportRequest.newBuilder() .setScan(ProtobufUtil.toScan(scan)) .setOutputPath(dir.toString()) .setCompressed(compressed) .setCompressCodec(compressionCodec) .setCompressType(compressionType); if (protoToken != null) { builder.setFsToken(protoToken); } return builder.build(); }
/** * Generate a DelegationTokenAuthenticatedURL.Token from the given generic * typed delegation token. * * @param dToken The delegation token. * @return The DelegationTokenAuthenticatedURL.Token, with its delegation * token set to the delegation token passed in. */ private DelegationTokenAuthenticatedURL.Token generateDelegationToken( final Token<?> dToken) { DelegationTokenAuthenticatedURL.Token token = new DelegationTokenAuthenticatedURL.Token(); Token<AbstractDelegationTokenIdentifier> dt = new Token<>(dToken.getIdentifier(), dToken.getPassword(), dToken.getKind(), dToken.getService()); token.setDelegationToken(dt); return token; }
/** Renew a token from a file in the local filesystem, matching alias. * @param tokenFile a local File object. * @param fileFormat a string equal to FORMAT_PB or FORMAT_JAVA, for output * @param alias renew only tokens matching alias; null matches all. * @param conf Configuration object passed along. * @throws IOException * @throws InterruptedException */ public static void renewTokenFile( File tokenFile, String fileFormat, Text alias, Configuration conf) throws IOException, InterruptedException { Credentials creds = Credentials.readTokenStorageFile(tokenFile, conf); for (Token<?> token : creds.getAllTokens()) { if (token.isManaged() && matchAlias(token, alias)) { long result = token.renew(conf); LOG.info("Renewed" + token.getKind() + ":" + token.getService() + " until " + formatDate(result)); } } doFormattedWrite(tokenFile, fileFormat, creds, conf); } }
/** Remove a token from a file in the local filesystem, matching alias. * @param cancel cancel token as well as remove from file. * @param tokenFile a local File object. * @param fileFormat a string equal to FORMAT_PB or FORMAT_JAVA, for output * @param alias remove only tokens matching alias; null matches all. * @param conf Configuration object passed along. * @throws IOException * @throws InterruptedException */ public static void removeTokenFromFile(boolean cancel, File tokenFile, String fileFormat, Text alias, Configuration conf) throws IOException, InterruptedException { Credentials newCreds = new Credentials(); Credentials creds = Credentials.readTokenStorageFile(tokenFile, conf); for (Token<?> token : creds.getAllTokens()) { if (matchAlias(token, alias)) { if (token.isManaged() && cancel) { token.cancel(conf); LOG.info("Canceled " + token.getKind() + ":" + token.getService()); } } else { newCreds.addToken(token.getService(), token); } } doFormattedWrite(tokenFile, fileFormat, newCreds, conf); }