/** * Queries RM for the list of applications with the given tag that have started * after the given timestamp. */ private Set<ApplicationId> getYarnChildJobs(String tag, long timestamp) { Set<ApplicationId> childYarnJobs = new HashSet<ApplicationId>(); LOG.info(String.format("Querying RM for tag = %s, starting with ts = %s", tag, timestamp)); GetApplicationsRequest gar = GetApplicationsRequest.newInstance(); gar.setScope(ApplicationsRequestScope.OWN); gar.setStartRange(timestamp, System.currentTimeMillis()); gar.setApplicationTags(Collections.singleton(tag)); try { ApplicationClientProtocol proxy = ClientRMProxy.createRMProxy(conf, ApplicationClientProtocol.class); GetApplicationsResponse apps = proxy.getApplications(gar); List<ApplicationReport> appsList = apps.getApplicationList(); for(ApplicationReport appReport : appsList) { childYarnJobs.add(appReport.getApplicationId()); } } catch (IOException ioe) { throw new RuntimeException("Exception occurred while finding child jobs", ioe); } catch (YarnException ye) { throw new RuntimeException("Exception occurred while finding child jobs", ye); } return childYarnJobs; } }
public Text getRMDelegationTokenService() { if (rmDTService == null) { rmDTService = ClientRMProxy.getRMDelegationTokenService(conf); } return rmDTService; }
private static void setAMRMTokenService(final Configuration conf) throws IOException { for (org.apache.hadoop.security.token.Token<? extends TokenIdentifier> token : UserGroupInformation .getCurrentUser().getTokens()) { if (token.getKind().equals(AMRMTokenIdentifier.KIND_NAME)) { token.setService(ClientRMProxy.getAMRMTokenService(conf)); } } }
/** * Create a proxy to the ResourceManager for the specified protocol. * @param configuration Configuration with all the required information. * @param protocol Client protocol for which proxy is being requested. * @param <T> Type of proxy. * @return Proxy to the ResourceManager for the specified client protocol. * @throws IOException */ public static <T> T createRMProxy(final Configuration configuration, final Class<T> protocol) throws IOException { ClientRMProxy<T> clientRMProxy = new ClientRMProxy<>(); return createRMProxy(configuration, protocol, clientRMProxy); }
@Unstable public static Text getAMRMTokenService(Configuration conf) { return getTokenService(conf, YarnConfiguration.RM_SCHEDULER_ADDRESS, YarnConfiguration.DEFAULT_RM_SCHEDULER_ADDRESS, YarnConfiguration.DEFAULT_RM_SCHEDULER_PORT); }
@Private @Override protected InetSocketAddress getRMAddress(YarnConfiguration conf, Class<?> protocol) throws IOException { if (protocol == ApplicationClientProtocol.class) { return conf.getSocketAddr(YarnConfiguration.RM_ADDRESS, YarnConfiguration.DEFAULT_RM_ADDRESS, YarnConfiguration.DEFAULT_RM_PORT); } else if (protocol == ResourceManagerAdministrationProtocol.class) { return conf.getSocketAddr( YarnConfiguration.RM_ADMIN_ADDRESS, YarnConfiguration.DEFAULT_RM_ADMIN_ADDRESS, YarnConfiguration.DEFAULT_RM_ADMIN_PORT); } else if (protocol == ApplicationMasterProtocol.class) { setAMRMTokenService(conf); return conf.getSocketAddr(YarnConfiguration.RM_SCHEDULER_ADDRESS, YarnConfiguration.DEFAULT_RM_SCHEDULER_ADDRESS, YarnConfiguration.DEFAULT_RM_SCHEDULER_PORT); } else { String message = "Unsupported protocol found when creating the proxy " + "connection to ResourceManager: " + ((protocol != null) ? protocol.getClass().getName() : "null"); LOG.error(message); throw new IllegalStateException(message); } }
private static void setAMRMTokenService(final Configuration conf) throws IOException { for (Token<? extends TokenIdentifier> token : UserGroupInformation .getCurrentUser().getTokens()) { if (token.getKind().equals(AMRMTokenIdentifier.KIND_NAME)) { token.setService(getAMRMTokenService(conf)); } } }
@Unstable public static Text getAMRMTokenService(Configuration conf) { return getTokenService(conf, YarnConfiguration.RM_SCHEDULER_ADDRESS, YarnConfiguration.DEFAULT_RM_SCHEDULER_ADDRESS, YarnConfiguration.DEFAULT_RM_SCHEDULER_PORT); }
@Private @Override protected InetSocketAddress getRMAddress(YarnConfiguration conf, Class<?> protocol) throws IOException { if (protocol == ApplicationClientProtocol.class) { return conf.getSocketAddr(YarnConfiguration.RM_ADDRESS, YarnConfiguration.DEFAULT_RM_ADDRESS, YarnConfiguration.DEFAULT_RM_PORT); } else if (protocol == ResourceManagerAdministrationProtocol.class) { return conf.getSocketAddr( YarnConfiguration.RM_ADMIN_ADDRESS, YarnConfiguration.DEFAULT_RM_ADMIN_ADDRESS, YarnConfiguration.DEFAULT_RM_ADMIN_PORT); } else if (protocol == ApplicationMasterProtocol.class) { setAMRMTokenService(conf); return conf.getSocketAddr(YarnConfiguration.RM_SCHEDULER_ADDRESS, YarnConfiguration.DEFAULT_RM_SCHEDULER_ADDRESS, YarnConfiguration.DEFAULT_RM_SCHEDULER_PORT); } else { String message = "Unsupported protocol found when creating the proxy " + "connection to ResourceManager: " + ((protocol != null) ? protocol.getClass().getName() : "null"); LOG.error(message); throw new IllegalStateException(message); } }
public static Set<ApplicationId> getChildYarnJobs(Configuration conf, String tag) throws IOException, YarnException { Set<ApplicationId> childYarnJobs = new HashSet<ApplicationId>(); GetApplicationsRequest gar = GetApplicationsRequest.newInstance(); gar.setScope(ApplicationsRequestScope.OWN); gar.setApplicationTags(Collections.singleton(tag)); ApplicationClientProtocol proxy = ClientRMProxy.createRMProxy(conf, ApplicationClientProtocol.class); GetApplicationsResponse apps = proxy.getApplications(gar); List<ApplicationReport> appsList = apps.getApplicationList(); for(ApplicationReport appReport : appsList) { if (isAdmin() || appReport.getApplicationTags().contains(QueryState.USERID_TAG + "=" + SessionState.get() .getUserName())) { childYarnJobs.add(appReport.getApplicationId()); } } if (childYarnJobs.isEmpty()) { LOG.info("No child applications found"); } else { LOG.info("Found child YARN applications: " + StringUtils.join(childYarnJobs, ",")); } return childYarnJobs; }
private static void setAMRMTokenService(final Configuration conf) throws IOException { for (Token<? extends TokenIdentifier> token : UserGroupInformation .getCurrentUser().getTokens()) { if (token.getKind().equals(AMRMTokenIdentifier.KIND_NAME)) { token.setService(getAMRMTokenService(conf)); } } }
public Text getRMDelegationTokenService() { if (rmDTService == null) { rmDTService = ClientRMProxy.getRMDelegationTokenService(conf); } return rmDTService; }
@Unstable public static Text getAMRMTokenService(Configuration conf) { return getTokenService(conf, YarnConfiguration.RM_SCHEDULER_ADDRESS, YarnConfiguration.DEFAULT_RM_SCHEDULER_ADDRESS, YarnConfiguration.DEFAULT_RM_SCHEDULER_PORT); }
@Private @Override public InetSocketAddress getRMAddress(YarnConfiguration conf, Class<?> protocol) throws IOException { if (protocol == ApplicationClientProtocol.class) { return conf.getSocketAddr(YarnConfiguration.RM_ADDRESS, YarnConfiguration.DEFAULT_RM_ADDRESS, YarnConfiguration.DEFAULT_RM_PORT); } else if (protocol == ResourceManagerAdministrationProtocol.class) { return conf.getSocketAddr( YarnConfiguration.RM_ADMIN_ADDRESS, YarnConfiguration.DEFAULT_RM_ADMIN_ADDRESS, YarnConfiguration.DEFAULT_RM_ADMIN_PORT); } else if (protocol == ApplicationMasterProtocol.class) { setAMRMTokenService(conf); return conf.getSocketAddr(YarnConfiguration.RM_SCHEDULER_ADDRESS, YarnConfiguration.DEFAULT_RM_SCHEDULER_ADDRESS, YarnConfiguration.DEFAULT_RM_SCHEDULER_PORT); } else { String message = "Unsupported protocol found when creating the proxy " + "connection to ResourceManager: " + ((protocol != null) ? protocol.getClass().getName() : "null"); LOG.error(message); throw new IllegalStateException(message); } }
@Override public ApplicationMasterProtocol run() throws Exception { setAMRMTokenService(conf); return ClientRMProxy.createRMProxy(conf, ApplicationMasterProtocol.class); } });
private static void setAMRMTokenService(final Configuration conf) throws IOException { for (Token<? extends TokenIdentifier> token : UserGroupInformation .getCurrentUser().getTokens()) { if (token.getKind().equals(AMRMTokenIdentifier.KIND_NAME)) { token.setService(getAMRMTokenService(conf)); } } }
/** * Create and add an RM delegation token to the credentials * @param yarnClient Yarn Client * @param credentials to add token to * @return the token which was added * @throws IOException * @throws YarnException */ public static Token<TokenIdentifier> addRMDelegationToken(YarnClient yarnClient, Credentials credentials) throws IOException, YarnException { Configuration conf = yarnClient.getConfig(); Text rmPrincipal = new Text(CredentialUtils.getRMPrincipal(conf)); Text rmDTService = ClientRMProxy.getRMDelegationTokenService(conf); Token<TokenIdentifier> rmDelegationToken = ConverterUtils.convertFromYarn( yarnClient.getRMDelegationToken(rmPrincipal), rmDTService); credentials.addToken(rmDelegationToken.getService(), rmDelegationToken); return rmDelegationToken; }
/** * Get the token service name to be used for RMDelegationToken. Depending * on whether HA is enabled or not, this method generates the appropriate * service name as a comma-separated list of service addresses. * * @param conf Configuration corresponding to the cluster we need the * RMDelegationToken for * @return - Service name for RMDelegationToken */ @Unstable public static Text getRMDelegationTokenService(Configuration conf) { return getTokenService(conf, YarnConfiguration.RM_ADDRESS, YarnConfiguration.DEFAULT_RM_ADDRESS, YarnConfiguration.DEFAULT_RM_PORT); }
@Override public ApplicationClientProtocol run() throws Exception { return ClientRMProxy.createRMProxy(conf, ApplicationClientProtocol.class); } });
private void updateAMRMToken(Token token) throws IOException { org.apache.hadoop.security.token.Token<AMRMTokenIdentifier> amrmToken = new org.apache.hadoop.security.token.Token<AMRMTokenIdentifier>(token .getIdentifier().array(), token.getPassword().array(), new Text( token.getKind()), new Text(token.getService())); // Preserve the token service sent by the RM when adding the token // to ensure we replace the previous token setup by the RM. // Afterwards we can update the service address for the RPC layer. UserGroupInformation currentUGI = UserGroupInformation.getCurrentUser(); currentUGI.addToken(amrmToken); amrmToken.setService(ClientRMProxy.getAMRMTokenService(getConfig())); }