private void setupSecurityTokens(ContainerLaunchContext containerLaunchContext) throws IOException { Credentials credentials = UserGroupInformation.getCurrentUser().getCredentials(); String tokenRenewer = this.yarnConfiguration.get(YarnConfiguration.RM_PRINCIPAL); if (tokenRenewer == null || tokenRenewer.length() == 0) { throw new IOException("Failed to get master Kerberos principal for the RM to use as renewer"); } // For now, only getting tokens for the default file-system. Token<?> tokens[] = this.fs.addDelegationTokens(tokenRenewer, credentials); if (tokens != null) { for (Token<?> token : tokens) { LOGGER.info("Got delegation token for " + this.fs.getUri() + "; " + token); } } Closer closer = Closer.create(); try { DataOutputBuffer dataOutputBuffer = closer.register(new DataOutputBuffer()); credentials.writeTokenStorageToStream(dataOutputBuffer); ByteBuffer fsTokens = ByteBuffer.wrap(dataOutputBuffer.getData(), 0, dataOutputBuffer.getLength()); containerLaunchContext.setTokens(fsTokens); } catch (Throwable t) { throw closer.rethrow(t); } finally { closer.close(); } }
private ContainerLaunchContext prepareContainerLaunchContext() throws IOException { ContainerLaunchContext clc = Records.newRecord(ContainerLaunchContext.class); clc.setCommands(Arrays.asList("/bin/sleep 5")); if (UserGroupInformation.isSecurityEnabled()) { clc.setTokens(getTokens()); clc.setTokensConf(getTokensConf()); } clc.setLocalResources(getLocalResources()); clc.setEnvironment(getEnvironment()); clc.setContainerRetryContext(getContainerRetryContext()); clc.setServiceData(getServiceData()); return clc; }
public static void setTokensFor(ContainerLaunchContext amContainer, List<Path> paths, Configuration conf) throws IOException { Credentials credentials = new Credentials(); // for HDFS TokenCache.obtainTokensForNamenodes(credentials, paths.toArray(new Path[0]), conf); // for HBase obtainTokenForHBase(credentials, conf); // for user UserGroupInformation currUsr = UserGroupInformation.getCurrentUser(); Collection<Token<? extends TokenIdentifier>> usrTok = currUsr.getTokens(); for (Token<? extends TokenIdentifier> token : usrTok) { final Text id = new Text(token.getIdentifier()); LOG.info("Adding user token " + id + " with " + token); credentials.addToken(id, token); } try (DataOutputBuffer dob = new DataOutputBuffer()) { credentials.writeTokenStorageToStream(dob); if (LOG.isDebugEnabled()) { LOG.debug("Wrote tokens. Credentials buffer length: " + dob.getLength()); } ByteBuffer securityTokens = ByteBuffer.wrap(dob.getData(), 0, dob.getLength()); amContainer.setTokens(securityTokens); } }
ctx.setTokens(securityTokens); } catch (Throwable t) { log.error("Failed to add Hadoop's security tokens.", t);
private ContainerLaunchContext newContainerLaunchContext(Container container, String helixInstanceName) throws IOException { Path appWorkDir = GobblinClusterUtils.getAppWorkDirPath(this.fs, this.applicationName, this.applicationId); Path containerWorkDir = new Path(appWorkDir, GobblinYarnConfigurationKeys.CONTAINER_WORK_DIR_NAME); Map<String, LocalResource> resourceMap = Maps.newHashMap(); addContainerLocalResources(new Path(appWorkDir, GobblinYarnConfigurationKeys.LIB_JARS_DIR_NAME), resourceMap); addContainerLocalResources(new Path(containerWorkDir, GobblinYarnConfigurationKeys.APP_JARS_DIR_NAME), resourceMap); addContainerLocalResources( new Path(containerWorkDir, GobblinYarnConfigurationKeys.APP_FILES_DIR_NAME), resourceMap); if (this.config.hasPath(GobblinYarnConfigurationKeys.CONTAINER_FILES_REMOTE_KEY)) { addRemoteAppFiles(this.config.getString(GobblinYarnConfigurationKeys.CONTAINER_FILES_REMOTE_KEY), resourceMap); } ContainerLaunchContext containerLaunchContext = Records.newRecord(ContainerLaunchContext.class); containerLaunchContext.setLocalResources(resourceMap); containerLaunchContext.setEnvironment(YarnHelixUtils.getEnvironmentVariables(this.yarnConfiguration)); containerLaunchContext.setCommands(Lists.newArrayList(buildContainerCommand(container, helixInstanceName))); if (UserGroupInformation.isSecurityEnabled()) { containerLaunchContext.setTokens(this.tokens.duplicate()); } return containerLaunchContext; }
credentials.writeTokenStorageToStream(dob); ByteBuffer fsTokens = ByteBuffer.wrap(dob.getData(), 0, dob.getLength()); amContainer.setTokens(fsTokens);
ctx.setTokens(allTokens.duplicate());
log.info("File system delegation tokens: " + Arrays.toString(tokens)); amContainer.setTokens(IgniteYarnUtils.createTokenBuffer(creds));
credentials.writeTokenStorageToStream(dob); ByteBuffer fsTokens = ByteBuffer.wrap(dob.getData(), 0, dob.getLength()); amContainer.setTokens(fsTokens);
@Override public void setCredentials(Credentials credentials) { launchContext.setTokens(YarnUtils.encodeCredentials(credentials)); }
@Override public void setCredentials(Credentials credentials) { launchContext.setTokens(YarnUtils.encodeCredentials(credentials)); }
/** * Adds RM delegation token to the given {@link ContainerLaunchContext} so that the AM can authenticate itself * with RM using the delegation token. */ protected void addRMToken(ContainerLaunchContext context, YarnClient yarnClient, ApplicationId appId) { if (!UserGroupInformation.isSecurityEnabled()) { return; } try { Credentials credentials = YarnUtils.decodeCredentials(context.getTokens()); Configuration config = yarnClient.getConfig(); Token<TokenIdentifier> token = ConverterUtils.convertFromYarn( yarnClient.getRMDelegationToken(new Text(YarnUtils.getYarnTokenRenewer(config))), YarnUtils.getRMAddress(config)); LOG.debug("Added RM delegation token {} for application {}", token, appId); credentials.addToken(token.getService(), token); context.setTokens(YarnUtils.encodeCredentials(credentials)); } catch (YarnException | IOException e) { throw new RuntimeException("Failed to acquire RM delegation token", e); } }
public static ContainerLaunchContext newContainerLaunchContext( Map<String, LocalResource> localResources, Map<String, String> environment, List<String> commands, Map<String, ByteBuffer> serviceData, ByteBuffer tokens, Map<ApplicationAccessType, String> acls) { ContainerLaunchContext container = recordFactory .newRecordInstance(ContainerLaunchContext.class); container.setLocalResources(localResources); container.setEnvironment(environment); container.setCommands(commands); container.setServiceData(serviceData); container.setTokens(tokens); container.setApplicationACLs(acls); return container; }
public static ContainerLaunchContext newContainerLaunchContext( Map<String, LocalResource> localResources, Map<String, String> environment, List<String> commands, Map<String, ByteBuffer> serviceData, ByteBuffer tokens, Map<ApplicationAccessType, String> acls) { ContainerLaunchContext container = recordFactory .newRecordInstance(ContainerLaunchContext.class); container.setLocalResources(localResources); container.setEnvironment(environment); container.setCommands(commands); container.setServiceData(serviceData); container.setTokens(tokens); container.setApplicationACLs(acls); return container; }
public static ContainerLaunchContext newContainerLaunchContext( Map<String, LocalResource> localResources, Map<String, String> environment, List<String> commands, Map<String, ByteBuffer> serviceData, ByteBuffer tokens, Map<ApplicationAccessType, String> acls) { ContainerLaunchContext container = recordFactory .newRecordInstance(ContainerLaunchContext.class); container.setLocalResources(localResources); container.setEnvironment(environment); container.setCommands(commands); container.setServiceData(serviceData); container.setTokens(tokens); container.setApplicationACLs(acls); return container; }
@Public @Stable public static ContainerLaunchContext newInstance( Map<String, LocalResource> localResources, Map<String, String> environment, List<String> commands, Map<String, ByteBuffer> serviceData, ByteBuffer tokens, Map<ApplicationAccessType, String> acls) { ContainerLaunchContext container = Records.newRecord(ContainerLaunchContext.class); container.setLocalResources(localResources); container.setEnvironment(environment); container.setCommands(commands); container.setServiceData(serviceData); container.setTokens(tokens); container.setApplicationACLs(acls); return container; }
@Public @Stable public static ContainerLaunchContext newInstance( Map<String, LocalResource> localResources, Map<String, String> environment, List<String> commands, Map<String, ByteBuffer> serviceData, ByteBuffer tokens, Map<ApplicationAccessType, String> acls) { ContainerLaunchContext container = Records.newRecord(ContainerLaunchContext.class); container.setLocalResources(localResources); container.setEnvironment(environment); container.setCommands(commands); container.setServiceData(serviceData); container.setTokens(tokens); container.setApplicationACLs(acls); return container; }
@Public @Stable public static ContainerLaunchContext newInstance( Map<String, LocalResource> localResources, Map<String, String> environment, List<String> commands, Map<String, ByteBuffer> serviceData, ByteBuffer tokens, Map<ApplicationAccessType, String> acls) { ContainerLaunchContext container = Records.newRecord(ContainerLaunchContext.class); container.setLocalResources(localResources); container.setEnvironment(environment); container.setCommands(commands); container.setServiceData(serviceData); container.setTokens(tokens); container.setApplicationACLs(acls); return container; }
@Public @Unstable public static ContainerLaunchContext newInstance( Map<String, LocalResource> localResources, Map<String, String> environment, List<String> commands, Map<String, ByteBuffer> serviceData, ByteBuffer tokens, Map<ApplicationAccessType, String> acls, ContainerRetryContext containerRetryContext) { ContainerLaunchContext container = Records.newRecord(ContainerLaunchContext.class); container.setLocalResources(localResources); container.setEnvironment(environment); container.setCommands(commands); container.setServiceData(serviceData); container.setTokens(tokens); container.setApplicationACLs(acls); container.setContainerRetryContext(containerRetryContext); return container; }
@Test (timeout = 30000) public void testAppRecoverPath() throws IOException { LOG.info("--- START: testAppRecoverPath ---"); ApplicationSubmissionContext sub = Records.newRecord(ApplicationSubmissionContext.class); ContainerLaunchContext clc = Records.newRecord(ContainerLaunchContext.class); Credentials credentials = new Credentials(); DataOutputBuffer dob = new DataOutputBuffer(); credentials.writeTokenStorageToStream(dob); ByteBuffer securityTokens = ByteBuffer.wrap(dob.getData(), 0, dob.getLength()); clc.setTokens(securityTokens); sub.setAMContainerSpec(clc); testCreateAppSubmittedRecovery(sub); }