@Override public String toString() { StringBuilder sb = new StringBuilder(); sb.append("shouldDie: ").append(shouldDie); sb.append(", TaskSpec: "); if (taskSpec == null) { sb.append("none"); } else { sb.append(taskSpec); } sb.append(", additionalResources: "); if (additionalResources == null) { sb.append("none"); } else { sb.append(additionalResources); } sb.append(", credentialsChanged: ").append(credentialsChanged); sb.append(", credentials: "); if (credentials == null) { sb.append("none"); } else { sb.append("#tokens=").append(credentials.numberOfTokens()) .append(", #secretKeys: ").append(credentials.numberOfSecretKeys()); } return sb.toString(); }
/** * attempts to access tokenCache as from client */ public void map(IntWritable key, IntWritable value, OutputCollector<IntWritable, NullWritable> output, Reporter reporter) throws IOException { // get token storage and a key byte[] key1 = ts.getSecretKey(new Text("alias1")); Collection<Token<? extends TokenIdentifier>> dts = ts.getAllTokens(); int dts_size = 0; if(dts != null) dts_size = dts.size(); if(dts.size() != 2) { // one job token and one delegation token throw new RuntimeException("tokens are not available"); // fail the test } if(key1 == null || ts == null || ts.numberOfSecretKeys() != NUM_OF_KEYS) { throw new RuntimeException("secret keys are not available"); // fail the test } output.collect(new IntWritable(1), NullWritable.get()); }
/** * load job token from a file * @deprecated Use {@link Credentials#readTokenStorageFile} instead, * this method is included for compatibility against Hadoop-1. * @param conf * @throws IOException */ @InterfaceAudience.Private @Deprecated public static Credentials loadTokens(String jobTokenFile, JobConf conf) throws IOException { Path localJobTokenFile = new Path ("file:///" + jobTokenFile); Credentials ts = Credentials.readTokenStorageFile(localJobTokenFile, conf); if(LOG.isDebugEnabled()) { LOG.debug("Task: Loaded jobTokenFile from: "+ localJobTokenFile.toUri().getPath() +"; num of sec keys = " + ts.numberOfSecretKeys() + " Number of tokens " + ts.numberOfTokens()); } return ts; }
/** * load job token from a file * @deprecated Use {@link Credentials#readTokenStorageFile} instead, * this method is included for compatibility against Hadoop-1. * @param conf * @throws IOException */ @InterfaceAudience.Private @Deprecated public static Credentials loadTokens(String jobTokenFile, JobConf conf) throws IOException { Path localJobTokenFile = new Path ("file:///" + jobTokenFile); Credentials ts = Credentials.readTokenStorageFile(localJobTokenFile, conf); if(LOG.isDebugEnabled()) { LOG.debug("Task: Loaded jobTokenFile from: "+ localJobTokenFile.toUri().getPath() +"; num of sec keys = " + ts.numberOfSecretKeys() + " Number of tokens " + ts.numberOfTokens()); } return ts; }
/** * load job token from a file * @param conf * @throws IOException */ @InterfaceAudience.Private public static Credentials loadTokens(String jobTokenFile, JobConf conf) throws IOException { Path localJobTokenFile = new Path ("file:///" + jobTokenFile); Credentials ts = Credentials.readTokenStorageFile(localJobTokenFile, conf); if(LOG.isDebugEnabled()) { LOG.debug("Task: Loaded jobTokenFile from: "+ localJobTokenFile.toUri().getPath() +"; num of sec keys = " + ts.numberOfSecretKeys() + " Number of tokens " + ts.numberOfTokens()); } return ts; } /**
/** * load job token from a file * @deprecated Use {@link Credentials#readTokenStorageFile} instead, * this method is included for compatibility against Hadoop-1. * @param conf * @throws IOException */ @InterfaceAudience.Private @Deprecated public static Credentials loadTokens(String jobTokenFile, JobConf conf) throws IOException { Path localJobTokenFile = new Path ("file:///" + jobTokenFile); Credentials ts = Credentials.readTokenStorageFile(localJobTokenFile, conf); if(LOG.isDebugEnabled()) { LOG.debug("Task: Loaded jobTokenFile from: "+ localJobTokenFile.toUri().getPath() +"; num of sec keys = " + ts.numberOfSecretKeys() + " Number of tokens " + ts.numberOfTokens()); } return ts; }
/** * attempts to access tokenCache as from client */ @Override public void map(IntWritable key, IntWritable value, Context context) throws IOException, InterruptedException { // get token storage and a key Credentials ts = context.getCredentials(); byte[] key1 = ts.getSecretKey(new Text("alias1")); Collection<Token<? extends TokenIdentifier>> dts = ts.getAllTokens(); int dts_size = 0; if(dts != null) dts_size = dts.size(); if(dts_size != 2) { // one job token and one delegation token throw new RuntimeException("tokens are not available"); // fail the test } if(key1 == null || ts == null || ts.numberOfSecretKeys() != NUM_OF_KEYS) { throw new RuntimeException("secret keys are not available"); // fail the test } super.map(key, value, context); } }
/** * load job token from a file * @deprecated Use {@link Credentials#readTokenStorageFile} instead, * this method is included for compatibility against Hadoop-1. * @param conf * @throws IOException */ @InterfaceAudience.Private @Deprecated public static Credentials loadTokens(String jobTokenFile, JobConf conf) throws IOException { Path localJobTokenFile = new Path ("file:///" + jobTokenFile); Credentials ts = Credentials.readTokenStorageFile(localJobTokenFile, conf); if(LOG.isDebugEnabled()) { LOG.debug("Task: Loaded jobTokenFile from: "+ localJobTokenFile.toUri().getPath() +"; num of sec keys = " + ts.numberOfSecretKeys() + " Number of tokens " + ts.numberOfTokens()); } return ts; }
mapLen, ts.numberOfSecretKeys()); for(Text a : m.keySet()) { byte [] kTS = ts.getSecretKey(a);
/** * Setup * * @param containerTask * the new task specification. Must be a valid task * @param childUGI * the old UGI instance being used * @return childUGI */ UserGroupInformation handleNewTaskCredentials(ContainerTask containerTask, UserGroupInformation childUGI) { // Re-use the UGI only if the Credentials have not changed. Preconditions.checkState(!containerTask.shouldDie()); Preconditions.checkState(containerTask.getTaskSpec() != null); if (containerTask.haveCredentialsChanged()) { Credentials taskCreds = containerTask.getCredentials(); if (taskCreds != null) { LOG.info("Refreshing UGI since Credentials have changed. Credentials : #Tokens=" + taskCreds.numberOfTokens() + ", #SecretKeys=" + taskCreds.numberOfSecretKeys()); childUGI = UserGroupInformation.createRemoteUser(user); childUGI.addCredentials(containerTask.getCredentials()); } else { LOG.info("Not loading any credentials, since no credentials provided"); } } return childUGI; }
+ credentials.numberOfSecretKeys() + " secret keys for NM use for launching container in common CLC");
+ credentials.numberOfSecretKeys() + " secret keys for NM use for launching container"); Credentials taskCredentials = new Credentials(credentials);
mapLen, ts.numberOfSecretKeys()); for(Text a : m.keySet()) { byte [] kTS = ts.getSecretKey(a);
mapLen, ts.numberOfSecretKeys()); for(Text a : m.keySet()) { byte [] kTS = ts.getSecretKey(a);
Credentials ugiCreds = ugi.getCredentials(); assertSame(secretKey, ugiCreds.getSecretKey(secretName)); assertEquals(1, ugiCreds.numberOfSecretKeys());
Credentials ugiCreds = ugi.getCredentials(); assertSame(secretKey, ugiCreds.getSecretKey(secretName)); assertEquals(1, ugiCreds.numberOfSecretKeys());
@Test public void mergeAll() { Credentials creds = new Credentials(); creds.addToken(service[0], token[0]); creds.addToken(service[1], token[1]); creds.addSecretKey(secret[0], secret[0].getBytes()); creds.addSecretKey(secret[1], secret[1].getBytes()); Credentials credsToAdd = new Credentials(); // one duplicate with different value, one new credsToAdd.addToken(service[0], token[3]); credsToAdd.addToken(service[2], token[2]); credsToAdd.addSecretKey(secret[0], secret[3].getBytes()); credsToAdd.addSecretKey(secret[2], secret[2].getBytes()); creds.mergeAll(credsToAdd); assertEquals(3, creds.numberOfTokens()); assertEquals(3, creds.numberOfSecretKeys()); // existing token & secret should not be overwritten assertEquals(token[0], creds.getToken(service[0])); assertEquals(secret[0], new Text(creds.getSecretKey(secret[0]))); // non-duplicate token & secret should be present assertEquals(token[1], creds.getToken(service[1])); assertEquals(secret[1], new Text(creds.getSecretKey(secret[1]))); // new token & secret should be added assertEquals(token[2], creds.getToken(service[2])); assertEquals(secret[2], new Text(creds.getSecretKey(secret[2]))); }
@Test public void addAll() { Credentials creds = new Credentials(); creds.addToken(service[0], token[0]); creds.addToken(service[1], token[1]); creds.addSecretKey(secret[0], secret[0].getBytes()); creds.addSecretKey(secret[1], secret[1].getBytes()); Credentials credsToAdd = new Credentials(); // one duplicate with different value, one new credsToAdd.addToken(service[0], token[3]); credsToAdd.addToken(service[2], token[2]); credsToAdd.addSecretKey(secret[0], secret[3].getBytes()); credsToAdd.addSecretKey(secret[2], secret[2].getBytes()); creds.addAll(credsToAdd); assertEquals(3, creds.numberOfTokens()); assertEquals(3, creds.numberOfSecretKeys()); // existing token & secret should be overwritten assertEquals(token[3], creds.getToken(service[0])); assertEquals(secret[3], new Text(creds.getSecretKey(secret[0]))); // non-duplicate token & secret should be present assertEquals(token[1], creds.getToken(service[1])); assertEquals(secret[1], new Text(creds.getSecretKey(secret[1]))); // new token & secret should be added assertEquals(token[2], creds.getToken(service[2])); assertEquals(secret[2], new Text(creds.getSecretKey(secret[2]))); }
@Test public void addAll() { Credentials creds = new Credentials(); creds.addToken(service[0], token[0]); creds.addToken(service[1], token[1]); creds.addSecretKey(secret[0], secret[0].getBytes()); creds.addSecretKey(secret[1], secret[1].getBytes()); Credentials credsToAdd = new Credentials(); // one duplicate with different value, one new credsToAdd.addToken(service[0], token[3]); credsToAdd.addToken(service[2], token[2]); credsToAdd.addSecretKey(secret[0], secret[3].getBytes()); credsToAdd.addSecretKey(secret[2], secret[2].getBytes()); creds.addAll(credsToAdd); assertEquals(3, creds.numberOfTokens()); assertEquals(3, creds.numberOfSecretKeys()); // existing token & secret should be overwritten assertEquals(token[3], creds.getToken(service[0])); assertEquals(secret[3], new Text(creds.getSecretKey(secret[0]))); // non-duplicate token & secret should be present assertEquals(token[1], creds.getToken(service[1])); assertEquals(secret[1], new Text(creds.getSecretKey(secret[1]))); // new token & secret should be added assertEquals(token[2], creds.getToken(service[2])); assertEquals(secret[2], new Text(creds.getSecretKey(secret[2]))); }
@Test public void mergeAll() { Credentials creds = new Credentials(); creds.addToken(service[0], token[0]); creds.addToken(service[1], token[1]); creds.addSecretKey(secret[0], secret[0].getBytes()); creds.addSecretKey(secret[1], secret[1].getBytes()); Credentials credsToAdd = new Credentials(); // one duplicate with different value, one new credsToAdd.addToken(service[0], token[3]); credsToAdd.addToken(service[2], token[2]); credsToAdd.addSecretKey(secret[0], secret[3].getBytes()); credsToAdd.addSecretKey(secret[2], secret[2].getBytes()); creds.mergeAll(credsToAdd); assertEquals(3, creds.numberOfTokens()); assertEquals(3, creds.numberOfSecretKeys()); // existing token & secret should not be overwritten assertEquals(token[0], creds.getToken(service[0])); assertEquals(secret[0], new Text(creds.getSecretKey(secret[0]))); // non-duplicate token & secret should be present assertEquals(token[1], creds.getToken(service[1])); assertEquals(secret[1], new Text(creds.getSecretKey(secret[1]))); // new token & secret should be added assertEquals(token[2], creds.getToken(service[2])); assertEquals(secret[2], new Text(creds.getSecretKey(secret[2]))); }