@Test public void testStartupFailure() throws Exception { Configuration conf = new Configuration(); DelegationTokenRenewer delegationTokenRenewer = new DelegationTokenRenewer(); RMContext mockContext = mock(RMContext.class); ClientRMService mockClientRMService = mock(ClientRMService.class); when(mockContext.getClientRMService()).thenReturn(mockClientRMService); delegationTokenRenewer.setRMContext(mockContext); delegationTokenRenewer.init(conf); delegationTokenRenewer.stop(); } }
@Override public void run() { dtr.addApplicationAsync(mock(ApplicationId.class), creds1, false, "user"); } };
protected synchronized void finishApplication(ApplicationId applicationId) { if (applicationId == null) { LOG.error("RMAppManager received completed appId of null, skipping"); } else { // Inform the DelegationTokenRenewer if (UserGroupInformation.isSecurityEnabled()) { rmContext.getDelegationTokenRenewer().applicationFinished(applicationId); } completedApps.add(applicationId); completedAppsInStateStore++; writeAuditLog(applicationId); } }
@Override public void run() { if (cancelled.get()) { return; } Token<?> token = dttr.token; try { requestNewHdfsDelegationTokenIfNeeded(dttr); // if the token is not replaced by a new token, renew the token if (!dttr.isTimerCancelled()) { renewToken(dttr); setTimerForTokenRenewal(dttr);// set the next one } else { LOG.info("The token was removed already. Token = [" +dttr +"]"); } } catch (Exception e) { LOG.error("Exception renewing token" + token + ". Not rescheduled", e); removeFailedDelegationToken(dttr); } }
@Override protected void serviceInit(Configuration conf) throws Exception { this.hasProxyUserPrivileges = conf.getBoolean(YarnConfiguration.RM_PROXY_USER_PRIVILEGES_ENABLED, YarnConfiguration.DEFAULT_RM_PROXY_USER_PRIVILEGES_ENABLED); this.tokenKeepAliveEnabled = conf.getBoolean(YarnConfiguration.LOG_AGGREGATION_ENABLED, YarnConfiguration.DEFAULT_LOG_AGGREGATION_ENABLED); this.tokenRemovalDelayMs = conf.getInt(YarnConfiguration.RM_NM_EXPIRY_INTERVAL_MS, YarnConfiguration.DEFAULT_RM_NM_EXPIRY_INTERVAL_MS); this.credentialsValidTimeRemaining = conf.getLong(RM_SYSTEM_CREDENTIALS_VALID_TIME_REMAINING, DEFAULT_RM_SYSTEM_CREDENTIALS_VALID_TIME_REMAINING); setLocalSecretManagerAndServiceAddr(); renewerService = createNewThreadPoolService(conf); pendingEventQueue = new LinkedBlockingQueue<DelegationTokenRenewerEvent>(); renewalTimer = new Timer(true); super.serviceInit(conf); }
InetSocketAddress.createUnresolved("localhost", 1234); when(mockClientRMService.getBindAddress()).thenReturn(sockAddr); dtr.setRMContext(mockContext); when(mockContext.getDelegationTokenRenewer()).thenReturn(dtr); dtr.init(conf); dtr.start(); dtr.addApplicationAsync(mock(ApplicationId.class), creds2, false, "user", new Configuration());
Token<?>[] newTokens = obtainSystemTokensForUser(user, credentials); if (token.isManaged()) { DelegationTokenToRenew tokenToRenew = new DelegationTokenToRenew(referringAppIds, token, getConfig(), Time.now(), shouldCancelAtEnd, user); renewToken(tokenToRenew); setTimerForTokenRenewal(tokenToRenew); for (ApplicationId applicationId : referringAppIds) { appTokens.get(applicationId).add(tokenToRenew);
/** * Test that the DelegationTokenRenewer class can gracefully handle * interactions that occur when it has been stopped. */ @Test public void testShutDown() { DelegationTokenRenewer dtr = createNewDelegationTokenRenewer(conf, counter); RMContext mockContext = mock(RMContext.class); when(mockContext.getSystemCredentialsForApps()).thenReturn( new ConcurrentHashMap<ApplicationId, ByteBuffer>()); ClientRMService mockClientRMService = mock(ClientRMService.class); when(mockContext.getClientRMService()).thenReturn(mockClientRMService); InetSocketAddress sockAddr = InetSocketAddress.createUnresolved("localhost", 1234); when(mockClientRMService.getBindAddress()).thenReturn(sockAddr); dtr.setRMContext(mockContext); when(mockContext.getDelegationTokenRenewer()).thenReturn(dtr); dtr.init(conf); dtr.start(); delegationTokenRenewer.stop(); delegationTokenRenewer.applicationFinished( BuilderUtils.newApplicationId(0, 1)); } }
delegationTokenRenewer.addApplicationAsync(applicationId_1, ts, false, "user", new Configuration()); waitForEventsToGetProcessed(delegationTokenRenewer); delegationTokenRenewer.applicationFinished(applicationId_1); waitForEventsToGetProcessed(delegationTokenRenewer); int numberOfExpectedRenewals = Renewer.counter; // number of renewals so far
if (dttr == null) { dttr = new DelegationTokenToRenew(Arrays.asList(applicationId), token, getConfig(), now, shouldCancelAtEnd, evt.getUser()); try { renewToken(dttr); } catch (IOException ioe) { throw new IOException("Failed to renew token: " + dttr.token, ioe); } else { appTokens.get(applicationId).add(dtr); setTimerForTokenRenewal(dtr); requestNewHdfsDelegationToken(Arrays.asList(applicationId), evt.getUser(), shouldCancelAtEnd);
hasHdfsToken = true; if (skipTokenRenewal(token)) { continue; tokenConf = getConfig(); renewToken(dttr); } catch (IOException ioe) { if (ioe instanceof SecretManager.InvalidToken + " on recovery as it expired, requesting new hdfs token for " + applicationId + ", user=" + evt.getUser(), ioe); requestNewHdfsDelegationTokenAsProxyUser( Arrays.asList(applicationId), evt.getUser(), evt.shouldCancelAtEnd()); } else { appTokens.get(applicationId).add(dtr); setTimerForTokenRenewal(dtr); requestNewHdfsDelegationTokenAsProxyUser(Arrays.asList(applicationId), evt.getUser(), shouldCancelAtEnd);
InetSocketAddress.createUnresolved("localhost", 1234); when(mockClientRMService.getBindAddress()).thenReturn(sockAddr); dtr.setRMContext(mockContext); when(mockContext.getDelegationTokenRenewer()).thenReturn(dtr); dtr.init(conf); dtr.start(); dtr.addApplicationSync(mock(ApplicationId.class), credsx, false, "user"); fail("Catch IOException on app submission"); } catch (IOException e){
@Before public void setUp() throws Exception { counter = new AtomicInteger(0); conf.set(CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHENTICATION, "kerberos"); UserGroupInformation.setConfiguration(conf); eventQueue = new LinkedBlockingQueue<Event>(); dispatcher = new AsyncDispatcher(eventQueue); Renewer.reset(); delegationTokenRenewer = createNewDelegationTokenRenewer(conf, counter); RMContext mockContext = mock(RMContext.class); ClientRMService mockClientRMService = mock(ClientRMService.class); when(mockContext.getSystemCredentialsForApps()).thenReturn( new ConcurrentHashMap<ApplicationId, ByteBuffer>()); when(mockContext.getDelegationTokenRenewer()).thenReturn( delegationTokenRenewer); when(mockContext.getDispatcher()).thenReturn(dispatcher); when(mockContext.getClientRMService()).thenReturn(mockClientRMService); InetSocketAddress sockAddr = InetSocketAddress.createUnresolved("localhost", 1234); when(mockClientRMService.getBindAddress()).thenReturn(sockAddr); delegationTokenRenewer.setRMContext(mockContext); delegationTokenRenewer.init(conf); delegationTokenRenewer.start(); }
private DelegationTokenRenewer createNewDelegationTokenRenewer( Configuration conf, final AtomicInteger counter) { DelegationTokenRenewer renew = new DelegationTokenRenewer() { @Override protected ThreadPoolExecutor createNewThreadPoolService(Configuration conf) { ThreadPoolExecutor pool = new ThreadPoolExecutor(5, 5, 3L, TimeUnit.SECONDS, new LinkedBlockingQueue<Runnable>()) { @Override protected void afterExecute(Runnable r, Throwable t) { counter.decrementAndGet(); super.afterExecute(r, t); } @Override public void execute(Runnable command) { counter.incrementAndGet(); super.execute(command); } }; return pool; } }; renew.setRMContext(TestUtils.getMockRMContext()); return renew; }
protected DelegationTokenRenewer createDelegationTokenRenewer() { return new DelegationTokenRenewer(); }
rmNode.context.getDelegationTokenRenewer().updateKeepAliveApplications( statusEvent.getKeepAliveAppIds());
@Test public void testFSLeakInObtainSystemTokensForUser() throws Exception{ Credentials credentials = new Credentials(); String user = "test"; int oldCounter = MyFS.getInstanceCounter(); delegationTokenRenewer.obtainSystemTokensForUser(user, credentials); delegationTokenRenewer.obtainSystemTokensForUser(user, credentials); delegationTokenRenewer.obtainSystemTokensForUser(user, credentials); Assert.assertEquals(oldCounter, MyFS.getInstanceCounter()); }
@Test public void testTokenRenewerInvalidReturn() throws Exception { DelegationTokenToRenew mockDttr = mock(DelegationTokenToRenew.class); mockDttr.expirationDate = 0; delegationTokenRenewer.setTimerForTokenRenewal(mockDttr); assertNull(mockDttr.timerTask); mockDttr.expirationDate = -1; delegationTokenRenewer.setTimerForTokenRenewal(mockDttr); assertNull(mockDttr.timerTask); mockDttr.expirationDate = System.currentTimeMillis() - 1; delegationTokenRenewer.setTimerForTokenRenewal(mockDttr); assertNull(mockDttr.timerTask); }
private void removeApplicationFromRenewal(ApplicationId applicationId) { rmContext.getSystemCredentialsForApps().remove(applicationId); Set<DelegationTokenToRenew> tokens = appTokens.remove(applicationId); if (tokens != null && !tokens.isEmpty()) { synchronized (tokens) { Iterator<DelegationTokenToRenew> it = tokens.iterator(); while (it.hasNext()) { DelegationTokenToRenew dttr = it.next(); if (LOG.isDebugEnabled()) { LOG.debug("Removing delegation token for appId=" + applicationId + "; token=" + dttr.token.getService()); } // continue if the app list isn't empty synchronized(dttr.referringAppIds) { dttr.referringAppIds.remove(applicationId); if (!dttr.referringAppIds.isEmpty()) { continue; } } // cancel the timer dttr.cancelTimer(); // cancel the token cancelToken(dttr); allTokens.remove(dttr.token); } } } }
.addApplicationAsyncDuringRecovery(app.getApplicationId(), BuilderUtils.parseCredentials(app.submissionContext), app.submissionContext.getCancelTokensWhenComplete(),