@VisibleForTesting ApplicationPlacementContext placeApplication( PlacementManager placementManager, ApplicationSubmissionContext context, String user, boolean isRecovery) throws YarnException { ApplicationPlacementContext placementContext = null; if (placementManager != null) { try { placementContext = placementManager.placeApplication(context, user); } catch (YarnException e) { // Placement could also fail if the user doesn't exist in system // skip if the user is not found during recovery. if (isRecovery) { LOG.warn("PlaceApplication failed,skipping on recovery of rm"); return placementContext; } throw e; } } if (placementContext == null && (context.getQueue() == null) || context .getQueue().isEmpty()) { String msg = "Failed to place application " + context.getApplicationId() + " to queue and specified " + "queue is invalid : " + context .getQueue(); LOG.error(msg); throw new YarnException(msg); } return placementContext; }
public AppAddedSchedulerEvent(String user, ApplicationSubmissionContext submissionContext, boolean isAppRecovering, Priority appPriority, ApplicationPlacementContext placementContext) { this(submissionContext.getApplicationId(), submissionContext.getQueue(), user, isAppRecovering, submissionContext.getReservationID(), appPriority, placementContext); }
public AppAddedSchedulerEvent(String user, ApplicationSubmissionContext submissionContext, boolean isAppRecovering, Priority appPriority) { this(submissionContext.getApplicationId(), submissionContext.getQueue(), user, isAppRecovering, submissionContext.getReservationID(), appPriority, null); }
@Override public void transition(RMAppImpl app, RMAppEvent event) { app.handler.handle(new AppAddedSchedulerEvent(app.applicationId, app.submissionContext.getQueue(), app.user, app.submissionContext.getReservationID())); } }
void replaceQueueFromPlacementContext( ApplicationPlacementContext placementContext, ApplicationSubmissionContext context) { // Set it to ApplicationSubmissionContext //apply queue mapping only to new application submissions if (placementContext != null && !StringUtils.equalsIgnoreCase( context.getQueue(), placementContext.getQueue())) { LOG.info("Placed application=" + context.getApplicationId() + " to queue=" + placementContext.getQueue() + ", original queue=" + context .getQueue()); context.setQueue(placementContext.getQueue()); } } }
@Override public void transition(RMAppImpl app, RMAppEvent event) { app.handler.handle(new AppAddedSchedulerEvent(app.applicationId, app.submissionContext.getQueue(), app.user, app.submissionContext.getReservationID())); } }
@Override public ApplicationPlacementContext getPlacementForApp( ApplicationSubmissionContext asc, String user) throws YarnException { String queueName = asc.getQueue(); ApplicationId applicationId = asc.getApplicationId(); if (mappings != null && mappings.size() > 0) { try { ApplicationPlacementContext mappedQueue = getPlacementForUser(user); if (mappedQueue != null) { // We have a mapping, should we use it? if (queueName.equals(YarnConfiguration.DEFAULT_QUEUE_NAME) //queueName will be same as mapped queue name in case of recovery || queueName.equals(mappedQueue.getQueue()) || overrideWithQueueMappings) { LOG.info("Application " + applicationId + " user " + user + " mapping [" + queueName + "] to [" + mappedQueue + "] override " + overrideWithQueueMappings); return mappedQueue; } } } catch (IOException ioex) { String message = "Failed to submit application " + applicationId + " submitted by user " + user + " reason: " + ioex.getMessage(); throw new YarnException(message); } } return null; }
@Test public void testHashSpreadUniformlyAmongSubclusters() throws YarnException { SubClusterId chosen; Map<SubClusterId, AtomicLong> counter = new HashMap<>(); for (SubClusterId id : getActiveSubclusters().keySet()) { counter.put(id, new AtomicLong(0)); } long jobPerSub = 100; ApplicationSubmissionContext applicationSubmissionContext = mock(ApplicationSubmissionContext.class); for (int i = 0; i < jobPerSub * numSubclusters; i++) { when(applicationSubmissionContext.getQueue()).thenReturn("queue" + i); chosen = ((FederationRouterPolicy) getPolicy()) .getHomeSubcluster(applicationSubmissionContext, null); counter.get(chosen).addAndGet(1); } // hash spread the jobs equally among the subclusters for (AtomicLong a : counter.values()) { Assert.assertEquals(a.get(), jobPerSub); } } }
@Test public void testFallbacks() throws YarnException { // this tests the behavior of the system when the queue requested is // not configured (or null) and there is no default policy configured // for DEFAULT_FEDERATION_POLICY_KEY (*). This is our second line of // defense. ApplicationSubmissionContext applicationSubmissionContext = mock(ApplicationSubmissionContext.class); // The facade answers also for non-initialized policies (using the // defaultPolicy) String uninitQueue = "non-initialized-queue"; when(applicationSubmissionContext.getQueue()).thenReturn(uninitQueue); SubClusterId chosen = routerFacade.getHomeSubcluster(applicationSubmissionContext, null); Assert.assertTrue(subClusterIds.contains(chosen)); Assert.assertFalse(routerFacade.globalPolicyMap.containsKey(uninitQueue)); // empty string when(applicationSubmissionContext.getQueue()).thenReturn(""); chosen = routerFacade.getHomeSubcluster(applicationSubmissionContext, null); Assert.assertTrue(subClusterIds.contains(chosen)); Assert.assertFalse(routerFacade.globalPolicyMap.containsKey(uninitQueue)); // null queue also falls back to default when(applicationSubmissionContext.getQueue()).thenReturn(null); chosen = routerFacade.getHomeSubcluster(applicationSubmissionContext, null); Assert.assertTrue(subClusterIds.contains(chosen)); Assert.assertFalse(routerFacade.globalPolicyMap.containsKey(uninitQueue)); }
@Test public void testGetHomeSubcluster() throws YarnException { ApplicationSubmissionContext applicationSubmissionContext = mock(ApplicationSubmissionContext.class); when(applicationSubmissionContext.getQueue()).thenReturn(queue1); // the facade only contains the fallback behavior Assert.assertTrue(routerFacade.globalPolicyMap.containsKey(defQueueKey) && routerFacade.globalPolicyMap.size() == 1); // when invoked it returns the expected SubClusterId. SubClusterId chosen = routerFacade.getHomeSubcluster(applicationSubmissionContext, null); Assert.assertTrue(subClusterIds.contains(chosen)); // now the caching of policies must have added an entry for this queue Assert.assertTrue(routerFacade.globalPolicyMap.size() == 2); // after the facade is used the policyMap contains the expected policy type. Assert.assertTrue(routerFacade.globalPolicyMap .get(queue1) instanceof UniformRandomRouterPolicy); // the facade is again empty after reset routerFacade.reset(); // the facade only contains the fallback behavior Assert.assertTrue(routerFacade.globalPolicyMap.containsKey(defQueueKey) && routerFacade.globalPolicyMap.size() == 1); }
new RMAppImpl(applicationId, rmContext, this.conf, submissionContext.getApplicationName(), user, submissionContext.getQueue(), submissionContext, this.scheduler, this.masterService, submitTime, submissionContext.getApplicationType(),
new RMAppImpl(applicationId, rmContext, this.conf, submissionContext.getApplicationName(), user, submissionContext.getQueue(), submissionContext, this.scheduler, this.masterService, submitTime, submissionContext.getApplicationType(),
@Test public void testConfigurationUpdate() throws YarnException { // in this test we see what happens when the configuration is changed // between calls. We achieve this by changing what is in the store. ApplicationSubmissionContext applicationSubmissionContext = mock(ApplicationSubmissionContext.class); when(applicationSubmissionContext.getQueue()).thenReturn(queue1); // first call runs using standard UniformRandomRouterPolicy SubClusterId chosen = routerFacade.getHomeSubcluster(applicationSubmissionContext, null); Assert.assertTrue(subClusterIds.contains(chosen)); Assert.assertTrue(routerFacade.globalPolicyMap .get(queue1) instanceof UniformRandomRouterPolicy); // then the operator changes how queue1 is routed setting it to // PriorityRouterPolicy with weights favoring the first subcluster in // subClusterIds. store.setPolicyConfiguration(SetSubClusterPolicyConfigurationRequest .newInstance(getPriorityPolicy(queue1))); // second call is routed by new policy PriorityRouterPolicy chosen = routerFacade.getHomeSubcluster(applicationSubmissionContext, null); Assert.assertTrue(chosen.equals(subClusterIds.get(0))); Assert.assertTrue(routerFacade.globalPolicyMap .get(queue1) instanceof PriorityRouterPolicy); }
@Override protected void submitApplication( ApplicationSubmissionContext submissionContext, long submitTime, String user) throws YarnException { //Do nothing, just add the application to RMContext RMAppImpl application = new RMAppImpl(submissionContext.getApplicationId(), this.rmContext, this.conf, submissionContext.getApplicationName(), user, submissionContext.getQueue(), submissionContext, this.rmContext.getScheduler(), this.rmContext.getApplicationMasterService(), submitTime, submissionContext.getApplicationType(), submissionContext.getApplicationTags(), null); this.rmContext.getRMApps().put(submissionContext.getApplicationId(), application); //Do not send RMAppEventType.START event //so the state of Application will not reach to NEW_SAVING state. } }
SchedulerUtils.normalizeAndValidateRequest(amReq, scheduler.getMaximumResourceCapability(), submissionContext.getQueue(), scheduler, isRecovery, rmContext); } catch (InvalidResourceRequestException e) { LOG.warn("RM app submission failed in validating AM resource request"
@Override protected void submitApplication( ApplicationSubmissionContext submissionContext, long submitTime, String user) throws YarnException { //Do nothing, just add the application to RMContext RMAppImpl application = new RMAppImpl(submissionContext.getApplicationId(), this.rmContext, this.conf, submissionContext.getApplicationName(), user, submissionContext.getQueue(), submissionContext, this.rmContext.getScheduler(), this.rmContext.getApplicationMasterService(), submitTime, submissionContext.getApplicationType(), submissionContext.getApplicationTags(), null); this.rmContext.getRMApps().put(submissionContext.getApplicationId(), application); //Do not send RMAppEventType.START event //so the state of Application will not reach to NEW_SAVING state. } }
MockRM.finishAMAndVerifyAppState(app, rm1, nm1, am1); String fsQueueContext = app.getApplicationSubmissionContext().getQueue(); String fsQueueApp = app.getQueue(); assertEquals("Queue in app not equal to submission context", fsQueueApp, assertNotNull("No AppAttempt found after recovery", rmAttemptRecovered); String fsQueueContextRecovered = recoveredApp.getApplicationSubmissionContext().getQueue(); String fsQueueAppRecovered = recoveredApp.getQueue(); assertEquals(RMAppState.FINISHED, recoveredApp.getState());
private void pruneAppState(ApplicationStateData appState) { ApplicationSubmissionContext srcCtx = appState.getApplicationSubmissionContext(); ApplicationSubmissionContextPBImpl context = new ApplicationSubmissionContextPBImpl(); // most fields in the ApplicationSubmissionContext are not needed, // but the following few need to be present for recovery to succeed context.setApplicationId(srcCtx.getApplicationId()); context.setResource(srcCtx.getResource()); context.setQueue(srcCtx.getQueue()); context.setAMContainerResourceRequests( srcCtx.getAMContainerResourceRequests()); context.setApplicationName(srcCtx.getApplicationName()); context.setPriority(srcCtx.getPriority()); context.setApplicationTags(srcCtx.getApplicationTags()); context.setApplicationType(srcCtx.getApplicationType()); context.setUnmanagedAM(srcCtx.getUnmanagedAM()); context.setNodeLabelExpression(srcCtx.getNodeLabelExpression()); ContainerLaunchContextPBImpl amContainerSpec = new ContainerLaunchContextPBImpl(); amContainerSpec.setApplicationACLs( srcCtx.getAMContainerSpec().getApplicationACLs()); context.setAMContainerSpec(amContainerSpec); appState.setApplicationSubmissionContext(context); } }
rmContext, conf, submissionContext.getApplicationName(), null, submissionContext.getQueue(), submissionContext, scheduler, null, appState.getSubmitTime(), submissionContext.getApplicationType(), submissionContext.getApplicationTags(),
oldAppSubCtxt.getApplicationId(), oldAppSubCtxt.getApplicationName(), oldAppSubCtxt.getQueue(), oldAppSubCtxt.getPriority(), oldAppSubCtxt.getAMContainerSpec(),