/** * Move all apps in the set of queues to the parent plan queue's default * reservation queue in a synchronous fashion */ private void moveAppsInQueueSync(String expiredReservation, String defReservationQueue) { List<ApplicationAttemptId> activeApps = scheduler.getAppsInQueue(expiredReservation); if (activeApps.isEmpty()) { return; } for (ApplicationAttemptId app : activeApps) { // fallback to parent's default queue try { scheduler.moveApplication(app.getApplicationId(), defReservationQueue); } catch (YarnException e) { LOG.warn( "Encountered unexpected error during migration of application: {}" + " from reservation: {}", app, expiredReservation, e); } } }
/** * Move all apps in the set of queues to the parent plan queue's default * reservation queue in a synchronous fashion */ private void moveAppsInQueueSync(String expiredReservation, String defReservationQueue) { List<ApplicationAttemptId> activeApps = scheduler.getAppsInQueue(expiredReservation); if (activeApps.isEmpty()) { return; } for (ApplicationAttemptId app : activeApps) { // fallback to parent's default queue try { scheduler.moveApplication(app.getApplicationId(), defReservationQueue); } catch (YarnException e) { LOG.warn( "Encountered unexpected error during migration of application: {}" + " from reservation: {}", app, expiredReservation, e); } } }
/** * Move all apps in the set of queues to the parent plan queue's default * reservation queue in a synchronous fashion */ private void moveAppsInQueueSync(String expiredReservation, String defReservationQueue) { List<ApplicationAttemptId> activeApps = scheduler.getAppsInQueue(expiredReservation); if (activeApps.isEmpty()) { return; } for (ApplicationAttemptId app : activeApps) { // fallback to parent's default queue try { scheduler.moveApplication(app.getApplicationId(), defReservationQueue); } catch (YarnException e) { LOG.warn( "Encountered unexpected error during migration of application: {}" + " from reservation: {}", app, expiredReservation, e); } } }
/** * First sets entitlement of queues to zero to prevent new app submission. * Then move all apps in the set of queues to the parent plan queue's default * reservation queue if move is enabled. Finally cleanups the queue by killing * any apps (if move is disabled or move failed) and removing the queue */ protected void cleanupExpiredQueues(String planQueueName, boolean shouldMove, Set<String> toRemove, String defReservationQueue) { for (String expiredReservationId : toRemove) { try { // reduce entitlement to 0 String expiredReservation = getReservationQueueName(planQueueName, expiredReservationId); setQueueEntitlement(planQueueName, expiredReservation, 0.0f, 0.0f); if (shouldMove) { moveAppsInQueueSync(expiredReservation, defReservationQueue); } if (scheduler.getAppsInQueue(expiredReservation).size() > 0) { scheduler.killAllAppsInQueue(expiredReservation); LOG.info("Killing applications in queue: {}", expiredReservation); } else { scheduler.removeQueue(expiredReservation); LOG.info("Queue: " + expiredReservation + " removed"); } } catch (YarnException e) { LOG.warn("Exception while trying to expire reservation: {}", expiredReservationId, e); } } }
/** * First sets entitlement of queues to zero to prevent new app submission. * Then move all apps in the set of queues to the parent plan queue's default * reservation queue if move is enabled. Finally cleanups the queue by killing * any apps (if move is disabled or move failed) and removing the queue */ protected void cleanupExpiredQueues(String planQueueName, boolean shouldMove, Set<String> toRemove, String defReservationQueue) { for (String expiredReservationId : toRemove) { try { // reduce entitlement to 0 String expiredReservation = getReservationQueueName(planQueueName, expiredReservationId); setQueueEntitlement(planQueueName, expiredReservation, 0.0f, 0.0f); if (shouldMove) { moveAppsInQueueSync(expiredReservation, defReservationQueue); } if (scheduler.getAppsInQueue(expiredReservation).size() > 0) { scheduler.killAllAppsInQueue(expiredReservation); LOG.info("Killing applications in queue: {}", expiredReservation); } else { scheduler.removeQueue(expiredReservation); LOG.info("Queue: " + expiredReservation + " removed"); } } catch (YarnException e) { LOG.warn("Exception while trying to expire reservation: {}", expiredReservationId, e); } } }
getAppsInQueue(expiredReservation); int size = (appsInQueue == null ? 0 : appsInQueue.size()); if (size > 0) {
private static YarnScheduler mockYarnScheduler() { YarnScheduler yarnScheduler = mock(YarnScheduler.class); when(yarnScheduler.getMinimumResourceCapability()).thenReturn( Resources.createResource( YarnConfiguration.DEFAULT_RM_SCHEDULER_MINIMUM_ALLOCATION_MB)); when(yarnScheduler.getMaximumResourceCapability()).thenReturn( Resources.createResource( YarnConfiguration.DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_MB)); when(yarnScheduler.getAppsInQueue(QUEUE_1)).thenReturn( Arrays.asList(getApplicationAttemptId(101), getApplicationAttemptId(102))); when(yarnScheduler.getAppsInQueue(QUEUE_2)).thenReturn( Arrays.asList(getApplicationAttemptId(103))); ApplicationAttemptId attemptId = getApplicationAttemptId(1); when(yarnScheduler.getAppResourceUsageReport(attemptId)).thenReturn(null); ResourceCalculator rc = new DefaultResourceCalculator(); when(yarnScheduler.getResourceCalculator()).thenReturn(rc); return yarnScheduler; }
new ArrayList<List<ApplicationAttemptId>>(); for (String queue : queues) { List<ApplicationAttemptId> appsInQueue = scheduler.getAppsInQueue(queue); if (appsInQueue != null && !appsInQueue.isEmpty()) { queueAppLists.add(appsInQueue);
if (request.getIncludeApplications()) { List<ApplicationAttemptId> apps = scheduler.getAppsInQueue(request.getQueueName()); appReports = new ArrayList<ApplicationReport>(apps.size()); for (ApplicationAttemptId app : apps) {
new ArrayList<List<ApplicationAttemptId>>(); for (String queue : queues) { List<ApplicationAttemptId> appsInQueue = scheduler.getAppsInQueue(queue); if (appsInQueue != null && !appsInQueue.isEmpty()) { queueAppLists.add(appsInQueue);
if (request.getIncludeApplications()) { List<ApplicationAttemptId> apps = scheduler.getAppsInQueue(request.getQueueName()); appReports = new ArrayList<ApplicationReport>(apps.size()); for (ApplicationAttemptId app : apps) {
new ArrayList<List<ApplicationAttemptId>>(); for (String queue : queues) { List<ApplicationAttemptId> appsInQueue = scheduler.getAppsInQueue(queue); if (appsInQueue != null && !appsInQueue.isEmpty()) { queueAppLists.add(appsInQueue);
List<ApplicationAttemptId> appsInA1 = scheduler.getAppsInQueue("a1"); assertEquals(1, appsInA1.size()); List<ApplicationAttemptId> appsInA = scheduler.getAppsInQueue("a"); assertTrue(appsInA.contains(appAttemptId)); assertEquals(1, appsInA.size()); List<ApplicationAttemptId> appsInRoot = scheduler.getAppsInQueue("root"); assertTrue(appsInRoot.contains(appAttemptId)); assertEquals(1, appsInRoot.size()); List<ApplicationAttemptId> appsInB1 = scheduler.getAppsInQueue("b1"); assertTrue(appsInB1.isEmpty()); List<ApplicationAttemptId> appsInB = scheduler.getAppsInQueue("b"); assertTrue(appsInB.isEmpty()); appsInA1 = scheduler.getAppsInQueue("a1"); assertEquals(1, appsInA1.size()); appsInA = scheduler.getAppsInQueue("a"); assertTrue(appsInA.contains(appAttemptId)); assertEquals(1, appsInA.size()); appsInRoot = scheduler.getAppsInQueue("root"); assertTrue(appsInRoot.contains(appAttemptId)); assertEquals(1, appsInRoot.size()); appsInB1 = scheduler.getAppsInQueue("b1");
private void mockRMContext(YarnScheduler yarnScheduler, RMContext rmContext) throws IOException { Dispatcher dispatcher = mock(Dispatcher.class); when(rmContext.getDispatcher()).thenReturn(dispatcher); EventHandler eventHandler = mock(EventHandler.class); when(dispatcher.getEventHandler()).thenReturn(eventHandler); QueueInfo queInfo = recordFactory.newRecordInstance(QueueInfo.class); queInfo.setQueueName("testqueue"); when(yarnScheduler.getQueueInfo(eq("testqueue"), anyBoolean(), anyBoolean())) .thenReturn(queInfo); when(yarnScheduler.getQueueInfo(eq("nonexistentqueue"), anyBoolean(), anyBoolean())) .thenThrow(new IOException("queue does not exist")); RMApplicationHistoryWriter writer = mock(RMApplicationHistoryWriter.class); when(rmContext.getRMApplicationHistoryWriter()).thenReturn(writer); SystemMetricsPublisher publisher = mock(SystemMetricsPublisher.class); when(rmContext.getSystemMetricsPublisher()).thenReturn(publisher); when(rmContext.getYarnConfiguration()).thenReturn(new YarnConfiguration()); ConcurrentHashMap<ApplicationId, RMApp> apps = getRMApps(rmContext, yarnScheduler); when(rmContext.getRMApps()).thenReturn(apps); when(yarnScheduler.getAppsInQueue(eq("testqueue"))).thenReturn( getSchedulerApps(apps)); ResourceScheduler rs = mock(ResourceScheduler.class); when(rmContext.getScheduler()).thenReturn(rs); }
List<ApplicationAttemptId> appsInA1 = scheduler.getAppsInQueue("a1"); assertEquals(1, appsInA1.size()); List<ApplicationAttemptId> appsInA = scheduler.getAppsInQueue("a"); assertTrue(appsInA.contains(appAttemptId)); assertEquals(1, appsInA.size()); List<ApplicationAttemptId> appsInRoot = scheduler.getAppsInQueue("root"); assertTrue(appsInRoot.contains(appAttemptId)); assertEquals(1, appsInRoot.size()); List<ApplicationAttemptId> appsInB1 = scheduler.getAppsInQueue("b1"); assertTrue(appsInB1.isEmpty()); List<ApplicationAttemptId> appsInB = scheduler.getAppsInQueue("b"); assertTrue(appsInB.isEmpty()); appsInA1 = scheduler.getAppsInQueue("a1"); assertEquals(1, appsInA1.size()); appsInA = scheduler.getAppsInQueue("a"); assertTrue(appsInA.contains(appAttemptId)); assertEquals(1, appsInA.size()); appsInRoot = scheduler.getAppsInQueue("root"); assertTrue(appsInRoot.contains(appAttemptId)); assertEquals(1, appsInRoot.size()); appsInB1 = scheduler.getAppsInQueue("b1");
if (request.getIncludeApplications()) { List<ApplicationAttemptId> apps = scheduler.getAppsInQueue(request.getQueueName()); appReports = new ArrayList<ApplicationReport>(apps.size()); for (ApplicationAttemptId app : apps) {
List<ApplicationAttemptId> appsInA1 = scheduler.getAppsInQueue("a1"); assertEquals(1, appsInA1.size()); List<ApplicationAttemptId> appsInA = scheduler.getAppsInQueue("a"); assertTrue(appsInA.contains(appAttemptId)); assertEquals(1, appsInA.size()); List<ApplicationAttemptId> appsInRoot = scheduler.getAppsInQueue("root"); assertTrue(appsInRoot.contains(appAttemptId)); assertEquals(1, appsInRoot.size()); appsInA1 = scheduler.getAppsInQueue("a1"); assertEquals(1, appsInA1.size()); appsInA = scheduler.getAppsInQueue("a"); assertTrue(appsInA.contains(appAttemptId)); assertEquals(1, appsInA.size()); appsInRoot = scheduler.getAppsInQueue("root"); assertTrue(appsInRoot.contains(appAttemptId)); assertEquals(1, appsInRoot.size());