@Override protected Resource getReservationQueueResourceIfExists(Plan plan, ReservationId reservationId) { String reservationQueueName = getReservationQueueName(plan.getQueueName(), reservationId.toString()); FSLeafQueue reservationQueue = fs.getQueueManager().getLeafQueue(reservationQueueName, false); Resource reservationResource = null; if (reservationQueue != null) { reservationResource = reservationQueue.getSteadyFairShare(); } return reservationResource; }
@Override protected Resource getReservationQueueResourceIfExists(Plan plan, ReservationId reservationId) { String reservationQueueName = getReservationQueueName(plan.getQueueName(), reservationId.toString()); FSLeafQueue reservationQueue = fs.getQueueManager().getLeafQueue(reservationQueueName, false); Resource reservationResource = null; if (reservationQueue != null) { reservationResource = reservationQueue.getSteadyFairShare(); } return reservationResource; }
@Override protected Resource getReservationQueueResourceIfExists(Plan plan, ReservationId reservationId) { String reservationQueueName = getReservationQueueName(plan.getQueueName(), reservationId.toString()); FSLeafQueue reservationQueue = fs.getQueueManager().getLeafQueue(reservationQueueName, false); Resource reservationResource = null; if (reservationQueue != null) { reservationResource = reservationQueue.getSteadyFairShare(); } return reservationResource; }
/** * Verify whether steady fair shares for all leaf queues still follow * their weight, not related to active/inactive status. * * @param leafQueues * @param nodeCapacity */ private void verifySteadyFairShareMemory(Collection<FSLeafQueue> leafQueues, int nodeCapacity) { for (FSLeafQueue leaf : leafQueues) { if (leaf.getName().startsWith("root.parentA")) { assertEquals(0.2, (double) leaf.getSteadyFairShare().getMemorySize() / nodeCapacity, 0.001); } else if (leaf.getName().startsWith("root.parentB")) { assertEquals(0.05, (double) leaf.getSteadyFairShare().getMemorySize() / nodeCapacity, 0.001); } } } }
/** * Verify whether steady fair shares for all leaf queues still follow * their weight, not related to active/inactive status. * * @param leafQueues * @param nodeCapacity */ private void verifySteadyFairShareMemory(Collection<FSLeafQueue> leafQueues, int nodeCapacity) { for (FSLeafQueue leaf : leafQueues) { if (leaf.getName().startsWith("root.parentA")) { assertEquals(0.2, (double) leaf.getSteadyFairShare().getMemory() / nodeCapacity, 0.001); } else if (leaf.getName().startsWith("root.parentB")) { assertEquals(0.05, (double) leaf.getSteadyFairShare().getMemory() / nodeCapacity, 0.001); } } } }
@Override protected void dumpStateInternal(StringBuilder sb) { sb.append("{Name: " + getName() + ", Weight: " + weights + ", Policy: " + policy.getName() + ", FairShare: " + getFairShare() + ", SteadyFairShare: " + getSteadyFairShare() + ", MaxShare: " + getMaxShare() + ", MinShare: " + minShare + ", ResourceUsage: " + getResourceUsage() + ", Demand: " + getDemand() + ", Runnable: " + getNumRunnableApps() + ", NumPendingApps: " + getNumPendingApps() + ", NonRunnable: " + getNumNonRunnableApps() + ", MaxAMShare: " + maxAMShare + ", MaxAMResource: " + computeMaxAMResource() + ", AMResourceUsage: " + getAmResourceUsage() + ", LastTimeAtMinShare: " + lastTimeAtMinShare + "}"); }
.getSteadyFairShare().getMemorySize()); assertEquals(0, queueManager.getLeafQueue("child2", false) .getSteadyFairShare().getMemorySize()); .getSteadyFairShare().getMemorySize()); assertEquals(2048, queueManager.getLeafQueue("child2", false) .getSteadyFairShare().getMemorySize()); .getSteadyFairShare().getMemorySize()); assertEquals(2048, queueManager.getLeafQueue("child2", false) .getSteadyFairShare().getMemorySize()); assertEquals(2048, queueManager.getLeafQueue("child3", false) .getSteadyFairShare().getMemorySize()); assertEquals(0, scheduler.getClusterResource().getMemorySize()); assertEquals(0, queueManager.getLeafQueue("child1", false) .getSteadyFairShare().getMemorySize()); assertEquals(0, queueManager.getLeafQueue("child2", false) .getSteadyFairShare().getMemorySize());
.getSteadyFairShare().getMemory()); assertEquals(0, queueManager.getLeafQueue("child2", false) .getSteadyFairShare().getMemory()); .getSteadyFairShare().getMemory()); assertEquals(2048, queueManager.getLeafQueue("child2", false) .getSteadyFairShare().getMemory()); .getSteadyFairShare().getMemory()); assertEquals(2048, queueManager.getLeafQueue("child2", false) .getSteadyFairShare().getMemory()); assertEquals(2048, queueManager.getLeafQueue("child3", false) .getSteadyFairShare().getMemory()); assertEquals(0, scheduler.getClusterResource().getMemory()); assertEquals(0, queueManager.getLeafQueue("child1", false) .getSteadyFairShare().getMemory()); assertEquals(0, queueManager.getLeafQueue("child2", false) .getSteadyFairShare().getMemory());
assertEquals(capacity / 4, leaf.getSteadyFairShare().getMemorySize());
assertEquals(capacity / 4, leaf.getSteadyFairShare().getMemory());
@Test public void testSteadyFairShareWithQueueCreatedRuntime() throws Exception { conf.setClass(CommonConfigurationKeys.HADOOP_SECURITY_GROUP_MAPPING, SimpleGroupsMapping.class, GroupMappingServiceProvider.class); conf.set(FairSchedulerConfiguration.USER_AS_DEFAULT_QUEUE, "true"); scheduler.init(conf); scheduler.start(); scheduler.reinitialize(conf, resourceManager.getRMContext()); // Add one node RMNode node1 = MockNodes .newNodeInfo(1, Resources.createResource(6144), 1, "127.0.0.1"); NodeAddedSchedulerEvent nodeEvent1 = new NodeAddedSchedulerEvent(node1); scheduler.handle(nodeEvent1); assertEquals(6144, scheduler.getClusterResource().getMemory()); assertEquals(6144, scheduler.getQueueManager().getRootQueue() .getSteadyFairShare().getMemory()); assertEquals(6144, scheduler.getQueueManager() .getLeafQueue("default", false).getSteadyFairShare().getMemory()); // Submit one application ApplicationAttemptId appAttemptId1 = createAppAttemptId(1, 1); createApplicationWithAMResource(appAttemptId1, "default", "user1", null); assertEquals(3072, scheduler.getQueueManager() .getLeafQueue("default", false).getSteadyFairShare().getMemory()); assertEquals(3072, scheduler.getQueueManager() .getLeafQueue("user1", false).getSteadyFairShare().getMemory()); }
@Test public void testSteadyFairShareWithQueueCreatedRuntime() throws Exception { conf.setClass(CommonConfigurationKeys.HADOOP_SECURITY_GROUP_MAPPING, SimpleGroupsMapping.class, GroupMappingServiceProvider.class); conf.set(FairSchedulerConfiguration.USER_AS_DEFAULT_QUEUE, "true"); scheduler.init(conf); scheduler.start(); scheduler.reinitialize(conf, resourceManager.getRMContext()); // Add one node RMNode node1 = MockNodes .newNodeInfo(1, Resources.createResource(6144), 1, "127.0.0.1"); NodeAddedSchedulerEvent nodeEvent1 = new NodeAddedSchedulerEvent(node1); scheduler.handle(nodeEvent1); assertEquals(6144, scheduler.getClusterResource().getMemorySize()); assertEquals(6144, scheduler.getQueueManager().getRootQueue() .getSteadyFairShare().getMemorySize()); assertEquals(6144, scheduler.getQueueManager() .getLeafQueue("default", false).getSteadyFairShare().getMemorySize()); // Submit one application ApplicationAttemptId appAttemptId1 = createAppAttemptId(1, 1); createApplicationWithAMResource(appAttemptId1, "default", "user1", null); assertEquals(3072, scheduler.getQueueManager() .getLeafQueue("default", false).getSteadyFairShare().getMemorySize()); assertEquals(3072, scheduler.getQueueManager() .getLeafQueue("user1", false).getSteadyFairShare().getMemorySize()); }
if (leaf.getName().startsWith("root.parentA")) { assertEquals(0.2, (double) leaf.getSteadyFairShare().getMemory() / nodeMem, 0.001); assertEquals(0.2, (double) leaf.getSteadyFairShare().getVirtualCores() / nodeVCores, 0.001); } else if (leaf.getName().startsWith("root.parentB")) { assertEquals(0.05, (double) leaf.getSteadyFairShare().getMemory() / nodeMem, 0.001); assertEquals(0.1, (double) leaf.getSteadyFairShare().getVirtualCores() / nodeVCores, 0.001);
if (leaf.getName().startsWith("root.parentA")) { assertEquals(0.2, (double) leaf.getSteadyFairShare().getMemorySize() / nodeMem, 0.001); assertEquals(0.2, (double) leaf.getSteadyFairShare().getVirtualCores() / nodeVCores, 0.001); } else if (leaf.getName().startsWith("root.parentB")) { assertEquals(0.05, (double) leaf.getSteadyFairShare().getMemorySize() / nodeMem, 0.001); assertEquals(0.1, (double) leaf.getSteadyFairShare().getVirtualCores() / nodeVCores, 0.001);
assertEquals(capacity / 2, queue1.getFairShare().getMemory()); assertEquals(capacity / 2, queue1.getMetrics().getFairShareMB()); assertEquals(capacity / 2, queue1.getSteadyFairShare().getMemory()); assertEquals(capacity / 2, queue1.getMetrics().getSteadyFairShareMB()); assertEquals(capacity / 4, queue2.getFairShare().getMemory()); assertEquals(capacity / 4, queue2.getMetrics().getFairShareMB()); assertEquals(capacity / 4, queue2.getSteadyFairShare().getMemory()); assertEquals(capacity / 4, queue2.getMetrics().getSteadyFairShareMB()); assertEquals(capacity / 4, queue3.getFairShare().getMemory()); assertEquals(capacity / 4, queue3.getMetrics().getFairShareMB()); assertEquals(capacity / 4, queue3.getSteadyFairShare().getMemory()); assertEquals(capacity / 4, queue3.getMetrics().getSteadyFairShareMB());
assertEquals(capacity / 2, queue1.getFairShare().getMemorySize()); assertEquals(capacity / 2, queue1.getMetrics().getFairShareMB()); assertEquals(capacity / 2, queue1.getSteadyFairShare().getMemorySize()); assertEquals(capacity / 2, queue1.getMetrics().getSteadyFairShareMB()); assertEquals(capacity / 4, queue2.getFairShare().getMemorySize()); assertEquals(capacity / 4, queue2.getMetrics().getFairShareMB()); assertEquals(capacity / 4, queue2.getSteadyFairShare().getMemorySize()); assertEquals(capacity / 4, queue2.getMetrics().getSteadyFairShareMB()); assertEquals(capacity / 4, queue3.getFairShare().getMemorySize()); assertEquals(capacity / 4, queue3.getMetrics().getFairShareMB()); assertEquals(capacity / 4, queue3.getSteadyFairShare().getMemorySize()); assertEquals(capacity / 4, queue3.getMetrics().getSteadyFairShareMB());
@Test public void testSimpleFairShareCalculation() throws IOException { scheduler.init(conf); scheduler.start(); scheduler.reinitialize(conf, resourceManager.getRMContext()); // Add one big node (only care about aggregate capacity) RMNode node1 = MockNodes.newNodeInfo(1, Resources.createResource(10 * 1024), 1, "127.0.0.1"); NodeAddedSchedulerEvent nodeEvent1 = new NodeAddedSchedulerEvent(node1); scheduler.handle(nodeEvent1); // Have two queues which want entire cluster capacity createSchedulingRequest(10 * 1024, "queue1", "user1"); createSchedulingRequest(10 * 1024, "queue2", "user1"); createSchedulingRequest(10 * 1024, "root.default", "user1"); scheduler.update(); scheduler.getQueueManager().getRootQueue() .setSteadyFairShare(scheduler.getClusterResource()); scheduler.getQueueManager().getRootQueue().recomputeSteadyShares(); Collection<FSLeafQueue> queues = scheduler.getQueueManager().getLeafQueues(); assertEquals(3, queues.size()); // Divided three ways - between the two queues and the default queue for (FSLeafQueue p : queues) { assertEquals(3414, p.getFairShare().getMemorySize()); assertEquals(3414, p.getMetrics().getFairShareMB()); assertEquals(3414, p.getSteadyFairShare().getMemorySize()); assertEquals(3414, p.getMetrics().getSteadyFairShareMB()); } }
@Test public void testSimpleFairShareCalculation() throws IOException { scheduler.init(conf); scheduler.start(); scheduler.reinitialize(conf, resourceManager.getRMContext()); // Add one big node (only care about aggregate capacity) RMNode node1 = MockNodes.newNodeInfo(1, Resources.createResource(10 * 1024), 1, "127.0.0.1"); NodeAddedSchedulerEvent nodeEvent1 = new NodeAddedSchedulerEvent(node1); scheduler.handle(nodeEvent1); // Have two queues which want entire cluster capacity createSchedulingRequest(10 * 1024, "queue1", "user1"); createSchedulingRequest(10 * 1024, "queue2", "user1"); createSchedulingRequest(10 * 1024, "root.default", "user1"); scheduler.update(); scheduler.getQueueManager().getRootQueue() .setSteadyFairShare(scheduler.getClusterResource()); scheduler.getQueueManager().getRootQueue().recomputeSteadyShares(); Collection<FSLeafQueue> queues = scheduler.getQueueManager().getLeafQueues(); assertEquals(3, queues.size()); // Divided three ways - between the two queues and the default queue for (FSLeafQueue p : queues) { assertEquals(3414, p.getFairShare().getMemory()); assertEquals(3414, p.getMetrics().getFairShareMB()); assertEquals(3414, p.getSteadyFairShare().getMemory()); assertEquals(3414, p.getMetrics().getSteadyFairShareMB()); } }