public synchronized static QueueMetrics forQueue(String queueName, Queue parent, boolean enableUserMetrics, Configuration conf) { return forQueue(DefaultMetricsSystem.instance(), queueName, parent, enableUserMetrics, conf); }
public synchronized static QueueMetrics forQueue(String queueName, Queue parent, boolean enableUserMetrics, Configuration conf) { return forQueue(DefaultMetricsSystem.instance(), queueName, parent, enableUserMetrics, conf); }
public synchronized static QueueMetrics forQueue(String queueName, Queue parent, boolean enableUserMetrics, Configuration conf) { return forQueue(DefaultMetricsSystem.instance(), queueName, parent, enableUserMetrics, conf); }
private synchronized void initScheduler(Configuration conf) { validateConf(conf); //Use ConcurrentSkipListMap because applications need to be ordered this.applications = new ConcurrentSkipListMap<ApplicationId, SchedulerApplication<FiCaSchedulerApp>>(); this.minimumAllocation = Resources.createResource(conf.getInt( YarnConfiguration.RM_SCHEDULER_MINIMUM_ALLOCATION_MB, YarnConfiguration.DEFAULT_RM_SCHEDULER_MINIMUM_ALLOCATION_MB)); initMaximumResourceCapability( Resources.createResource(conf.getInt( YarnConfiguration.RM_SCHEDULER_MAXIMUM_ALLOCATION_MB, YarnConfiguration.DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_MB), conf.getInt( YarnConfiguration.RM_SCHEDULER_MAXIMUM_ALLOCATION_VCORES, YarnConfiguration.DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_VCORES))); this.usePortForNodeName = conf.getBoolean( YarnConfiguration.RM_SCHEDULER_INCLUDE_PORT_IN_NODE_NAME, YarnConfiguration.DEFAULT_RM_SCHEDULER_USE_PORT_FOR_NODE_NAME); this.metrics = QueueMetrics.forQueue(DEFAULT_QUEUE_NAME, null, false, conf); this.activeUsersManager = new ActiveUsersManager(metrics); }
private synchronized void initScheduler(Configuration conf) { validateConf(conf); //Use ConcurrentSkipListMap because applications need to be ordered this.applications = new ConcurrentSkipListMap<ApplicationId, SchedulerApplication<FiCaSchedulerApp>>(); this.minimumAllocation = Resources.createResource(conf.getInt( YarnConfiguration.RM_SCHEDULER_MINIMUM_ALLOCATION_MB, YarnConfiguration.DEFAULT_RM_SCHEDULER_MINIMUM_ALLOCATION_MB)); initMaximumResourceCapability( Resources.createResource(conf.getInt( YarnConfiguration.RM_SCHEDULER_MAXIMUM_ALLOCATION_MB, YarnConfiguration.DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_MB), conf.getInt( YarnConfiguration.RM_SCHEDULER_MAXIMUM_ALLOCATION_VCORES, YarnConfiguration.DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_VCORES))); this.usePortForNodeName = conf.getBoolean( YarnConfiguration.RM_SCHEDULER_INCLUDE_PORT_IN_NODE_NAME, YarnConfiguration.DEFAULT_RM_SCHEDULER_USE_PORT_FOR_NODE_NAME); this.metrics = QueueMetrics.forQueue(DEFAULT_QUEUE_NAME, null, false, conf); this.activeUsersManager = new ActiveUsersManager(metrics); }
private synchronized void initScheduler(Configuration conf) { validateConf(conf); //Use ConcurrentSkipListMap because applications need to be ordered this.applications = new ConcurrentSkipListMap<>(); this.minimumAllocation = super.getMinimumAllocation(); initMaximumResourceCapability(super.getMaximumAllocation()); this.usePortForNodeName = conf.getBoolean( YarnConfiguration.RM_SCHEDULER_INCLUDE_PORT_IN_NODE_NAME, YarnConfiguration.DEFAULT_RM_SCHEDULER_USE_PORT_FOR_NODE_NAME); this.metrics = QueueMetrics.forQueue(DEFAULT_QUEUE_NAME, null, false, conf); this.activeUsersManager = new ActiveUsersManager(metrics); }
private Queue createQueue(String name, Queue parent) { QueueMetrics metrics = QueueMetrics.forQueue(name, parent, false, conf); ActiveUsersManager activeUsersManager = new ActiveUsersManager(metrics); Queue queue = mock(Queue.class); when(queue.getMetrics()).thenReturn(metrics); when(queue.getActiveUsersManager()).thenReturn(activeUsersManager); return queue; }
private Queue createQueue(String name, Queue parent, float capacity) { QueueMetrics metrics = QueueMetrics.forQueue(name, parent, false, conf); QueueInfo queueInfo = QueueInfo.newInstance(name, capacity, 1.0f, 0, null, null, QueueState.RUNNING, null, "", null, false, null, false); ActiveUsersManager activeUsersManager = new ActiveUsersManager(metrics); Queue queue = mock(Queue.class); when(queue.getMetrics()).thenReturn(metrics); when(queue.getAbstractUsersManager()).thenReturn(activeUsersManager); when(queue.getQueueInfo(false, false)).thenReturn(queueInfo); return queue; }
@Test(timeout = 5000) public void testManyRuns() { QueueMetrics qm = QueueMetrics.forQueue(metricsSystem, "root", null, false, conf); qm.setAvailableResourcesToQueue(RMNodeLabelsManager.NO_LABEL, Resource.newInstance(1, 1)); int numIterations = 1000; long start = System.currentTimeMillis(); for (int i = 0; i < numIterations; i++) { ic.editSchedule(); } long end = System.currentTimeMillis(); System.out.println("Runtime per iteration (avg of " + numIterations + " iterations): " + (end - start) + " tot time"); }
@Test public void testMetricsCache() { MetricsSystem ms = new MetricsSystemImpl("cache"); ms.start(); try { String p1 = "root1"; String leafQueueName = "root1.leaf"; QueueMetrics p1Metrics = QueueMetrics.forQueue(ms, p1, null, true, conf); Queue parentQueue1 = mock(Queue.class); when(parentQueue1.getMetrics()).thenReturn(p1Metrics); QueueMetrics metrics = QueueMetrics.forQueue(ms, leafQueueName, parentQueue1, true, conf); Assert.assertNotNull("QueueMetrics for A shoudn't be null", metrics); // Re-register to check for cache hit, shouldn't blow up metrics-system... // also, verify parent-metrics QueueMetrics alterMetrics = QueueMetrics.forQueue(ms, leafQueueName, parentQueue1, true, conf); Assert.assertNotNull("QueueMetrics for alterMetrics shoudn't be null", alterMetrics); } finally { ms.shutdown(); } }
@Test public void testViolation() { // create a "wrong" condition in which the invariants are not respected QueueMetrics qm = QueueMetrics.forQueue(metricsSystem, "root", null, false, conf); qm.setAvailableResourcesToQueue(RMNodeLabelsManager.NO_LABEL, Resource.newInstance(-1, -1)); // test with throwing exception turned on try { ic.editSchedule(); fail(); } catch (InvariantViolationException i) { // expected } // test log-only mode conf.setBoolean(MetricsInvariantChecker.THROW_ON_VIOLATION, false); ic.init(conf, null, null); ic.editSchedule(); }
@Test public void testCollectAllMetrics() { String queueName = "single"; QueueMetrics.forQueue(ms, queueName, null, false, conf); MetricsSource queueSource = queueSource(ms, queueName); checkApps(queueSource, 0, 0, 0, 0, 0, 0, true); try { // do not collect all metrics checkApps(queueSource, 0, 0, 0, 0, 0, 0, false); Assert.fail(); } catch (AssertionError e) { Assert.assertTrue(e.getMessage().contains( "Expected exactly one metric for name ")); } // collect all metrics checkApps(queueSource, 0, 0, 0, 0, 0, 0, true); }
@Test public void testCollectAllMetrics() { String queueName = "single"; QueueMetrics.forQueue(ms, queueName, null, false, conf); MetricsSource queueSource = queueSource(ms, queueName); checkApps(queueSource, 0, 0, 0, 0, 0, 0, true); try { // do not collect all metrics checkApps(queueSource, 0, 0, 0, 0, 0, 0, false); Assert.fail(); } catch (AssertionError e) { Assert.assertTrue(e.getMessage().contains( "Expected exactly one metric for name ")); } // collect all metrics checkApps(queueSource, 0, 0, 0, 0, 0, 0, true); }
@Test public void testMetricsCache() { MetricsSystem ms = new MetricsSystemImpl("cache"); ms.start(); try { String p1 = "root1"; String leafQueueName = "root1.leaf"; QueueMetrics p1Metrics = QueueMetrics.forQueue(ms, p1, null, true, conf); Queue parentQueue1 = make(stub(Queue.class).returning(p1Metrics). from.getMetrics()); QueueMetrics metrics = QueueMetrics.forQueue(ms, leafQueueName, parentQueue1, true, conf); Assert.assertNotNull("QueueMetrics for A shoudn't be null", metrics); // Re-register to check for cache hit, shouldn't blow up metrics-system... // also, verify parent-metrics QueueMetrics alterMetrics = QueueMetrics.forQueue(ms, leafQueueName, parentQueue1, true, conf); Assert.assertNotNull("QueueMetrics for alterMetrics shoudn't be null", alterMetrics); } finally { ms.shutdown(); } }
public AbstractCSQueue(CapacitySchedulerContext cs, String queueName, CSQueue parent, CSQueue old) throws IOException { this.labelManager = cs.getRMContext().getNodeLabelManager(); this.parent = parent; this.queueName = queueName; this.resourceCalculator = cs.getResourceCalculator(); // must be called after parent and queueName is set this.metrics = old != null ? old.getMetrics() : QueueMetrics.forQueue(getQueuePath(), parent, cs.getConfiguration().getEnableUserMetrics(), cs.getConf()); this.csContext = cs; this.minimumAllocation = csContext.getMinimumResourceCapability(); // initialize ResourceUsage queueUsage = new ResourceUsage(); queueEntity = new PrivilegedEntity(EntityType.QUEUE, getQueuePath()); // initialize QueueCapacities queueCapacities = new QueueCapacities(parent == null); }
public AbstractCSQueue(CapacitySchedulerContext cs, String queueName, CSQueue parent, CSQueue old) throws IOException { this.labelManager = cs.getRMContext().getNodeLabelManager(); this.parent = parent; this.queueName = queueName; this.resourceCalculator = cs.getResourceCalculator(); // must be called after parent and queueName is set this.metrics = old != null ? old.getMetrics() : QueueMetrics.forQueue(getQueuePath(), parent, cs.getConfiguration().getEnableUserMetrics(), cs.getConf()); this.csContext = cs; this.minimumAllocation = csContext.getMinimumResourceCapability(); // initialize ResourceUsage queueUsage = new ResourceUsage(); queueEntity = new PrivilegedEntity(EntityType.QUEUE, getQueuePath()); // initialize QueueCapacities queueCapacities = new QueueCapacities(parent == null); }
QueueMetrics.forQueue(ms, parentQueueName, null, true, conf); Queue parentQueue = mock(Queue.class); when(parentQueue.getMetrics()).thenReturn(parentMetrics); QueueMetrics metrics = QueueMetrics.forQueue(ms, leafQueueName, parentQueue, true, conf); MetricsSource parentQueueSource = queueSource(ms, parentQueueName); MetricsSource queueSource = queueSource(ms, leafQueueName);
String user = "alice"; QueueMetrics metrics = QueueMetrics.forQueue(ms, queueName, null, false, new Configuration()); MetricsSource queueSource = queueSource(ms, queueName);
String user = "alice"; QueueMetrics metrics = QueueMetrics.forQueue(ms, queueName, null, false, new Configuration()); MetricsSource queueSource = queueSource(ms, queueName);
String user = "alice"; QueueMetrics metrics = QueueMetrics.forQueue(ms, queueName, null, false, conf); MetricsSource queueSource= queueSource(ms, queueName);