private void startRM() throws IOException, ClassNotFoundException { Configuration rmConf = new YarnConfiguration(); String schedulerClass = rmConf.get(YarnConfiguration.RM_SCHEDULER); // For CapacityScheduler we use a sub-classing instead of wrapping // to allow scheduler-specific invocations from monitors to work // this can be used for other schedulers as well if we care to // exercise/track behaviors that are not common to the scheduler api if(Class.forName(schedulerClass) == CapacityScheduler.class) { rmConf.set(YarnConfiguration.RM_SCHEDULER, SLSCapacityScheduler.class.getName()); } else { rmConf.set(YarnConfiguration.RM_SCHEDULER, ResourceSchedulerWrapper.class.getName()); rmConf.set(SLSConfiguration.RM_SCHEDULER, schedulerClass); } rmConf.set(SLSConfiguration.METRICS_OUTPUT_DIR, metricsOutputDir); rm = new ResourceManager(); rm.init(rmConf); rm.start(); }
@Before public void setup() throws IOException { createConfiguration(); writeAllocFile(30, 40); conf.set(FairSchedulerConfiguration.ALLOCATION_FILE, ALLOC_FILE.getAbsolutePath()); resourceManager = new MockRM(conf); resourceManager.start(); scheduler = (FairScheduler) resourceManager.getResourceScheduler(); }
private synchronized void startResourceManager(final int index) { try { resourceManagers[index].start(); if (resourceManagers[index].getServiceState() != STATE.STARTED) { // RM could have failed. throw new IOException( "ResourceManager failed to start. Final state is " + resourceManagers[index].getServiceState()); } } catch (Throwable t) { throw new YarnRuntimeException(t); } Configuration conf = resourceManagers[index].getConfig(); LOG.info("MiniYARN ResourceManager address: " + conf.get(YarnConfiguration.RM_ADDRESS)); LOG.info("MiniYARN ResourceManager web address: " + WebAppUtils.getRMWebAppURLWithoutScheme(conf)); }
private synchronized void startResourceManager(final int index) { try { resourceManagers[index].start(); if (resourceManagers[index].getServiceState() != STATE.STARTED) { // RM could have failed. throw new IOException( "ResourceManager failed to start. Final state is " + resourceManagers[index].getServiceState()); } } catch (Throwable t) { throw new YarnRuntimeException(t); } LOG.info("MiniYARN ResourceManager address: " + getConfig().get(YarnConfiguration.RM_ADDRESS)); LOG.info("MiniYARN ResourceManager web address: " + WebAppUtils.getRMWebAppURLWithoutScheme(getConfig())); }
@Before public void setup() { Configuration conf = createConfiguration(); resourceManager = new MockRM(conf); resourceManager.start(); scheduler = (FairScheduler) resourceManager.getResourceScheduler(); }
@Before public void setup() { Configuration conf = createConfiguration(); resourceManager = new MockRM(conf); resourceManager.start(); scheduler = (FairScheduler) resourceManager.getResourceScheduler(); }
@Before public void setUp() throws IOException { conf = createConfiguration(); resourceManager = new MockRM(conf); resourceManager.start(); scheduler = (FairScheduler) resourceManager.getResourceScheduler(); }
@Before public void setup() { conf = createConfiguration(); resourceManager = new MockRM(conf); resourceManager.start(); scheduler = (FairScheduler) resourceManager.getResourceScheduler(); }
@Test public void testUpdateDemand() { conf.set(FairSchedulerConfiguration.ASSIGN_MULTIPLE, "false"); resourceManager = new MockRM(conf); resourceManager.start(); scheduler = (FairScheduler) resourceManager.getResourceScheduler(); scheduler.allocConf = mock(AllocationConfiguration.class); String queueName = "root.queue1"; when(scheduler.allocConf.getMaxResources(queueName)).thenReturn(maxResource); when(scheduler.allocConf.getMinResources(queueName)).thenReturn(Resources.none()); FSLeafQueue schedulable = new FSLeafQueue(queueName, scheduler, null); FSAppAttempt app = mock(FSAppAttempt.class); Mockito.when(app.getDemand()).thenReturn(maxResource); schedulable.addAppSchedulable(app); schedulable.addAppSchedulable(app); schedulable.updateDemand(); assertTrue("Demand is greater than max allowed ", Resources.equals(schedulable.getDemand(), maxResource)); }
@Before public void setUp() throws Exception { super.setUp(); curatorTestingServer = setupCuratorServer(); curatorFramework = setupCuratorFramework(curatorTestingServer); conf.set(CommonConfigurationKeys.ZK_ADDRESS, curatorTestingServer.getConnectString()); rm = new MockRM(conf); rm.start(); rmContext = rm.getRMContext(); }
@Before public void setUp() throws Exception { conf = new YarnConfiguration(); conf.setClass(YarnConfiguration.RM_SCHEDULER, FifoSchedulerWithMove.class, FifoSchedulerWithMove.class); conf.set(YarnConfiguration.YARN_ADMIN_ACL, " "); conf.setBoolean(YarnConfiguration.YARN_ACL_ENABLE, true); resourceManager = new MockRM(conf); resourceManager.getRMContext().getContainerTokenSecretManager().rollMasterKey(); resourceManager.getRMContext().getNMTokenSecretManager().rollMasterKey(); resourceManager.start(); failMove = false; }
private void startResourceManager(float utilizationThreshold) { conf.setFloat(FairSchedulerConfiguration.PREEMPTION_THRESHOLD, utilizationThreshold); resourceManager = new MockRM(conf); resourceManager.start(); assertTrue( resourceManager.getResourceScheduler() instanceof StubbedFairScheduler); scheduler = (FairScheduler)resourceManager.getResourceScheduler(); scheduler.setClock(clock); scheduler.updateInterval = 60 * 1000; }
@Test public void testServiceAclsRefreshWithLocalConfigurationProvider() { configuration.setBoolean( CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHORIZATION, true); ResourceManager resourceManager = null; try { resourceManager = new ResourceManager(); resourceManager.init(configuration); resourceManager.start(); resourceManager.adminService.refreshServiceAcls(RefreshServiceAclsRequest .newInstance()); } catch (Exception ex) { fail("Using localConfigurationProvider. Should not get any exception."); } finally { if (resourceManager != null) { resourceManager.stop(); } } }
@Before public void setUp() throws Exception { conf = new YarnConfiguration(); conf.setClass(YarnConfiguration.RM_SCHEDULER, FifoSchedulerWithMove.class, FifoSchedulerWithMove.class); conf.set(YarnConfiguration.YARN_ADMIN_ACL, " "); conf.setBoolean(YarnConfiguration.YARN_ACL_ENABLE, true); resourceManager = new ResourceManager(); resourceManager.init(conf); resourceManager.getRMContext().getContainerTokenSecretManager().rollMasterKey(); resourceManager.getRMContext().getNMTokenSecretManager().rollMasterKey(); resourceManager.start(); failMove = false; }
@Test public void testServiceAclsRefreshWithLocalConfigurationProvider() { configuration.setBoolean( CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHORIZATION, true); ResourceManager resourceManager = null; try { resourceManager = new ResourceManager(); resourceManager.init(configuration); resourceManager.start(); resourceManager.adminService.refreshServiceAcls(RefreshServiceAclsRequest .newInstance()); } catch (Exception ex) { fail("Using localConfigurationProvider. Should not get any exception."); } finally { if (resourceManager != null) { resourceManager.stop(); } } }
@Test public void testUpdateDemand() { conf.set(FairSchedulerConfiguration.ASSIGN_MULTIPLE, "false"); resourceManager = new MockRM(conf); resourceManager.start(); scheduler = (FairScheduler) resourceManager.getResourceScheduler(); String queueName = "root.queue1"; FSLeafQueue schedulable = new FSLeafQueue(queueName, scheduler, null); schedulable.setMaxShare(new ConfigurableResource(maxResource)); assertEquals(schedulable.getMetrics().getMaxApps(), Integer.MAX_VALUE); assertEquals(schedulable.getMetrics().getSchedulingPolicy(), SchedulingPolicy.DEFAULT_POLICY.getName()); FSAppAttempt app = mock(FSAppAttempt.class); Mockito.when(app.getDemand()).thenReturn(maxResource); Mockito.when(app.getResourceUsage()).thenReturn(Resources.none()); schedulable.addApp(app, true); schedulable.addApp(app, true); schedulable.updateDemand(); assertTrue("Demand is greater than max allowed ", Resources.equals(schedulable.getDemand(), maxResource)); }
@Before public void setup() { mockClock = new MockClock(); conf = createConfiguration(); resourceManager = new MockRM(conf); resourceManager.start(); scheduler = (FairScheduler) resourceManager.getResourceScheduler(); scheduler.setClock(mockClock); assertTrue(scheduler.isContinuousSchedulingEnabled()); assertEquals( FairSchedulerConfiguration.DEFAULT_CONTINUOUS_SCHEDULING_SLEEP_MS, scheduler.getContinuousSchedulingSleepMs()); assertEquals(mockClock, scheduler.getClock()); }
@SuppressWarnings("deprecation") @Before public void setup() { mockClock = new ControlledClock(); conf = createConfiguration(); resourceManager = new MockRM(conf); resourceManager.start(); scheduler = (FairScheduler) resourceManager.getResourceScheduler(); scheduler.setClock(mockClock); assertTrue(scheduler.isContinuousSchedulingEnabled()); assertEquals( FairSchedulerConfiguration.DEFAULT_CONTINUOUS_SCHEDULING_SLEEP_MS, scheduler.getContinuousSchedulingSleepMs()); assertEquals(mockClock, scheduler.getClock()); }
@Test public void testNotUserAsDefaultQueue() throws Exception { // Restarting resource manager since the Conf object is changed changed. resourceManager.stop(); conf.set(FairSchedulerConfiguration.USER_AS_DEFAULT_QUEUE, "false"); resourceManager = new MockRM(conf); resourceManager.start(); scheduler = (FairScheduler) resourceManager.getResourceScheduler(); ApplicationAttemptId appAttemptId = createAppAttemptId(1, 1); createApplicationWithAMResource(appAttemptId, "default", "user2", null); assertEquals(0, scheduler.getQueueManager().getLeafQueue("user1", true) .getNumRunnableApps()); assertEquals(1, scheduler.getQueueManager().getLeafQueue("default", true) .getNumRunnableApps()); assertEquals(0, scheduler.getQueueManager().getLeafQueue("user2", true) .getNumRunnableApps()); }
private void setupCluster() throws IOException { resourceManager = new MockRM(conf); scheduler = (FairScheduler) resourceManager.getResourceScheduler(); // YARN-6249, FSLeafQueue#lastTimeAtMinShare is initialized to the time in // the real world, so we should keep the clock up with it. clock.setTime(SystemClock.getInstance().getTime()); scheduler.setClock(clock); resourceManager.start(); // Create and add two nodes to the cluster, with capacities // disproportional to the container requests. addNode(NODE_CAPACITY_MULTIPLE * GB, 3 * NODE_CAPACITY_MULTIPLE); addNode(NODE_CAPACITY_MULTIPLE * GB, 3 * NODE_CAPACITY_MULTIPLE); // Reinitialize the scheduler so DRF policy picks up cluster capacity // TODO (YARN-6194): One shouldn't need to call this scheduler.reinitialize(conf, resourceManager.getRMContext()); // Verify if child-1 and child-2 are preemptable FSQueue child1 = scheduler.getQueueManager().getQueue("nonpreemptable.child-1"); assertFalse(child1.isPreemptable()); FSQueue child2 = scheduler.getQueueManager().getQueue("nonpreemptable.child-2"); assertFalse(child2.isPreemptable()); }