public static RMContext createRMContext(Configuration conf) { RMContext mockRmContext = Mockito.spy(new RMContextImpl(null, null, null, null, null, null, new RMContainerTokenSecretManager(conf), new NMTokenSecretManagerInRM(conf), new ClientToAMTokenSecretManagerInRM(), null)); RMNodeLabelsManager nlm = mock(RMNodeLabelsManager.class); when(nlm.getQueueResource(any(String.class), anySetOf(String.class), any(Resource.class))).thenAnswer(new Answer<Resource>() { @Override public Resource answer(InvocationOnMock invocation) throws Throwable { Object[] args = invocation.getArguments(); return (Resource) args[2]; } }); when(nlm.getResourceByLabel(any(String.class), any(Resource.class))) .thenAnswer(new Answer<Resource>() { @Override public Resource answer(InvocationOnMock invocation) throws Throwable { Object[] args = invocation.getArguments(); return (Resource) args[1]; } }); mockRmContext.setNodeLabelManager(nlm); mockRmContext .setMultiNodeSortingManager(mock(MultiNodeSortingManager.class)); return mockRmContext; }
private void resetRMContext() { RMContextImpl rmContextImpl = new RMContextImpl(); // transfer service context to new RM service Context rmContextImpl.setServiceContext(rmContext.getServiceContext()); // reset dispatcher Dispatcher dispatcher = setupDispatcher(); ((Service) dispatcher).init(this.conf); ((Service) dispatcher).start(); removeService((Service) rmDispatcher); // Need to stop previous rmDispatcher before assigning new dispatcher // otherwise causes "AsyncDispatcher event handler" thread leak ((Service) rmDispatcher).stop(); rmDispatcher = dispatcher; addIfService(rmDispatcher); rmContextImpl.setDispatcher(dispatcher); rmContext = rmContextImpl; }
private static FairScheduler mockFairScheduler() throws IOException { FairScheduler fs = new FairScheduler(); FairSchedulerConfiguration conf = new FairSchedulerConfiguration(); fs.setRMContext(new RMContextImpl(null, null, null, null, null, null, new RMContainerTokenSecretManager(conf), new NMTokenSecretManagerInRM(conf), new ClientToAMTokenSecretManagerInRM(), null)); fs.init(conf); return fs; }
private static FairScheduler mockFairScheduler() throws IOException { FairScheduler fs = new FairScheduler(); FairSchedulerConfiguration conf = new FairSchedulerConfiguration(); fs.setRMContext(new RMContextImpl(null, null, null, null, null, null, new RMContainerTokenSecretManager(conf), new NMTokenSecretManagerInRM(conf), new ClientToAMTokenSecretManagerInRM(), null)); fs.init(conf); return fs; }
@Test public void testQueueParsingWithLabels() throws IOException { nodeLabelManager.addToCluserNodeLabels(ImmutableSet.of("red", "blue")); YarnConfiguration conf = new YarnConfiguration(); CapacitySchedulerConfiguration csConf = new CapacitySchedulerConfiguration(conf); setupQueueConfigurationWithLabels(csConf); CapacityScheduler capacityScheduler = new CapacityScheduler(); RMContextImpl rmContext = new RMContextImpl(null, null, null, null, null, null, new RMContainerTokenSecretManager(csConf), new NMTokenSecretManagerInRM(csConf), new ClientToAMTokenSecretManagerInRM(), null); rmContext.setNodeLabelManager(nodeLabelManager); capacityScheduler.setConf(csConf); capacityScheduler.setRMContext(rmContext); capacityScheduler.init(csConf); capacityScheduler.start(); checkQueueLabels(capacityScheduler); ServiceOperations.stopQuietly(capacityScheduler); }
@Test public void testQueueParsingWithLabels() throws IOException { nodeLabelManager.addToCluserNodeLabelsWithDefaultExclusivity(ImmutableSet.of("red", "blue")); YarnConfiguration conf = new YarnConfiguration(); CapacitySchedulerConfiguration csConf = new CapacitySchedulerConfiguration(conf); setupQueueConfigurationWithLabels(csConf); CapacityScheduler capacityScheduler = new CapacityScheduler(); RMContextImpl rmContext = new RMContextImpl(null, null, null, null, null, null, new RMContainerTokenSecretManager(csConf), new NMTokenSecretManagerInRM(csConf), new ClientToAMTokenSecretManagerInRM(), null); rmContext.setNodeLabelManager(nodeLabelManager); capacityScheduler.setConf(csConf); capacityScheduler.setRMContext(rmContext); capacityScheduler.init(csConf); capacityScheduler.start(); checkQueueLabels(capacityScheduler); ServiceOperations.stopQuietly(capacityScheduler); }
@Test public void testQueueParsingWithLabelsInherit() throws IOException { nodeLabelManager.addToCluserNodeLabelsWithDefaultExclusivity(ImmutableSet.of("red", "blue")); YarnConfiguration conf = new YarnConfiguration(); CapacitySchedulerConfiguration csConf = new CapacitySchedulerConfiguration(conf); setupQueueConfigurationWithLabelsInherit(csConf); CapacityScheduler capacityScheduler = new CapacityScheduler(); RMContextImpl rmContext = new RMContextImpl(null, null, null, null, null, null, new RMContainerTokenSecretManager(csConf), new NMTokenSecretManagerInRM(csConf), new ClientToAMTokenSecretManagerInRM(), null); rmContext.setNodeLabelManager(nodeLabelManager); capacityScheduler.setConf(csConf); capacityScheduler.setRMContext(rmContext); capacityScheduler.init(csConf); capacityScheduler.start(); checkQueueLabelsInheritConfig(capacityScheduler); ServiceOperations.stopQuietly(capacityScheduler); }
public static CapacityScheduler mockCapacityScheduler() throws IOException { // stolen from TestCapacityScheduler CapacitySchedulerConfiguration conf = new CapacitySchedulerConfiguration(); setupQueueConfiguration(conf); CapacityScheduler cs = new CapacityScheduler(); cs.setConf(new YarnConfiguration()); RMContext rmContext = new RMContextImpl(null, null, null, null, null, null, new RMContainerTokenSecretManager(conf), new NMTokenSecretManagerInRM(conf), new ClientToAMTokenSecretManagerInRM(), null); rmContext.setNodeLabelManager(new NullRMNodeLabelsManager()); cs.setRMContext(rmContext); cs.init(conf); return cs; }
public static CapacityScheduler mockCapacityScheduler() throws IOException { // stolen from TestCapacityScheduler CapacitySchedulerConfiguration conf = new CapacitySchedulerConfiguration(); setupQueueConfiguration(conf); CapacityScheduler cs = new CapacityScheduler(); YarnConfiguration yarnConf = new YarnConfiguration(); cs.setConf(yarnConf); RMContext rmContext = new RMContextImpl(null, null, null, null, null, null, new RMContainerTokenSecretManager(conf), new NMTokenSecretManagerInRM(conf), new ClientToAMTokenSecretManagerInRM(), null); RMNodeLabelsManager labelManager = new NullRMNodeLabelsManager(); labelManager.init(yarnConf); rmContext.setNodeLabelManager(labelManager); cs.setRMContext(rmContext); cs.init(conf); return cs; }
@Test public void testQueueParsingReinitializeWithLabels() throws IOException { nodeLabelManager.addToCluserNodeLabelsWithDefaultExclusivity(ImmutableSet.of("red", "blue")); CapacitySchedulerConfiguration csConf = new CapacitySchedulerConfiguration(); setupQueueConfigurationWithoutLabels(csConf); YarnConfiguration conf = new YarnConfiguration(csConf); CapacityScheduler capacityScheduler = new CapacityScheduler(); RMContextImpl rmContext = new RMContextImpl(null, null, null, null, null, null, new RMContainerTokenSecretManager(conf), new NMTokenSecretManagerInRM(conf), new ClientToAMTokenSecretManagerInRM(), null); rmContext.setNodeLabelManager(nodeLabelManager); capacityScheduler.setConf(conf); capacityScheduler.setRMContext(rmContext); capacityScheduler.init(conf); capacityScheduler.start(); csConf = new CapacitySchedulerConfiguration(); setupQueueConfigurationWithLabels(csConf); conf = new YarnConfiguration(csConf); capacityScheduler.reinitialize(conf, rmContext); checkQueueLabels(capacityScheduler); ServiceOperations.stopQuietly(capacityScheduler); }
@Test public void testQueueParsingReinitializeWithLabels() throws IOException { nodeLabelManager.addToCluserNodeLabels(ImmutableSet.of("red", "blue")); CapacitySchedulerConfiguration csConf = new CapacitySchedulerConfiguration(); setupQueueConfigurationWithoutLabels(csConf); YarnConfiguration conf = new YarnConfiguration(csConf); CapacityScheduler capacityScheduler = new CapacityScheduler(); RMContextImpl rmContext = new RMContextImpl(null, null, null, null, null, null, new RMContainerTokenSecretManager(conf), new NMTokenSecretManagerInRM(conf), new ClientToAMTokenSecretManagerInRM(), null); rmContext.setNodeLabelManager(nodeLabelManager); capacityScheduler.setConf(conf); capacityScheduler.setRMContext(rmContext); capacityScheduler.init(conf); capacityScheduler.start(); csConf = new CapacitySchedulerConfiguration(); setupQueueConfigurationWithLabels(csConf); conf = new YarnConfiguration(csConf); capacityScheduler.reinitialize(conf, rmContext); checkQueueLabels(capacityScheduler); ServiceOperations.stopQuietly(capacityScheduler); }
/** Test that parseQueue throws an exception when two leaf queues have the * same name * @throws IOException */ @Test(expected=IOException.class) public void testParseQueue() throws IOException { CapacityScheduler cs = new CapacityScheduler(); cs.setConf(new YarnConfiguration()); cs.setRMContext(resourceManager.getRMContext()); CapacitySchedulerConfiguration conf = new CapacitySchedulerConfiguration(); setupQueueConfiguration(conf); cs.init(conf); cs.start(); conf.setQueues(CapacitySchedulerConfiguration.ROOT + ".a.a1", new String[] {"b1"} ); conf.setCapacity(CapacitySchedulerConfiguration.ROOT + ".a.a1.b1", 100.0f); conf.setUserLimitFactor(CapacitySchedulerConfiguration.ROOT + ".a.a1.b1", 100.0f); cs.reinitialize(conf, new RMContextImpl(null, null, null, null, null, null, new RMContainerTokenSecretManager(conf), new NMTokenSecretManagerInRM(conf), new ClientToAMTokenSecretManagerInRM(), null)); }
/** Test that parseQueue throws an exception when two leaf queues have the * same name * @throws IOException */ @Test(expected=IOException.class) public void testParseQueue() throws IOException { CapacityScheduler cs = new CapacityScheduler(); cs.setConf(new YarnConfiguration()); cs.setRMContext(resourceManager.getRMContext()); CapacitySchedulerConfiguration conf = new CapacitySchedulerConfiguration(); setupQueueConfiguration(conf); cs.init(conf); cs.start(); conf.setQueues(CapacitySchedulerConfiguration.ROOT + ".a.a1", new String[] {"b1"} ); conf.setCapacity(CapacitySchedulerConfiguration.ROOT + ".a.a1.b1", 100.0f); conf.setUserLimitFactor(CapacitySchedulerConfiguration.ROOT + ".a.a1.b1", 100.0f); cs.reinitialize(conf, new RMContextImpl(null, null, null, null, null, null, new RMContainerTokenSecretManager(conf), new NMTokenSecretManagerInRM(conf), new ClientToAMTokenSecretManagerInRM(), null)); }
@Test public void testConvertAutoCreateDisabledOnManagedParentQueueFails() throws Exception { CapacityScheduler newCS = new CapacityScheduler(); try { CapacitySchedulerConfiguration newConf = new CapacitySchedulerConfiguration(); setupQueueConfiguration(newConf); newConf.setAutoCreateChildQueueEnabled(C, false); newCS.setConf(new YarnConfiguration()); newCS.setRMContext(mockRM.getRMContext()); newCS.init(cs.getConf()); newCS.start(); newCS.reinitialize(newConf, new RMContextImpl(null, null, null, null, null, null, new RMContainerTokenSecretManager(newConf), new NMTokenSecretManagerInRM(newConf), new ClientToAMTokenSecretManagerInRM(), null)); } catch (IOException e) { //expected exception } finally { newCS.stop(); } }
@Test public void testQueueParsingWhenLabelsInheritedNotExistedInNodeLabelManager() throws IOException { YarnConfiguration conf = new YarnConfiguration(); CapacitySchedulerConfiguration csConf = new CapacitySchedulerConfiguration(conf); setupQueueConfigurationWithLabelsInherit(csConf); CapacityScheduler capacityScheduler = new CapacityScheduler(); RMContextImpl rmContext = new RMContextImpl(null, null, null, null, null, null, new RMContainerTokenSecretManager(csConf), new NMTokenSecretManagerInRM(csConf), new ClientToAMTokenSecretManagerInRM(), null); RMNodeLabelsManager nodeLabelsManager = new NullRMNodeLabelsManager(); nodeLabelsManager.init(conf); nodeLabelsManager.start(); rmContext.setNodeLabelManager(nodeLabelsManager); capacityScheduler.setConf(csConf); capacityScheduler.setRMContext(rmContext); capacityScheduler.init(csConf); capacityScheduler.start(); ServiceOperations.stopQuietly(capacityScheduler); ServiceOperations.stopQuietly(nodeLabelsManager); }
@Test public void testQueueParsingWhenLabelsNotExist() throws IOException { YarnConfiguration conf = new YarnConfiguration(); CapacitySchedulerConfiguration csConf = new CapacitySchedulerConfiguration(conf); setupQueueConfigurationWithLabels(csConf); CapacityScheduler capacityScheduler = new CapacityScheduler(); RMContextImpl rmContext = new RMContextImpl(null, null, null, null, null, null, new RMContainerTokenSecretManager(csConf), new NMTokenSecretManagerInRM(csConf), new ClientToAMTokenSecretManagerInRM(), null); RMNodeLabelsManager nodeLabelsManager = new NullRMNodeLabelsManager(); nodeLabelsManager.init(conf); nodeLabelsManager.start(); rmContext.setNodeLabelManager(nodeLabelsManager); capacityScheduler.setConf(csConf); capacityScheduler.setRMContext(rmContext); capacityScheduler.init(csConf); capacityScheduler.start(); ServiceOperations.stopQuietly(capacityScheduler); ServiceOperations.stopQuietly(nodeLabelsManager); }
@Test public void testRefreshQueues() throws Exception { CapacityScheduler cs = new CapacityScheduler(); CapacitySchedulerConfiguration conf = new CapacitySchedulerConfiguration(); RMContextImpl rmContext = new RMContextImpl(null, null, null, null, null, null, new RMContainerTokenSecretManager(conf), new NMTokenSecretManagerInRM(conf), new ClientToAMTokenSecretManagerInRM(), null); setupQueueConfiguration(conf); cs.setConf(new YarnConfiguration()); cs.setRMContext(resourceManager.getRMContext()); cs.init(conf); cs.start(); cs.reinitialize(conf, rmContext); checkQueueCapacities(cs, A_CAPACITY, B_CAPACITY); conf.setCapacity(A, 80f); conf.setCapacity(B, 20f); cs.reinitialize(conf, mockContext); checkQueueCapacities(cs, 80f, 20f); cs.stop(); }
@Test public void testQueueParsingWhenLabelsNotExistedInNodeLabelManager() throws IOException { YarnConfiguration conf = new YarnConfiguration(); CapacitySchedulerConfiguration csConf = new CapacitySchedulerConfiguration(conf); setupQueueConfigurationWithLabels(csConf); CapacityScheduler capacityScheduler = new CapacityScheduler(); RMContextImpl rmContext = new RMContextImpl(null, null, null, null, null, null, new RMContainerTokenSecretManager(csConf), new NMTokenSecretManagerInRM(csConf), new ClientToAMTokenSecretManagerInRM(), null); RMNodeLabelsManager nodeLabelsManager = new NullRMNodeLabelsManager(); nodeLabelsManager.init(conf); nodeLabelsManager.start(); rmContext.setNodeLabelManager(nodeLabelsManager); capacityScheduler.setConf(csConf); capacityScheduler.setRMContext(rmContext); capacityScheduler.init(csConf); capacityScheduler.start(); ServiceOperations.stopQuietly(capacityScheduler); ServiceOperations.stopQuietly(nodeLabelsManager); }
@Test public void testQueueParsingWhenLabelsInheritedNotExistedInNodeLabelManager() throws IOException { YarnConfiguration conf = new YarnConfiguration(); CapacitySchedulerConfiguration csConf = new CapacitySchedulerConfiguration(conf); setupQueueConfigurationWithLabelsInherit(csConf); CapacityScheduler capacityScheduler = new CapacityScheduler(); RMContextImpl rmContext = new RMContextImpl(null, null, null, null, null, null, new RMContainerTokenSecretManager(csConf), new NMTokenSecretManagerInRM(csConf), new ClientToAMTokenSecretManagerInRM(), null); RMNodeLabelsManager nodeLabelsManager = new NullRMNodeLabelsManager(); nodeLabelsManager.init(conf); nodeLabelsManager.start(); rmContext.setNodeLabelManager(nodeLabelsManager); capacityScheduler.setConf(csConf); capacityScheduler.setRMContext(rmContext); capacityScheduler.init(csConf); capacityScheduler.start(); ServiceOperations.stopQuietly(capacityScheduler); ServiceOperations.stopQuietly(nodeLabelsManager); }
@Test public void testRefreshQueues() throws Exception { CapacityScheduler cs = new CapacityScheduler(); CapacitySchedulerConfiguration conf = new CapacitySchedulerConfiguration(); RMContextImpl rmContext = new RMContextImpl(null, null, null, null, null, null, new RMContainerTokenSecretManager(conf), new NMTokenSecretManagerInRM(conf), new ClientToAMTokenSecretManagerInRM(), null); setupQueueConfiguration(conf); cs.setConf(new YarnConfiguration()); cs.setRMContext(resourceManager.getRMContext()); cs.init(conf); cs.start(); cs.reinitialize(conf, rmContext); checkQueueCapacities(cs, A_CAPACITY, B_CAPACITY); conf.setCapacity(A, 80f); conf.setCapacity(B, 20f); cs.reinitialize(conf, mockContext); checkQueueCapacities(cs, 80f, 20f); cs.stop(); }