requestedQueue = queuePolicy.assignAppToQueue(requestedQueue, userName); if (StringUtils.isNotBlank(requestedQueue)) { LOG.debug("Setting queue name to " + requestedQueue + " for user "
private QueuePlacementPolicy getQueuePlacementPolicy( AllocationFileParser allocationFileParser, QueueProperties queueProperties, Configuration conf) throws AllocationConfigurationException { if (allocationFileParser.getQueuePlacementPolicy().isPresent()) { return QueuePlacementPolicy.fromXml( allocationFileParser.getQueuePlacementPolicy().get(), queueProperties.getConfiguredQueues(), conf); } else { return QueuePlacementPolicy.fromConfiguration(conf, queueProperties.getConfiguredQueues()); } }
/** * Builds a QueuePlacementPolicy from an xml element. */ public static QueuePlacementPolicy fromXml(Element el, Map<FSQueueType, Set<String>> configuredQueues, Configuration conf) throws AllocationConfigurationException { List<QueuePlacementRule> rules = new ArrayList<QueuePlacementRule>(); NodeList elements = el.getChildNodes(); for (int i = 0; i < elements.getLength(); i++) { Node node = elements.item(i); if (node instanceof Element) { QueuePlacementRule rule = createAndInitializeRule(node); rules.add(rule); } } return new QueuePlacementPolicy(rules, configuredQueues, conf); }
/** * Build a simple queue placement policy from the allow-undeclared-pools and * user-as-default-queue configuration options. */ public static QueuePlacementPolicy fromConfiguration(Configuration conf, Map<FSQueueType, Set<String>> configuredQueues) { boolean create = conf.getBoolean( FairSchedulerConfiguration.ALLOW_UNDECLARED_POOLS, FairSchedulerConfiguration.DEFAULT_ALLOW_UNDECLARED_POOLS); boolean userAsDefaultQueue = conf.getBoolean( FairSchedulerConfiguration.USER_AS_DEFAULT_QUEUE, FairSchedulerConfiguration.DEFAULT_USER_AS_DEFAULT_QUEUE); List<QueuePlacementRule> rules = new ArrayList<QueuePlacementRule>(); rules.add(new QueuePlacementRule.Specified().initialize(create, null)); if (userAsDefaultQueue) { rules.add(new QueuePlacementRule.User().initialize(create, null)); } if (!userAsDefaultQueue || !create) { rules.add(new QueuePlacementRule.Default().initialize(true, null)); } try { return new QueuePlacementPolicy(rules, configuredQueues, conf); } catch (AllocationConfigurationException ex) { throw new RuntimeException("Should never hit exception when loading" + "placement policy from conf", ex); } }
private QueuePlacementPolicy parse(String str) throws Exception { // Read and parse the allocations file. DocumentBuilderFactory docBuilderFactory = DocumentBuilderFactory .newInstance(); docBuilderFactory.setIgnoringComments(true); DocumentBuilder builder = docBuilderFactory.newDocumentBuilder(); Document doc = builder.parse(IOUtils.toInputStream(str)); Element root = doc.getDocumentElement(); return QueuePlacementPolicy.fromXml(root, configuredQueues, conf); } }
public AllocationConfiguration(Configuration conf) { minQueueResources = new HashMap<String, Resource>(); maxQueueResources = new HashMap<String, Resource>(); queueWeights = new HashMap<String, ResourceWeights>(); queueMaxApps = new HashMap<String, Integer>(); userMaxApps = new HashMap<String, Integer>(); queueMaxAMShares = new HashMap<String, Float>(); userMaxAppsDefault = Integer.MAX_VALUE; queueMaxAppsDefault = Integer.MAX_VALUE; queueMaxAMShareDefault = 0.5f; queueAcls = new HashMap<String, Map<QueueACL, AccessControlList>>(); minSharePreemptionTimeouts = new HashMap<String, Long>(); fairSharePreemptionTimeouts = new HashMap<String, Long>(); fairSharePreemptionThresholds = new HashMap<String, Float>(); schedulingPolicies = new HashMap<String, SchedulingPolicy>(); defaultSchedulingPolicy = SchedulingPolicy.DEFAULT_POLICY; reservableQueues = new HashSet<>(); configuredQueues = new HashMap<FSQueueType, Set<String>>(); for (FSQueueType queueType : FSQueueType.values()) { configuredQueues.put(queueType, new HashSet<String>()); } placementPolicy = QueuePlacementPolicy.fromConfiguration(conf, configuredQueues); }
if ("rule".equals(element.getTagName())) { QueuePlacementRule rule = QueuePlacementPolicy .createAndInitializeRule(node); if (rule == null) { throw new AllocationConfigurationException(
@Test public void testDefaultRuleInitializesProperlyWhenPolicyNotConfigured() throws IOException { // This test verifies if default rule in queue placement policy // initializes properly when policy is not configured and // undeclared pools is not allowed. conf.set(FairSchedulerConfiguration.ALLOCATION_FILE, ALLOC_FILE); conf.setBoolean(FairSchedulerConfiguration.ALLOW_UNDECLARED_POOLS, false); // Create an alloc file with no queue placement policy PrintWriter out = new PrintWriter(new FileWriter(ALLOC_FILE)); out.println("<?xml version=\"1.0\"?>"); out.println("<allocations>"); out.println("</allocations>"); out.close(); scheduler.init(conf); scheduler.start(); scheduler.reinitialize(conf, resourceManager.getRMContext()); List<QueuePlacementRule> rules = scheduler.allocConf.placementPolicy.getRules(); for (QueuePlacementRule rule : rules) { if (rule instanceof Default) { Default defaultRule = (Default) rule; assertNotNull(defaultRule.defaultQueueName); } } }
newPlacementPolicy = QueuePlacementPolicy.fromXml(placementPolicyElement, configuredQueues, conf); } else { newPlacementPolicy = QueuePlacementPolicy.fromConfiguration(conf, configuredQueues);
/** * Builds a QueuePlacementPolicy from an xml element. */ public static QueuePlacementPolicy fromXml(Element el, Map<FSQueueType, Set<String>> configuredQueues, Configuration conf) throws AllocationConfigurationException { List<QueuePlacementRule> rules = new ArrayList<QueuePlacementRule>(); NodeList elements = el.getChildNodes(); for (int i = 0; i < elements.getLength(); i++) { Node node = elements.item(i); if (node instanceof Element) { QueuePlacementRule rule = createAndInitializeRule(node); rules.add(rule); } } return new QueuePlacementPolicy(rules, configuredQueues, conf); }
/** * Build a simple queue placement policy from the allow-undeclared-pools and * user-as-default-queue configuration options. */ public static QueuePlacementPolicy fromConfiguration(Configuration conf, Map<FSQueueType, Set<String>> configuredQueues) { boolean create = conf.getBoolean( FairSchedulerConfiguration.ALLOW_UNDECLARED_POOLS, FairSchedulerConfiguration.DEFAULT_ALLOW_UNDECLARED_POOLS); boolean userAsDefaultQueue = conf.getBoolean( FairSchedulerConfiguration.USER_AS_DEFAULT_QUEUE, FairSchedulerConfiguration.DEFAULT_USER_AS_DEFAULT_QUEUE); List<QueuePlacementRule> rules = new ArrayList<QueuePlacementRule>(); rules.add(new QueuePlacementRule.Specified().initialize(create, null)); if (userAsDefaultQueue) { rules.add(new QueuePlacementRule.User().initialize(create, null)); } if (!userAsDefaultQueue || !create) { rules.add(new QueuePlacementRule.Default().initialize(true, null)); } try { return new QueuePlacementPolicy(rules, configuredQueues, conf); } catch (AllocationConfigurationException ex) { throw new RuntimeException("Should never hit exception when loading" + "placement policy from conf", ex); } }
private QueuePlacementPolicy parse(String str) throws Exception { // Read and parse the allocations file. DocumentBuilderFactory docBuilderFactory = DocumentBuilderFactory .newInstance(); docBuilderFactory.setIgnoringComments(true); DocumentBuilder builder = docBuilderFactory.newDocumentBuilder(); Document doc = builder.parse(IOUtils.toInputStream(str)); Element root = doc.getDocumentElement(); return QueuePlacementPolicy.fromXml(root, configuredQueues, conf); } }
public AllocationConfiguration(Configuration conf) { minQueueResources = new HashMap<String, Resource>(); maxQueueResources = new HashMap<String, Resource>(); queueWeights = new HashMap<String, ResourceWeights>(); queueMaxApps = new HashMap<String, Integer>(); userMaxApps = new HashMap<String, Integer>(); queueMaxAMShares = new HashMap<String, Float>(); userMaxAppsDefault = Integer.MAX_VALUE; queueMaxAppsDefault = Integer.MAX_VALUE; queueMaxAMShareDefault = 0.5f; queueAcls = new HashMap<String, Map<QueueACL, AccessControlList>>(); minSharePreemptionTimeouts = new HashMap<String, Long>(); fairSharePreemptionTimeouts = new HashMap<String, Long>(); fairSharePreemptionThresholds = new HashMap<String, Float>(); schedulingPolicies = new HashMap<String, SchedulingPolicy>(); defaultSchedulingPolicy = SchedulingPolicy.DEFAULT_POLICY; reservableQueues = new HashSet<>(); configuredQueues = new HashMap<FSQueueType, Set<String>>(); for (FSQueueType queueType : FSQueueType.values()) { configuredQueues.put(queueType, new HashSet<String>()); } placementPolicy = QueuePlacementPolicy.fromConfiguration(conf, configuredQueues); }
if ("rule".equals(element.getTagName())) { QueuePlacementRule rule = QueuePlacementPolicy .createAndInitializeRule(node); if (rule == null) { throw new AllocationConfigurationException(
@Test public void testDefaultRuleInitializesProperlyWhenPolicyNotConfigured() throws IOException { // This test verifies if default rule in queue placement policy // initializes properly when policy is not configured and // undeclared pools is not allowed. conf.set(FairSchedulerConfiguration.ALLOCATION_FILE, ALLOC_FILE); conf.setBoolean(FairSchedulerConfiguration.ALLOW_UNDECLARED_POOLS, false); // Create an alloc file with no queue placement policy PrintWriter out = new PrintWriter(new FileWriter(ALLOC_FILE)); out.println("<?xml version=\"1.0\"?>"); out.println("<allocations>"); out.println("</allocations>"); out.close(); scheduler.init(conf); scheduler.start(); scheduler.reinitialize(conf, resourceManager.getRMContext()); List<QueuePlacementRule> rules = scheduler.allocConf.placementPolicy .getRules(); for (QueuePlacementRule rule : rules) { if (rule instanceof Default) { Default defaultRule = (Default) rule; assertNotNull(defaultRule.defaultQueueName); } } }
requestedQueue = queuePolicy.assignAppToQueue(requestedQueue, userName); if (StringUtils.isNotBlank(requestedQueue)) { LOG.debug("Setting queue name to " + requestedQueue + " for user "
newPlacementPolicy = QueuePlacementPolicy.fromXml(placementPolicyElement, configuredQueues, conf); } else { newPlacementPolicy = QueuePlacementPolicy.fromConfiguration(conf, configuredQueues);
/** * Builds a QueuePlacementPolicy from an xml element. */ public static QueuePlacementPolicy fromXml(Element el, Map<FSQueueType, Set<String>> configuredQueues, Configuration conf) throws AllocationConfigurationException { List<QueuePlacementRule> rules = new ArrayList<QueuePlacementRule>(); NodeList elements = el.getChildNodes(); for (int i = 0; i < elements.getLength(); i++) { Node node = elements.item(i); if (node instanceof Element) { QueuePlacementRule rule = createAndInitializeRule(node); rules.add(rule); } } return new QueuePlacementPolicy(rules, configuredQueues, conf); }
/** * Build a simple queue placement policy from the allow-undeclared-pools and * user-as-default-queue configuration options. */ public static QueuePlacementPolicy fromConfiguration(Configuration conf, Map<FSQueueType, Set<String>> configuredQueues) { boolean create = conf.getBoolean( FairSchedulerConfiguration.ALLOW_UNDECLARED_POOLS, FairSchedulerConfiguration.DEFAULT_ALLOW_UNDECLARED_POOLS); boolean userAsDefaultQueue = conf.getBoolean( FairSchedulerConfiguration.USER_AS_DEFAULT_QUEUE, FairSchedulerConfiguration.DEFAULT_USER_AS_DEFAULT_QUEUE); List<QueuePlacementRule> rules = new ArrayList<QueuePlacementRule>(); rules.add(new QueuePlacementRule.Specified().initialize(create, null)); if (userAsDefaultQueue) { rules.add(new QueuePlacementRule.User().initialize(create, null)); } if (!userAsDefaultQueue || !create) { rules.add(new QueuePlacementRule.Default().initialize(true, null)); } try { return new QueuePlacementPolicy(rules, configuredQueues, conf); } catch (AllocationConfigurationException ex) { throw new RuntimeException("Should never hit exception when loading" + "placement policy from conf", ex); } }
public AllocationConfiguration(Configuration conf) { minQueueResources = new HashMap<>(); maxChildQueueResources = new HashMap<>(); maxQueueResources = new HashMap<>(); queueWeights = new HashMap<>(); queueMaxApps = new HashMap<>(); userMaxApps = new HashMap<>(); queueMaxAMShares = new HashMap<>(); userMaxAppsDefault = Integer.MAX_VALUE; queueMaxAppsDefault = Integer.MAX_VALUE; queueMaxResourcesDefault = new ConfigurableResource(Resources.unbounded()); queueMaxAMShareDefault = 0.5f; queueAcls = new HashMap<>(); resAcls = new HashMap<>(); minSharePreemptionTimeouts = new HashMap<>(); fairSharePreemptionTimeouts = new HashMap<>(); fairSharePreemptionThresholds = new HashMap<>(); schedulingPolicies = new HashMap<>(); defaultSchedulingPolicy = SchedulingPolicy.DEFAULT_POLICY; reservableQueues = new HashSet<>(); configuredQueues = new HashMap<>(); for (FSQueueType queueType : FSQueueType.values()) { configuredQueues.put(queueType, new HashSet<>()); } placementPolicy = QueuePlacementPolicy.fromConfiguration(conf, configuredQueues); nonPreemptableQueues = new HashSet<>(); queueMaxContainerAllocationMap = new HashMap<>(); }