private int getMetric(ClusterNode cn) { return (cn != null) ? ((LoadComparator)nodeSelector.getComparator()) .getMetric(cn) : 0; }
public void initThresholdCalculator(float sigma, int limitMin, int limitMax) { this.thresholdCalculator = new QueueLimitCalculator(this, sigma, limitMin, limitMax); }
public ClusterNode(NodeId nodeId) { this.nodeId = nodeId; updateTimestamp(); }
@Test public void testContainerQueuingLimit() { NodeQueueLoadMonitor selector = new NodeQueueLoadMonitor( NodeQueueLoadMonitor.LoadComparator.QUEUE_LENGTH); selector.updateNode(createRMNode("h1", 1, -1, 15)); selector.updateNode(createRMNode("h2", 2, -1, 5)); selector.updateNode(createRMNode("h3", 3, -1, 10)); selector.initThresholdCalculator(0, 6, 100); QueueLimitCalculator calculator = selector.getThresholdCalculator(); ContainerQueuingLimit containerQueuingLimit = calculator .createContainerQueuingLimit(); Assert.assertEquals(6, containerQueuingLimit.getMaxQueueLength()); Assert.assertEquals(-1, containerQueuingLimit.getMaxQueueWaitTimeInMs()); selector.computeTask.run(); containerQueuingLimit = calculator.createContainerQueuingLimit(); Assert.assertEquals(10, containerQueuingLimit.getMaxQueueLength()); Assert.assertEquals(-1, containerQueuingLimit.getMaxQueueWaitTimeInMs()); selector.updateNode(createRMNode("h1", 1, -1, 110)); selector.updateNode(createRMNode("h2", 2, -1, 120)); selector.updateNode(createRMNode("h3", 3, -1, 130)); selector.updateNode(createRMNode("h4", 4, -1, 140)); selector.updateNode(createRMNode("h5", 5, -1, 150)); selector.updateNode(createRMNode("h6", 6, -1, 160)); selector.computeTask.run(); containerQueuingLimit = calculator.createContainerQueuingLimit(); Assert.assertEquals(100, containerQueuingLimit.getMaxQueueLength()); selector.updateNode(createRMNode("h1", 1, -1, 1));
@Test public void testWaitTimeSort() { NodeQueueLoadMonitor selector = new NodeQueueLoadMonitor( NodeQueueLoadMonitor.LoadComparator.QUEUE_WAIT_TIME); selector.updateNode(createRMNode("h1", 1, 15, 10)); selector.updateNode(createRMNode("h2", 2, 5, 10)); selector.updateNode(createRMNode("h3", 3, 10, 10)); selector.computeTask.run(); List<NodeId> nodeIds = selector.selectNodes(); System.out.println("1-> " + nodeIds); Assert.assertEquals("h2:2", nodeIds.get(0).toString()); selector.updateNode(createRMNode("h3", 3, 2, 10)); selector.computeTask.run(); nodeIds = selector.selectNodes(); System.out.println("2-> "+ nodeIds); Assert.assertEquals("h3:3", nodeIds.get(0).toString()); selector.updateNode(createRMNode("h4", 4, -1, 10)); selector.computeTask.run(); nodeIds = selector.selectNodes(); System.out.println("3-> "+ nodeIds);
this.lastCacheUpdateTime = System.currentTimeMillis(); NodeQueueLoadMonitor.LoadComparator comparator = NodeQueueLoadMonitor.LoadComparator.valueOf( rmContext.getYarnConfiguration().get( YarnConfiguration.NM_CONTAINER_QUEUING_LOAD_COMPARATOR, new NodeQueueLoadMonitor(nodeSortInterval, comparator); topKSelector.initThresholdCalculator(sigma, limitMin, limitMax); this.nodeMonitor = topKSelector;
/** * Not thread safe. Caller should synchronize on sorted nodes list. */ void update() { List<NodeId> sortedNodes = nodeSelector.getSortedNodes(); if (sortedNodes.size() > 0) { // Calculate mean int sum = 0; for (NodeId n : sortedNodes) { sum += getMetric(getNode(n)); } mean.set(sum / sortedNodes.size()); // Calculate stdev int sqrSumMean = 0; for (NodeId n : sortedNodes) { int val = getMetric(getNode(n)); sqrSumMean += Math.pow(val - mean.get(), 2); } stdev.set( (int) Math.round(Math.sqrt( sqrSumMean / (float) sortedNodes.size()))); } }
public QueueLimitCalculator getNodeManagerQueueLimitCalculator() { return nodeMonitor.getThresholdCalculator(); }
public ContainerQueuingLimit createContainerQueuingLimit() { ContainerQueuingLimit containerQueuingLimit = ContainerQueuingLimit.newInstance(); if (nodeSelector.getComparator() == LoadComparator.QUEUE_WAIT_TIME) { containerQueuingLimit.setMaxQueueWaitTimeInMs(getThreshold()); containerQueuingLimit.setMaxQueueLength(-1); } else { containerQueuingLimit.setMaxQueueWaitTimeInMs(-1); containerQueuingLimit.setMaxQueueLength(getThreshold()); } return containerQueuingLimit; } }
/** * Returns all Node Ids as ordered list from Least to Most Loaded. * @return ordered list of nodes */ public List<NodeId> selectNodes() { return selectLeastLoadedNodes(-1); }
void update() { this.stats.update(); }
private ClusterNode getNode(NodeId nId) { return nodeSelector.getClusterNodes().get(nId); }
private int getThreshold() { int thres = determineThreshold(); return Math.min(rangeMax, Math.max(rangeMin, thres)); }
private RMNode createRMNode(String host, int port, int waitTime, int queueLength) { return createRMNode(host, port, waitTime, queueLength, DEFAULT_MAX_QUEUE_LENGTH); }
@Override public void run() { ReentrantReadWriteLock.WriteLock writeLock = sortedNodesLock.writeLock(); writeLock.lock(); try { try { List<NodeId> nodeIds = sortNodes(); sortedNodes.clear(); sortedNodes.addAll(nodeIds); } catch (Exception ex) { LOG.warn("Got Exception while sorting nodes..", ex); } if (thresholdCalculator != null) { thresholdCalculator.update(); } } finally { writeLock.unlock(); } } };
private RMNode createRMNode(String host, int port, int waitTime, int queueLength, int queueCapacity) { RMNode node1 = Mockito.mock(RMNode.class); NodeId nID1 = new FakeNodeId(host, port); Mockito.when(node1.getNodeID()).thenReturn(nID1); OpportunisticContainersStatus status1 = Mockito.mock(OpportunisticContainersStatus.class); Mockito.when(status1.getEstimatedQueueWaitTime()) .thenReturn(waitTime); Mockito.when(status1.getWaitQueueLength()) .thenReturn(queueLength); Mockito.when(status1.getOpportQueueCapacity()) .thenReturn(queueCapacity); Mockito.when(node1.getOpportunisticContainersStatus()).thenReturn(status1); return node1; } }
@Test public void testQueueLengthSort() { NodeQueueLoadMonitor selector = new NodeQueueLoadMonitor( NodeQueueLoadMonitor.LoadComparator.QUEUE_LENGTH); selector.updateNode(createRMNode("h1", 1, -1, 15)); selector.updateNode(createRMNode("h2", 2, -1, 5)); selector.updateNode(createRMNode("h3", 3, -1, 10)); selector.computeTask.run(); List<NodeId> nodeIds = selector.selectNodes(); System.out.println("1-> " + nodeIds); Assert.assertEquals("h2:2", nodeIds.get(0).toString()); selector.updateNode(createRMNode("h3", 3, -1, 2)); selector.computeTask.run(); nodeIds = selector.selectNodes(); System.out.println("2-> "+ nodeIds); Assert.assertEquals("h3:3", nodeIds.get(0).toString()); selector.updateNode(createRMNode("h4", 4, -1, 20)); selector.computeTask.run(); nodeIds = selector.selectNodes(); System.out.println("3-> "+ nodeIds); selector.updateNode(createRMNode("h3", 3, -1, DEFAULT_MAX_QUEUE_LENGTH)); selector.computeTask.run(); nodeIds = selector.selectNodes(); System.out.println("4-> "+ nodeIds); Assert.assertEquals(3, nodeIds.size());
@VisibleForTesting synchronized List<RemoteNode> getLeastLoadedNodes() { long currTime = System.currentTimeMillis(); if ((currTime - lastCacheUpdateTime > cacheRefreshInterval) || (cachedNodes == null)) { cachedNodes = convertToRemoteNodes( this.nodeMonitor.selectLeastLoadedNodes(this.k)); if (cachedNodes.size() > 0) { lastCacheUpdateTime = currTime; } } return cachedNodes; }