private void clearReservation(SchedulerNode node) { String rackName = node.getRackName() == null ? "NULL" : node.getRackName(); try { writeLock.lock(); Set<String> rackReservations = reservations.get(rackName); if (rackReservations != null) { rackReservations.remove(node.getNodeName()); } } finally { writeLock.unlock(); } }
private void setReservation(SchedulerNode node) { String rackName = node.getRackName() == null ? "NULL" : node.getRackName(); try { writeLock.lock(); Set<String> rackReservations = reservations.get(rackName); if (rackReservations == null) { rackReservations = new HashSet<>(); reservations.put(rackName, rackReservations); } rackReservations.add(node.getNodeName()); } finally { writeLock.unlock(); } }
/** * The {@link ResourceScheduler} is allocating data-local resources to the * application. * * @param allocatedContainers * resources allocated to the application */ synchronized private void allocateNodeLocal(SchedulerNode node, Priority priority, ResourceRequest nodeLocalRequest, Container container, List<ResourceRequest> resourceRequests) { // Update future requirements decResourceRequest(node.getNodeName(), priority, nodeLocalRequest); ResourceRequest rackLocalRequest = requests.get(priority).get( node.getRackName()); decResourceRequest(node.getRackName(), priority, rackLocalRequest); ResourceRequest offRackRequest = requests.get(priority).get( ResourceRequest.ANY); decrementOutstanding(offRackRequest); // Update cloned NodeLocal, RackLocal and OffRack requests for recovery resourceRequests.add(cloneResourceRequest(nodeLocalRequest)); resourceRequests.add(cloneResourceRequest(rackLocalRequest)); resourceRequests.add(cloneResourceRequest(offRackRequest)); }
/** * The {@link ResourceScheduler} is allocating data-local resources to the * application. * * @param allocatedContainers * resources allocated to the application */ synchronized private void allocateNodeLocal(SchedulerNode node, Priority priority, ResourceRequest nodeLocalRequest, Container container, List<ResourceRequest> resourceRequests) { // Update future requirements decResourceRequest(node.getNodeName(), priority, nodeLocalRequest); ResourceRequest rackLocalRequest = requests.get(priority).get( node.getRackName()); decResourceRequest(node.getRackName(), priority, rackLocalRequest); ResourceRequest offRackRequest = requests.get(priority).get( ResourceRequest.ANY); decrementOutstanding(offRackRequest); // Update cloned NodeLocal, RackLocal and OffRack requests for recovery resourceRequests.add(cloneResourceRequest(nodeLocalRequest)); resourceRequests.add(cloneResourceRequest(rackLocalRequest)); resourceRequests.add(cloneResourceRequest(offRackRequest)); }
public static boolean isBlacklisted(SchedulerApplicationAttempt application, SchedulerNode node, Log LOG) { if (application.isBlacklisted(node.getNodeName())) { if (LOG.isDebugEnabled()) { LOG.debug("Skipping 'host' " + node.getNodeName() + " for " + application.getApplicationId() + " since it has been blacklisted"); } return true; } if (application.isBlacklisted(node.getRackName())) { if (LOG.isDebugEnabled()) { LOG.debug("Skipping 'rack' " + node.getRackName() + " for " + application.getApplicationId() + " since it has been blacklisted"); } return true; } return false; }
public static boolean isBlacklisted(SchedulerApplicationAttempt application, SchedulerNode node, Log LOG) { if (application.isBlacklisted(node.getNodeName())) { if (LOG.isDebugEnabled()) { LOG.debug("Skipping 'host' " + node.getNodeName() + " for " + application.getApplicationId() + " since it has been blacklisted"); } return true; } if (application.isBlacklisted(node.getRackName())) { if (LOG.isDebugEnabled()) { LOG.debug("Skipping 'rack' " + node.getRackName() + " for " + application.getApplicationId() + " since it has been blacklisted"); } return true; } return false; }
/** * The {@link ResourceScheduler} is allocating data-local resources to the * application. */ private void allocateRackLocal(SchedulerRequestKey schedulerKey, SchedulerNode node, ResourceRequest rackLocalRequest, List<ResourceRequest> resourceRequests) { // Update future requirements decResourceRequest(node.getRackName(), rackLocalRequest); ResourceRequest offRackRequest = resourceRequestMap.get( ResourceRequest.ANY); decrementOutstanding(schedulerKey, offRackRequest); // Update cloned RackLocal and OffRack requests for recovery resourceRequests.add(cloneResourceRequest(rackLocalRequest)); resourceRequests.add(cloneResourceRequest(offRackRequest)); }
/** * The {@link ResourceScheduler} is allocating data-local resources to the * application. * * @param allocatedContainers * resources allocated to the application */ synchronized private void allocateRackLocal(SchedulerNode node, Priority priority, ResourceRequest rackLocalRequest, Container container, List<ResourceRequest> resourceRequests) { // Update future requirements decResourceRequest(node.getRackName(), priority, rackLocalRequest); ResourceRequest offRackRequest = requests.get(priority).get( ResourceRequest.ANY); decrementOutstanding(offRackRequest); // Update cloned RackLocal and OffRack requests for recovery resourceRequests.add(cloneResourceRequest(rackLocalRequest)); resourceRequests.add(cloneResourceRequest(offRackRequest)); }
/** * The {@link ResourceScheduler} is allocating data-local resources to the * application. * * @param allocatedContainers * resources allocated to the application */ synchronized private void allocateRackLocal(SchedulerNode node, Priority priority, ResourceRequest rackLocalRequest, Container container, List<ResourceRequest> resourceRequests) { // Update future requirements decResourceRequest(node.getRackName(), priority, rackLocalRequest); ResourceRequest offRackRequest = requests.get(priority).get( ResourceRequest.ANY); decrementOutstanding(offRackRequest); // Update cloned RackLocal and OffRack requests for recovery resourceRequests.add(cloneResourceRequest(rackLocalRequest)); resourceRequests.add(cloneResourceRequest(offRackRequest)); }
public static boolean isPlaceBlacklisted( SchedulerApplicationAttempt application, SchedulerNode node, Log log) { if (application.isPlaceBlacklisted(node.getNodeName())) { if (log.isDebugEnabled()) { log.debug("Skipping 'host' " + node.getNodeName() + " for " + application.getApplicationId() + " since it has been blacklisted"); } return true; } if (application.isPlaceBlacklisted(node.getRackName())) { if (log.isDebugEnabled()) { log.debug("Skipping 'rack' " + node.getRackName() + " for " + application.getApplicationId() + " since it has been blacklisted"); } return true; } return false; }
@Override public boolean canAllocate(NodeType type, SchedulerNode node) { try { readLock.lock(); ResourceRequest r = resourceRequestMap.get( ResourceRequest.ANY); if (r == null || r.getNumContainers() <= 0) { return false; } if (type == NodeType.RACK_LOCAL || type == NodeType.NODE_LOCAL) { r = resourceRequestMap.get(node.getRackName()); if (r == null || r.getNumContainers() <= 0) { return false; } if (type == NodeType.NODE_LOCAL) { r = resourceRequestMap.get(node.getNodeName()); if (r == null || r.getNumContainers() <= 0) { return false; } } } return true; } finally { readLock.unlock(); } }
/** * The {@link ResourceScheduler} is allocating data-local resources to the * application. */ private void allocateNodeLocal(SchedulerRequestKey schedulerKey, SchedulerNode node, ResourceRequest nodeLocalRequest, List<ResourceRequest> resourceRequests) { // Update future requirements decResourceRequest(node.getNodeName(), nodeLocalRequest); ResourceRequest rackLocalRequest = resourceRequestMap.get( node.getRackName()); decResourceRequest(node.getRackName(), rackLocalRequest); ResourceRequest offRackRequest = resourceRequestMap.get( ResourceRequest.ANY); decrementOutstanding(schedulerKey, offRackRequest); // Update cloned NodeLocal, RackLocal and OffRack requests for recovery resourceRequests.add(cloneResourceRequest(nodeLocalRequest)); resourceRequests.add(cloneResourceRequest(rackLocalRequest)); resourceRequests.add(cloneResourceRequest(offRackRequest)); }
public void addNode(N node) { writeLock.lock(); try { nodes.put(node.getNodeID(), node); nodeNameToNodeMap.put(node.getNodeName(), node); List<N> nodesPerLabels = nodesPerLabel.get(node.getPartition()); if (nodesPerLabels == null) { nodesPerLabels = new ArrayList<N>(); } nodesPerLabels.add(node); // Update new set of nodes for given partition. nodesPerLabel.put(node.getPartition(), nodesPerLabels); // Update nodes per rack as well String rackName = node.getRackName(); List<N> nodesList = nodesPerRack.get(rackName); if (nodesList == null) { nodesList = new ArrayList<>(); nodesPerRack.put(rackName, nodesList); } nodesList.add(node); // Update cluster capacity Resources.addTo(clusterCapacity, node.getTotalResource()); staleClusterCapacity = Resources.clone(clusterCapacity); // Update maximumAllocation updateMaxResources(node, true); } finally { writeLock.unlock(); } }
@Override public ContainerRequest allocate(SchedulerRequestKey schedulerKey, NodeType type, SchedulerNode node) { try { writeLock.lock(); List<ResourceRequest> resourceRequests = new ArrayList<>(); ResourceRequest request; if (type == NodeType.NODE_LOCAL) { request = resourceRequestMap.get(node.getNodeName()); } else if (type == NodeType.RACK_LOCAL) { request = resourceRequestMap.get(node.getRackName()); } else{ request = resourceRequestMap.get(ResourceRequest.ANY); } if (type == NodeType.NODE_LOCAL) { allocateNodeLocal(schedulerKey, node, request, resourceRequests); } else if (type == NodeType.RACK_LOCAL) { allocateRackLocal(schedulerKey, node, request, resourceRequests); } else{ allocateOffSwitch(schedulerKey, request, resourceRequests); } return new ContainerRequest(resourceRequests); } finally { writeLock.unlock(); } }
private SchedulerNode createNode() { SchedulerNode node = mock(SchedulerNode.class); when(node.getNodeName()).thenReturn("somehost"); when(node.getRackName()).thenReturn("somerack"); when(node.getNodeID()).thenReturn(nodeId); return node; }
private SchedulerNode newSchedulerNode(String hostname, String rackName, NodeId nodeId) { SchedulerNode node = mock(SchedulerNode.class); when(node.getNodeName()).thenReturn(hostname); when(node.getRackName()).thenReturn(rackName); when(node.getNodeID()).thenReturn(nodeId); return node; }
private SchedulerNode createNode() { SchedulerNode node = mock(SchedulerNode.class); when(node.getNodeName()).thenReturn("somehost"); when(node.getRackName()).thenReturn("somerack"); when(node.getNodeID()).thenReturn(nodeId); return node; }
String rackName = node.getRackName(); List<N> nodesList = nodesPerRack.get(rackName); if (nodesList == null) {
private Map<String, ResourceRequest> createResourceRequests( RMContainer rmContainer, SchedulerNode schedulerNode, SchedulerRequestKey schedulerKey, Resource resToIncrease) { Map<String, ResourceRequest> resMap = new HashMap<>(); resMap.put(rmContainer.getContainer().getNodeId().getHost(), createResourceReqForIncrease(schedulerKey, resToIncrease, RECORD_FACTORY.newRecordInstance(ResourceRequest.class), rmContainer, rmContainer.getContainer().getNodeId().getHost())); resMap.put(schedulerNode.getRackName(), createResourceReqForIncrease(schedulerKey, resToIncrease, RECORD_FACTORY.newRecordInstance(ResourceRequest.class), rmContainer, schedulerNode.getRackName())); resMap.put(ResourceRequest.ANY, createResourceReqForIncrease(schedulerKey, resToIncrease, RECORD_FACTORY.newRecordInstance(ResourceRequest.class), rmContainer, ResourceRequest.ANY)); return resMap; }
private RemoteNode convertToRemoteNode(NodeId nodeId) { SchedulerNode node = ((AbstractYarnScheduler) rmContext.getScheduler()).getNode(nodeId); if (node != null) { RemoteNode rNode = RemoteNode.newInstance(nodeId, node.getHttpAddress()); rNode.setRackName(node.getRackName()); rNode.setNodePartition(node.getPartition()); return rNode; } return null; }