private void addHostSpecificRequest(StreamingContainerAgent.ContainerStartRequest csr, ContainerRequest cr) { String hostKey = StringUtils.join(cr.getNodes(), ":"); List<ContainerRequest> requests; if (hostSpecificRequestsMap.containsKey(hostKey)) { requests = hostSpecificRequestsMap.get(hostKey); } else { requests = new ArrayList<>(); } requests.add(cr); hostSpecificRequestsMap.put(hostKey, requests); LOG.info("Requesting container for node {} request = {}", cr.getNodes(), cr); hostSpecificRequests.put(cr, csr); }
public static String requestToString(AMRMClient.ContainerRequest request) { Preconditions.checkArgument(request != null, "Null request"); StringBuilder buffer = new StringBuilder(request.toString()); buffer.append("; "); buffer.append("relaxLocality=").append(request.getRelaxLocality()).append("; "); String labels = request.getNodeLabelExpression(); if (labels != null) { buffer.append("nodeLabels=").append(labels).append("; "); } List<String> nodes = request.getNodes(); if (nodes != null) { buffer.append("Nodes = [ "); int size = nodes.size(); for (int i = 0; i < Math.min(NODE_LIST_LIMIT, size); i++) { buffer.append(nodes.get(i)).append(' '); } if (size > NODE_LIST_LIMIT) { buffer.append(String.format("...(total %d entries)", size)); } buffer.append("]; "); } List<String> racks = request.getRacks(); if (racks != null) { buffer.append("racks = [") .append(join(racks, ", ", false)) .append("]; "); } return buffer.toString(); }
public static String requestToString(AMRMClient.ContainerRequest request) { Preconditions.checkArgument(request != null, "Null request"); StringBuilder buffer = new StringBuilder(request.toString()); buffer.append("; "); buffer.append("relaxLocality=").append(request.getRelaxLocality()).append("; "); String labels = request.getNodeLabelExpression(); if (labels != null) { buffer.append("nodeLabels=").append(labels).append("; "); } List<String> nodes = request.getNodes(); if (nodes != null) { buffer.append("Nodes = [ "); int size = nodes.size(); for (int i = 0; i < Math.min(NODE_LIST_LIMIT, size); i++) { buffer.append(nodes.get(i)).append(' '); } if (size > NODE_LIST_LIMIT) { buffer.append(String.format("...(total %d entries)", size)); } buffer.append("]; "); } List<String> racks = request.getRacks(); if (racks != null) { buffer.append("racks = [") .append(join(racks, ", ", false)) .append("]; "); } return buffer.toString(); }
Set<String> inferredRacks = resolveRacks(req.getNodes()); inferredRacks.removeAll(dedupedRacks); if (req.getNodes() != null) { HashSet<String> dedupedNodes = new HashSet<String>(req.getNodes()); if(dedupedNodes.size() != req.getNodes().size()) { Joiner joiner = Joiner.on(','); LOG.warn("ContainerRequest has duplicate nodes: " + joiner.join(req.getNodes()));
List<String> blacklistNodes = resourceRequestor.getNodesExceptHost(requests.get(0).getNodes()); amRmClient.updateBlacklist(blacklistNodes, requests.get(0).getNodes()); blacklistedNodesForHostSpecificRequests = blacklistNodes; LOG.info("Sending {} request(s) after blacklisting all nodes other than {}", requests.size(), requests.get(0).getNodes());
List<String> issuedRequestNodes = issuedRequest.getNodes(); if (SliderUtils.isUnset(label) && issuedRequestNodes != null) { nodes = issuedRequestNodes.toArray(new String[issuedRequestNodes.size()]);
List<String> issuedRequestNodes = issuedRequest.getNodes(); if (SliderUtils.isUnset(label) && issuedRequestNodes != null) { nodes = issuedRequestNodes.toArray(new String[issuedRequestNodes.size()]);
/** * Valid if a node label expression specified on container request is valid or * not * * @param containerRequest */ private void checkNodeLabelExpression(T containerRequest) { String exp = containerRequest.getNodeLabelExpression(); if (null == exp || exp.isEmpty()) { return; } // Don't support specifying >= 2 node labels in a node label expression now if (exp.contains("&&") || exp.contains("||")) { throw new InvalidContainerRequestException( "Cannot specify more than two node labels" + " in a single node label expression"); } // Don't allow specify node label against ANY request if ((containerRequest.getRacks() != null && (!containerRequest.getRacks().isEmpty())) || (containerRequest.getNodes() != null && (!containerRequest.getNodes().isEmpty()))) { throw new InvalidContainerRequestException( "Cannot specify node label with rack and node"); } }
/** * Inner Validation logic for container request * @param containerRequest request * @param priority raw priority of role * @param requestDetails details for error messages */ @VisibleForTesting public static void validateContainerRequest(AMRMClient.ContainerRequest containerRequest, int priority, String requestDetails) { String exp = containerRequest.getNodeLabelExpression(); boolean hasRacks = containerRequest.getRacks() != null && (!containerRequest.getRacks().isEmpty()); boolean hasNodes = containerRequest.getNodes() != null && (!containerRequest.getNodes().isEmpty()); boolean hasLabel = SliderUtils.isSet(exp); // Don't support specifying >= 2 node labels in a node label expression now if (hasLabel && (exp.contains("&&") || exp.contains("||"))) { throw new InvalidContainerRequestException( "Cannot specify more than two node labels" + " in a single node label expression: " + requestDetails); } // Don't allow specify node label against ANY request listing hosts or racks if (hasLabel && ( hasRacks || hasNodes)) { throw new InvalidContainerRequestException( "Cannot specify node label with rack or node: " + requestDetails); } }
/** * Inner Validation logic for container request * @param containerRequest request * @param priority raw priority of role * @param requestDetails details for error messages */ @VisibleForTesting public static void validateContainerRequest(AMRMClient.ContainerRequest containerRequest, int priority, String requestDetails) { String exp = containerRequest.getNodeLabelExpression(); boolean hasRacks = containerRequest.getRacks() != null && (!containerRequest.getRacks().isEmpty()); boolean hasNodes = containerRequest.getNodes() != null && (!containerRequest.getNodes().isEmpty()); boolean hasLabel = SliderUtils.isSet(exp); // Don't support specifying >= 2 node labels in a node label expression now if (hasLabel && (exp.contains("&&") || exp.contains("||"))) { throw new InvalidContainerRequestException( "Cannot specify more than two node labels" + " in a single node label expression: " + requestDetails); } // Don't allow specify node label against ANY request listing hosts or racks if (hasLabel && ( hasRacks || hasNodes)) { throw new InvalidContainerRequestException( "Cannot specify node label with rack or node: " + requestDetails); } }
@Override public synchronized void removeContainerRequest(T req) { Preconditions.checkArgument(req != null, "Resource request can not be null."); Set<String> allRacks = new HashSet<String>(); if (req.getRacks() != null) { allRacks.addAll(req.getRacks()); } allRacks.addAll(resolveRacks(req.getNodes())); // Update resource requests if (req.getNodes() != null) { for (String node : new HashSet<String>(req.getNodes())) { decResourceRequest(req.getPriority(), node, req.getCapability(), req); } } for (String rack : allRacks) { decResourceRequest(req.getPriority(), rack, req.getCapability(), req); } decResourceRequest(req.getPriority(), ResourceRequest.ANY, req.getCapability(), req); }
@Override public synchronized void removeContainerRequest(T req) { Preconditions.checkArgument(req != null, "Resource request can not be null."); Resource resource = checkAndGetResourceProfile(req.getResourceProfile(), req.getCapability()); Set<String> allRacks = new HashSet<String>(); if (req.getRacks() != null) { allRacks.addAll(req.getRacks()); } allRacks.addAll(resolveRacks(req.getNodes())); // Update resource requests if (req.getNodes() != null) { for (String node : new HashSet<String>(req.getNodes())) { decResourceRequest(req.getPriority(), node, req.getExecutionTypeRequest(), resource, req); } } for (String rack : allRacks) { decResourceRequest(req.getPriority(), rack, req.getExecutionTypeRequest(), resource, req); } decResourceRequest(req.getPriority(), ResourceRequest.ANY, req.getExecutionTypeRequest(), resource, req); }
@Override public synchronized void removeContainerRequest(T req) { Preconditions.checkArgument(req != null, "Resource request can not be null."); Set<String> allRacks = new HashSet<String>(); if (req.getRacks() != null) { allRacks.addAll(req.getRacks()); } allRacks.addAll(resolveRacks(req.getNodes())); // Update resource requests if (req.getNodes() != null) { for (String node : new HashSet<String>(req.getNodes())) { decResourceRequest(req.getPriority(), node, req.getCapability(), req); } } for (String rack : allRacks) { decResourceRequest(req.getPriority(), rack, req.getCapability(), req); } decResourceRequest(req.getPriority(), ResourceRequest.ANY, req.getCapability(), req); }
@Override public synchronized void removeContainerRequest(T req) { Preconditions.checkArgument(req != null, "Resource request can not be null."); Set<String> allRacks = new HashSet<String>(); if (req.getRacks() != null) { allRacks.addAll(req.getRacks()); } allRacks.addAll(resolveRacks(req.getNodes())); // Update resource requests if (req.getNodes() != null) { for (String node : new HashSet<String>(req.getNodes())) { decResourceRequest(req.getPriority(), node, req.getCapability(), req); } } for (String rack : allRacks) { decResourceRequest(req.getPriority(), rack, req.getCapability(), req); } decResourceRequest(req.getPriority(), ResourceRequest.ANY, req.getCapability(), req); }
/** * Match to see whether the container satisfies the request. * We take into consideration that RM has some freedom in rounding * up the allocation and in placing containers on other machines. */ private boolean matchContainerWithPendingRequest(final Container container) { if (this.requestsAfterSentToRM.isEmpty()) { return false; } final AMRMClient.ContainerRequest request = this.requestsAfterSentToRM.peek(); final boolean resourceCondition = container.getResource().getMemory() >= request.getCapability().getMemory(); // TODO[JIRA REEF-35]: check vcores once YARN-2380 is resolved final boolean nodeCondition = request.getNodes() == null || request.getNodes().contains(container.getNodeId().getHost()); final boolean rackCondition = request.getRacks() == null || request.getRacks().contains(this.nodeIdToRackName.get(container.getNodeId().toString())); return resourceCondition && (request.getRelaxLocality() || rackCondition && nodeCondition); }
@Override public synchronized void removeContainerRequest(T req) { super.removeContainerRequest(req); boolean hasLocality = (req.getNodes() != null && !req.getNodes().isEmpty()) || (req.getRacks() != null && !req.getRacks().isEmpty()); LocalityRequestCounter lrc = knownRequestsByPriority.get( req.getPriority()); if (hasLocality) { lrc.localityRequests.decrementAndGet(); } else { lrc.noLocalityRequests.decrementAndGet(); } if (lrc.localityRequests.get() == 0 && lrc.noLocalityRequests.get() == 0) { knownRequestsByPriority.remove(req.getPriority()); } }
@Override public synchronized void addContainerRequest(T req) { super.addContainerRequest(req); boolean hasLocality = (req.getNodes() != null && !req.getNodes().isEmpty()) || (req.getRacks() != null && !req.getRacks().isEmpty()); LocalityRequestCounter lrc = knownRequestsByPriority.get(req.getPriority()); if (lrc == null) { lrc = new LocalityRequestCounter(); knownRequestsByPriority.put(req.getPriority(), lrc); } if (hasLocality) { lrc.localityRequests.incrementAndGet(); } else { lrc.noLocalityRequests.incrementAndGet(); } }
private void recreateContainerRequest(Map<StreamingContainerAgent.ContainerStartRequest, MutablePair<Integer, ContainerRequest>> requestedResources, int loopCounter, ResourceRequestHandler resourceRequestor, List<ContainerRequest> removedContainerRequests) { for (Map.Entry<StreamingContainerAgent.ContainerStartRequest, MutablePair<Integer, ContainerRequest>> entry : requestedResources.entrySet()) { if ((loopCounter - entry.getValue().getKey()) > NUMBER_MISSED_HEARTBEATS) { StreamingContainerAgent.ContainerStartRequest csr = entry.getKey(); removedContainerRequests.add(entry.getValue().getRight()); ContainerRequest cr = resourceRequestor.createContainerRequest(csr, false); if (cr.getNodes() != null && !cr.getNodes().isEmpty()) { addHostSpecificRequest(csr, cr); } else { otherContainerRequests.put(cr, csr); } } } }
@Override public void addContainerRequest(Map<StreamingContainerAgent.ContainerStartRequest, MutablePair<Integer, ContainerRequest>> requestedResources, int loopCounter, List<ContainerRequest> containerRequests, StreamingContainerAgent.ContainerStartRequest csr, ContainerRequest cr) { if (cr.getNodes() != null && !cr.getNodes().isEmpty()) { // Put it in a Map to check if multiple requests can be combined addHostSpecificRequest(csr, cr); } else { LOG.info("No node specific request ", cr); otherContainerRequests.put(cr, csr); } }
private boolean isSameKindOfRequest(final AMRMClient.ContainerRequest r1, final AMRMClient.ContainerRequest r2) { return r1.getPriority().compareTo(r2.getPriority()) == 0 && r1.getCapability().compareTo(r2.getCapability()) == 0 && r1.getRelaxLocality() == r2.getRelaxLocality() && ListUtils.isEqualList(r1.getNodes(), r2.getNodes()) && ListUtils.isEqualList(r1.getRacks(), r2.getRacks()); }