public static String requestToString(AMRMClient.ContainerRequest request) { Preconditions.checkArgument(request != null, "Null request"); StringBuilder buffer = new StringBuilder(request.toString()); buffer.append("; "); buffer.append("relaxLocality=").append(request.getRelaxLocality()).append("; "); String labels = request.getNodeLabelExpression(); if (labels != null) { buffer.append("nodeLabels=").append(labels).append("; "); } List<String> nodes = request.getNodes(); if (nodes != null) { buffer.append("Nodes = [ "); int size = nodes.size(); for (int i = 0; i < Math.min(NODE_LIST_LIMIT, size); i++) { buffer.append(nodes.get(i)).append(' '); } if (size > NODE_LIST_LIMIT) { buffer.append(String.format("...(total %d entries)", size)); } buffer.append("]; "); } List<String> racks = request.getRacks(); if (racks != null) { buffer.append("racks = [") .append(join(racks, ", ", false)) .append("]; "); } return buffer.toString(); }
public static String requestToString(AMRMClient.ContainerRequest request) { Preconditions.checkArgument(request != null, "Null request"); StringBuilder buffer = new StringBuilder(request.toString()); buffer.append("; "); buffer.append("relaxLocality=").append(request.getRelaxLocality()).append("; "); String labels = request.getNodeLabelExpression(); if (labels != null) { buffer.append("nodeLabels=").append(labels).append("; "); } List<String> nodes = request.getNodes(); if (nodes != null) { buffer.append("Nodes = [ "); int size = nodes.size(); for (int i = 0; i < Math.min(NODE_LIST_LIMIT, size); i++) { buffer.append(nodes.get(i)).append(' '); } if (size > NODE_LIST_LIMIT) { buffer.append(String.format("...(total %d entries)", size)); } buffer.append("]; "); } List<String> racks = request.getRacks(); if (racks != null) { buffer.append("racks = [") .append(join(racks, ", ", false)) .append("]; "); } return buffer.toString(); }
@Override public synchronized void removeContainerRequest(T req) { super.removeContainerRequest(req); boolean hasLocality = (req.getNodes() != null && !req.getNodes().isEmpty()) || (req.getRacks() != null && !req.getRacks().isEmpty()); LocalityRequestCounter lrc = knownRequestsByPriority.get( req.getPriority()); if (hasLocality) { lrc.localityRequests.decrementAndGet(); } else { lrc.noLocalityRequests.decrementAndGet(); } if (lrc.localityRequests.get() == 0 && lrc.noLocalityRequests.get() == 0) { knownRequestsByPriority.remove(req.getPriority()); } }
"Resource request can not be null."); Set<String> dedupedRacks = new HashSet<String>(); if (req.getRacks() != null) { dedupedRacks.addAll(req.getRacks()); if(req.getRacks().size() != dedupedRacks.size()) { Joiner joiner = Joiner.on(','); LOG.warn("ContainerRequest has duplicate racks: " + joiner.join(req.getRacks()));
"Resource request can not be null."); Set<String> dedupedRacks = new HashSet<String>(); if (req.getRacks() != null) { dedupedRacks.addAll(req.getRacks()); if(req.getRacks().size() != dedupedRacks.size()) { Joiner joiner = Joiner.on(','); LOG.warn("ContainerRequest has duplicate racks: " + joiner.join(req.getRacks()));
"Resource request can not be null."); Set<String> dedupedRacks = new HashSet<String>(); if (req.getRacks() != null) { dedupedRacks.addAll(req.getRacks()); if(req.getRacks().size() != dedupedRacks.size()) { Joiner joiner = Joiner.on(','); LOG.warn("ContainerRequest has duplicate racks: " + joiner.join(req.getRacks()));
"Resource request can not be null."); Set<String> dedupedRacks = new HashSet<String>(); if (req.getRacks() != null) { dedupedRacks.addAll(req.getRacks()); if(req.getRacks().size() != dedupedRacks.size()) { Joiner joiner = Joiner.on(','); LOG.warn("ContainerRequest has duplicate racks: " + joiner.join(req.getRacks()));
/** * Valid if a node label expression specified on container request is valid or * not * * @param containerRequest */ private void checkNodeLabelExpression(T containerRequest) { String exp = containerRequest.getNodeLabelExpression(); if (null == exp || exp.isEmpty()) { return; } // Don't support specifying >= 2 node labels in a node label expression now if (exp.contains("&&") || exp.contains("||")) { throw new InvalidContainerRequestException( "Cannot specify more than two node labels" + " in a single node label expression"); } // Don't allow specify node label against ANY request if ((containerRequest.getRacks() != null && (!containerRequest.getRacks().isEmpty())) || (containerRequest.getNodes() != null && (!containerRequest.getNodes().isEmpty()))) { throw new InvalidContainerRequestException( "Cannot specify node label with rack and node"); } }
/** * Inner Validation logic for container request * @param containerRequest request * @param priority raw priority of role * @param requestDetails details for error messages */ @VisibleForTesting public static void validateContainerRequest(AMRMClient.ContainerRequest containerRequest, int priority, String requestDetails) { String exp = containerRequest.getNodeLabelExpression(); boolean hasRacks = containerRequest.getRacks() != null && (!containerRequest.getRacks().isEmpty()); boolean hasNodes = containerRequest.getNodes() != null && (!containerRequest.getNodes().isEmpty()); boolean hasLabel = SliderUtils.isSet(exp); // Don't support specifying >= 2 node labels in a node label expression now if (hasLabel && (exp.contains("&&") || exp.contains("||"))) { throw new InvalidContainerRequestException( "Cannot specify more than two node labels" + " in a single node label expression: " + requestDetails); } // Don't allow specify node label against ANY request listing hosts or racks if (hasLabel && ( hasRacks || hasNodes)) { throw new InvalidContainerRequestException( "Cannot specify node label with rack or node: " + requestDetails); } }
/** * Inner Validation logic for container request * @param containerRequest request * @param priority raw priority of role * @param requestDetails details for error messages */ @VisibleForTesting public static void validateContainerRequest(AMRMClient.ContainerRequest containerRequest, int priority, String requestDetails) { String exp = containerRequest.getNodeLabelExpression(); boolean hasRacks = containerRequest.getRacks() != null && (!containerRequest.getRacks().isEmpty()); boolean hasNodes = containerRequest.getNodes() != null && (!containerRequest.getNodes().isEmpty()); boolean hasLabel = SliderUtils.isSet(exp); // Don't support specifying >= 2 node labels in a node label expression now if (hasLabel && (exp.contains("&&") || exp.contains("||"))) { throw new InvalidContainerRequestException( "Cannot specify more than two node labels" + " in a single node label expression: " + requestDetails); } // Don't allow specify node label against ANY request listing hosts or racks if (hasLabel && ( hasRacks || hasNodes)) { throw new InvalidContainerRequestException( "Cannot specify node label with rack or node: " + requestDetails); } }
@Override public synchronized void removeContainerRequest(T req) { Preconditions.checkArgument(req != null, "Resource request can not be null."); Set<String> allRacks = new HashSet<String>(); if (req.getRacks() != null) { allRacks.addAll(req.getRacks()); } allRacks.addAll(resolveRacks(req.getNodes())); // Update resource requests if (req.getNodes() != null) { for (String node : new HashSet<String>(req.getNodes())) { decResourceRequest(req.getPriority(), node, req.getCapability(), req); } } for (String rack : allRacks) { decResourceRequest(req.getPriority(), rack, req.getCapability(), req); } decResourceRequest(req.getPriority(), ResourceRequest.ANY, req.getCapability(), req); }
@Override public synchronized void removeContainerRequest(T req) { Preconditions.checkArgument(req != null, "Resource request can not be null."); Resource resource = checkAndGetResourceProfile(req.getResourceProfile(), req.getCapability()); Set<String> allRacks = new HashSet<String>(); if (req.getRacks() != null) { allRacks.addAll(req.getRacks()); } allRacks.addAll(resolveRacks(req.getNodes())); // Update resource requests if (req.getNodes() != null) { for (String node : new HashSet<String>(req.getNodes())) { decResourceRequest(req.getPriority(), node, req.getExecutionTypeRequest(), resource, req); } } for (String rack : allRacks) { decResourceRequest(req.getPriority(), rack, req.getExecutionTypeRequest(), resource, req); } decResourceRequest(req.getPriority(), ResourceRequest.ANY, req.getExecutionTypeRequest(), resource, req); }
@Override public synchronized void removeContainerRequest(T req) { Preconditions.checkArgument(req != null, "Resource request can not be null."); Set<String> allRacks = new HashSet<String>(); if (req.getRacks() != null) { allRacks.addAll(req.getRacks()); } allRacks.addAll(resolveRacks(req.getNodes())); // Update resource requests if (req.getNodes() != null) { for (String node : new HashSet<String>(req.getNodes())) { decResourceRequest(req.getPriority(), node, req.getCapability(), req); } } for (String rack : allRacks) { decResourceRequest(req.getPriority(), rack, req.getCapability(), req); } decResourceRequest(req.getPriority(), ResourceRequest.ANY, req.getCapability(), req); }
@Override public synchronized void removeContainerRequest(T req) { Preconditions.checkArgument(req != null, "Resource request can not be null."); Set<String> allRacks = new HashSet<String>(); if (req.getRacks() != null) { allRacks.addAll(req.getRacks()); } allRacks.addAll(resolveRacks(req.getNodes())); // Update resource requests if (req.getNodes() != null) { for (String node : new HashSet<String>(req.getNodes())) { decResourceRequest(req.getPriority(), node, req.getCapability(), req); } } for (String rack : allRacks) { decResourceRequest(req.getPriority(), rack, req.getCapability(), req); } decResourceRequest(req.getPriority(), ResourceRequest.ANY, req.getCapability(), req); }
/** * Match to see whether the container satisfies the request. * We take into consideration that RM has some freedom in rounding * up the allocation and in placing containers on other machines. */ private boolean matchContainerWithPendingRequest(final Container container) { if (this.requestsAfterSentToRM.isEmpty()) { return false; } final AMRMClient.ContainerRequest request = this.requestsAfterSentToRM.peek(); final boolean resourceCondition = container.getResource().getMemory() >= request.getCapability().getMemory(); // TODO[JIRA REEF-35]: check vcores once YARN-2380 is resolved final boolean nodeCondition = request.getNodes() == null || request.getNodes().contains(container.getNodeId().getHost()); final boolean rackCondition = request.getRacks() == null || request.getRacks().contains(this.nodeIdToRackName.get(container.getNodeId().toString())); return resourceCondition && (request.getRelaxLocality() || rackCondition && nodeCondition); }
@Override public synchronized void addContainerRequest(T req) { super.addContainerRequest(req); boolean hasLocality = (req.getNodes() != null && !req.getNodes().isEmpty()) || (req.getRacks() != null && !req.getRacks().isEmpty()); LocalityRequestCounter lrc = knownRequestsByPriority.get(req.getPriority()); if (lrc == null) { lrc = new LocalityRequestCounter(); knownRequestsByPriority.put(req.getPriority(), lrc); } if (hasLocality) { lrc.localityRequests.incrementAndGet(); } else { lrc.noLocalityRequests.incrementAndGet(); } }
private boolean isSameKindOfRequest(final AMRMClient.ContainerRequest r1, final AMRMClient.ContainerRequest r2) { return r1.getPriority().compareTo(r2.getPriority()) == 0 && r1.getCapability().compareTo(r2.getCapability()) == 0 && r1.getRelaxLocality() == r2.getRelaxLocality() && ListUtils.isEqualList(r1.getNodes(), r2.getNodes()) && ListUtils.isEqualList(r1.getRacks(), r2.getRacks()); }