ContainerRequest newCr = new ContainerRequest(cr.getCapability(), null, null, cr.getPriority()); MutablePair<Integer, ContainerRequest> pair = new MutablePair<>(loopCounter, newCr); requestedResources.put(csr, pair);
void onContainerRequest(final AMRMClient.ContainerRequest... containerRequests) { synchronized (this) { this.containerRequestCounter.incrementBy(containerRequests.length); boolean queueWasEmpty = this.outstandingContainerRequests.isEmpty(); for (final AMRMClient.ContainerRequest containerRequest : containerRequests) { LOG.log(Level.FINEST, "Container Request: memory = {0}, core number = {1}", new Object[]{containerRequest.getCapability().getMemory(), containerRequest.getCapability().getVirtualCores()}); LOG.log(Level.FINEST, "Adding container request to local queue: {0}", containerRequest); this.outstandingContainerRequests.add(containerRequest); } if (queueWasEmpty && containerRequests.length != 0) { AMRMClient.ContainerRequest firstRequest = outstandingContainerRequests.peek(); LOG.log(Level.INFO, "Requesting first container from YARN: {0}", firstRequest); this.lastContainerMemory = firstRequest.getCapability().getMemory(); this.resourceManager.addContainerRequest(firstRequest); } } this.updateRuntimeStatus(); }
/** * Add a container request if the request is non-null * @param operations operations to add the entry to * @param containerAsk what to ask for * @return true if a request was added */ private boolean addContainerRequest(List<AbstractRMOperation> operations, AMRMClient.ContainerRequest containerAsk) { if (containerAsk != null) { log.info("Container ask is {} and label = {}", containerAsk, containerAsk.getNodeLabelExpression()); int askMemory = containerAsk.getCapability().getMemory(); if (askMemory > this.containerMaxMemory) { log.warn("Memory requested: {} > max of {}", askMemory, containerMaxMemory); } operations.add(new ContainerRequestOperation(containerAsk)); return true; } else { return false; } }
addResourceRequest(req.getPriority(), node, req.getCapability(), req, true, req.getNodeLabelExpression()); addResourceRequest(req.getPriority(), rack, req.getCapability(), req, true, req.getNodeLabelExpression()); addResourceRequest(req.getPriority(), rack, req.getCapability(), req, req.getRelaxLocality(), req.getNodeLabelExpression()); req.getCapability(), req, req.getRelaxLocality(), req.getNodeLabelExpression());
addResourceRequest(req.getPriority(), node, req.getCapability(), req, true, req.getNodeLabelExpression()); addResourceRequest(req.getPriority(), rack, req.getCapability(), req, true, req.getNodeLabelExpression()); addResourceRequest(req.getPriority(), rack, req.getCapability(), req, req.getRelaxLocality(), req.getNodeLabelExpression()); req.getCapability(), req, req.getRelaxLocality(), req.getNodeLabelExpression());
this.containers.add(container); final AMRMClient.ContainerRequest requestToBeSubmitted = this.outstandingContainerRequests.peek(); final int currentRequestMemory = requestToBeSubmitted.getCapability().getMemory();
addResourceRequest(req.getPriority(), node, req.getCapability(), req, true, req.getNodeLabelExpression()); addResourceRequest(req.getPriority(), rack, req.getCapability(), req, true, req.getNodeLabelExpression()); addResourceRequest(req.getPriority(), rack, req.getCapability(), req, req.getRelaxLocality(), req.getNodeLabelExpression()); req.getCapability(), req, req.getRelaxLocality(), req.getNodeLabelExpression());
issuedRequest = new AMRMClient.ContainerRequest(issuedRequest.getCapability(), nodes, null,
issuedRequest = new AMRMClient.ContainerRequest(issuedRequest.getCapability(), nodes, null,
@Override public synchronized void removeContainerRequest(T req) { Preconditions.checkArgument(req != null, "Resource request can not be null."); Resource resource = checkAndGetResourceProfile(req.getResourceProfile(), req.getCapability()); Set<String> allRacks = new HashSet<String>(); if (req.getRacks() != null) { allRacks.addAll(req.getRacks()); } allRacks.addAll(resolveRacks(req.getNodes())); // Update resource requests if (req.getNodes() != null) { for (String node : new HashSet<String>(req.getNodes())) { decResourceRequest(req.getPriority(), node, req.getExecutionTypeRequest(), resource, req); } } for (String rack : allRacks) { decResourceRequest(req.getPriority(), rack, req.getExecutionTypeRequest(), resource, req); } decResourceRequest(req.getPriority(), ResourceRequest.ANY, req.getExecutionTypeRequest(), resource, req); }
@Override public synchronized void removeContainerRequest(T req) { Preconditions.checkArgument(req != null, "Resource request can not be null."); Set<String> allRacks = new HashSet<String>(); if (req.getRacks() != null) { allRacks.addAll(req.getRacks()); } allRacks.addAll(resolveRacks(req.getNodes())); // Update resource requests if (req.getNodes() != null) { for (String node : new HashSet<String>(req.getNodes())) { decResourceRequest(req.getPriority(), node, req.getCapability(), req); } } for (String rack : allRacks) { decResourceRequest(req.getPriority(), rack, req.getCapability(), req); } decResourceRequest(req.getPriority(), ResourceRequest.ANY, req.getCapability(), req); }
/** * Add a container request if the request is non-null * @param operations operations to add the entry to * @param containerAsk what to ask for * @return true if a request was added */ private boolean addContainerRequest(List<AbstractRMOperation> operations, AMRMClient.ContainerRequest containerAsk) { if (containerAsk != null) { log.info("Container ask is {} and label = {}", containerAsk, containerAsk.getNodeLabelExpression()); int askMemory = containerAsk.getCapability().getMemory(); if (askMemory > this.containerMaxMemory) { log.warn("Memory requested: {} > max of {}", askMemory, containerMaxMemory); } operations.add(new ContainerRequestOperation(containerAsk)); return true; } else { return false; } }
@Override public synchronized void removeContainerRequest(T req) { Preconditions.checkArgument(req != null, "Resource request can not be null."); Set<String> allRacks = new HashSet<String>(); if (req.getRacks() != null) { allRacks.addAll(req.getRacks()); } allRacks.addAll(resolveRacks(req.getNodes())); // Update resource requests if (req.getNodes() != null) { for (String node : new HashSet<String>(req.getNodes())) { decResourceRequest(req.getPriority(), node, req.getCapability(), req); } } for (String rack : allRacks) { decResourceRequest(req.getPriority(), rack, req.getCapability(), req); } decResourceRequest(req.getPriority(), ResourceRequest.ANY, req.getCapability(), req); }
@Override public synchronized void removeContainerRequest(T req) { Preconditions.checkArgument(req != null, "Resource request can not be null."); Set<String> allRacks = new HashSet<String>(); if (req.getRacks() != null) { allRacks.addAll(req.getRacks()); } allRacks.addAll(resolveRacks(req.getNodes())); // Update resource requests if (req.getNodes() != null) { for (String node : new HashSet<String>(req.getNodes())) { decResourceRequest(req.getPriority(), node, req.getCapability(), req); } } for (String rack : allRacks) { decResourceRequest(req.getPriority(), rack, req.getCapability(), req); } decResourceRequest(req.getPriority(), ResourceRequest.ANY, req.getCapability(), req); }
/** * Match to see whether the container satisfies the request. * We take into consideration that RM has some freedom in rounding * up the allocation and in placing containers on other machines. */ private boolean matchContainerWithPendingRequest(final Container container) { if (this.requestsAfterSentToRM.isEmpty()) { return false; } final AMRMClient.ContainerRequest request = this.requestsAfterSentToRM.peek(); final boolean resourceCondition = container.getResource().getMemory() >= request.getCapability().getMemory(); // TODO[JIRA REEF-35]: check vcores once YARN-2380 is resolved final boolean nodeCondition = request.getNodes() == null || request.getNodes().contains(container.getNodeId().getHost()); final boolean rackCondition = request.getRacks() == null || request.getRacks().contains(this.nodeIdToRackName.get(container.getNodeId().toString())); return resourceCondition && (request.getRelaxLocality() || rackCondition && nodeCondition); }
/** * Query for the resource requirements matching; always false before a request is issued * @param resource * @return */ public synchronized boolean resourceRequirementsMatch(Resource resource) { return issuedRequest != null && Resources.fitsIn(issuedRequest.getCapability(), resource); }
/** * Query for the resource requirements matching; always false before a request is issued * @param resource * @return */ public synchronized boolean resourceRequirementsMatch(Resource resource) { return issuedRequest != null && Resources.fitsIn(issuedRequest.getCapability(), resource); }
private boolean isSameKindOfRequest(final AMRMClient.ContainerRequest r1, final AMRMClient.ContainerRequest r2) { return r1.getPriority().compareTo(r2.getPriority()) == 0 && r1.getCapability().compareTo(r2.getCapability()) == 0 && r1.getRelaxLocality() == r2.getRelaxLocality() && ListUtils.isEqualList(r1.getNodes(), r2.getNodes()) && ListUtils.isEqualList(r1.getRacks(), r2.getRacks()); }