@Override public boolean isInvalidDivisor(Resource r) { for (ResourceInformation res : r.getResources()) { if (res.getValue() == 0L) { return true; } } return false; }
public String getSchedulerResourceTypes() { if (minAllocResource != null) { return Arrays.toString(minAllocResource.getResource().getResources()); } return null; }
/** * Calculate a resource's min share ratios. The {@code ratios} array will be * populated with the {@code resource} divided by {@code minShare} for each * resource type. If the min shares are 5 MB and 10 vcores, and the usage * ({@code resource}) is 10 MB and 5 CPU, the ratios will be 2 and 0.5. * * The {@code ratios} array must be <i>n</i> x 3, where <i>n</i> is the * number of resource types. Only the third index of the inner arrays in * the {@code ratios} array will be used, e.g. {@code ratios[x][2]}. * * @param resource the resource for which to calculate min shares * @param minShare the min share * @param ratios the share ratios array to populate */ @VisibleForTesting void calculateMinShareRatios(Resource resource, Resource minShare, float[][] ratios) { ResourceInformation[] resourceInfo = resource.getResources(); ResourceInformation[] minShareInfo = minShare.getResources(); for (int i = 0; i < minShareInfo.length; i++) { ratios[i][2] = resourceInfo[i].getValue() / (float) minShareInfo[i].getValue(); } }
@Override public boolean equals(Object obj) { if (this == obj) { return true; } if (obj == null) { return false; } if (!(obj instanceof Resource)) { return false; } Resource other = (Resource) obj; ResourceInformation[] otherVectors = other.getResources(); if (resources.length != otherVectors.length) { return false; } for (int i = 0; i < resources.length; i++) { ResourceInformation a = resources[i]; ResourceInformation b = otherVectors[i]; if ((a != b) && ((a == null) || !a.equals(b))) { return false; } } return true; }
@Override public int compareTo(Resource other) { ResourceInformation[] otherResources = other.getResources(); int arrLenThis = this.resources.length; int arrLenOther = otherResources.length; // compare memory and vcores first(in that order) to preserve // existing behavior. for (int i = 0; i < arrLenThis; i++) { ResourceInformation otherEntry; try { otherEntry = otherResources[i]; } catch (ArrayIndexOutOfBoundsException e) { // For two vectors with different size and same prefix. Shorter vector // goes first. return 1; } ResourceInformation entry = resources[i]; long diff = entry.compareTo(otherEntry); if (diff > 0) { return 1; } else if (diff < 0) { return -1; } } if (arrLenThis < arrLenOther) { return -1; } return 0; }
private void updateUsageMap(Resource allocated, long deltaUsedMillis, Map<String, AtomicLong> targetMap) { for (ResourceInformation entry : allocated.getResources()) { AtomicLong resourceUsed; if (!targetMap.containsKey(entry.getName())) { resourceUsed = new AtomicLong(0); targetMap.put(entry.getName(), resourceUsed); } resourceUsed = targetMap.get(entry.getName()); resourceUsed.addAndGet((entry.getValue() * deltaUsedMillis) / DateUtils.MILLIS_PER_SECOND); } }
/** * Calculate the shares for {@code first} and {@code second} according to * {@code clusterRes}, and store the results in {@code firstShares} and * {@code secondShares}, respectively. All parameters must be non-null. * @param clusterRes the array of ResourceInformation instances that * represents the cluster's maximum resources * @param first the first resource to compare * @param second the second resource to compare * @param firstShares an array to store the shares for the first resource * @param secondShares an array to store the shares for the second resource * @return -1.0, 0.0, or 1.0, depending on whether the max share of the first * resource is less than, equal to, or greater than the max share of the * second resource, respectively * @throws NullPointerException if any parameter is null */ private void calculateShares(ResourceInformation[] clusterRes, Resource first, Resource second, double[] firstShares, double[] secondShares) { ResourceInformation[] firstRes = first.getResources(); ResourceInformation[] secondRes = second.getResources(); int maxLength = ResourceUtils.getNumberOfKnownResourceTypes(); for (int i = 0; i < maxLength; i++) { firstShares[i] = calculateShare(clusterRes[i], firstRes[i]); secondShares[i] = calculateShare(clusterRes[i], secondRes[i]); } }
@Override public float divide(Resource clusterResource, Resource numerator, Resource denominator) { int nKnownResourceTypes = ResourceUtils.getNumberOfKnownResourceTypes(); ResourceInformation[] clusterRes = clusterResource.getResources(); // We have to provide the calculateShares() method with somewhere to store // the shares. We don't actually need these shares afterwards. double[] numeratorShares = new double[nKnownResourceTypes]; double[] denominatorShares = new double[nKnownResourceTypes]; // We also have to provide a place for calculateShares() to store the max // shares so that we can use them. double[] max = new double[2]; calculateShares(clusterRes, numerator, denominator, numeratorShares, denominatorShares, max); return (float) (max[0] / max[1]); }
public static Map<String, Long> convertStringLongMapProtoListToMap( List<YarnProtos.StringLongMapProto> pList) { Resource tmp = Resource.newInstance(0, 0); Map<String, Long> ret = new HashMap<>(); for (ResourceInformation entry : tmp.getResources()) { ret.put(entry.getName(), 0L); } if (pList != null) { for (YarnProtos.StringLongMapProto p : pList) { ret.put(p.getKey(), p.getValue()); } } return ret; }
@InterfaceAudience.Private @InterfaceStability.Unstable public static void copy(Resource source, Resource dest) { for (ResourceInformation entry : source.getResources()) { dest.setResourceInformation(entry.getName(), entry); } }
private AggregateAppResourceUsage getRunningAggregateAppResourceUsage() { long currentTimeMillis = System.currentTimeMillis(); // Don't walk the whole container list if the resources were computed // recently. if ((currentTimeMillis - lastMemoryAggregateAllocationUpdateTime) > MEM_AGGREGATE_ALLOCATION_CACHE_MSECS) { Map<String, Long> resourceSecondsMap = new HashMap<>(); for (RMContainer rmContainer : this.liveContainers.values()) { long usedMillis = currentTimeMillis - rmContainer.getCreationTime(); Resource resource = rmContainer.getContainer().getResource(); for (ResourceInformation entry : resource.getResources()) { long value = RMServerUtils .getOrDefault(resourceSecondsMap, entry.getName(), 0L); value += entry.getValue() * usedMillis / DateUtils.MILLIS_PER_SECOND; resourceSecondsMap.put(entry.getName(), value); } } lastMemoryAggregateAllocationUpdateTime = currentTimeMillis; lastResourceSecondsMap = resourceSecondsMap; } return new AggregateAppResourceUsage(lastResourceSecondsMap); }
allowQueuesBalanceAfterAllQueuesSatisfied; stepFactor = Resource.newInstance(0, 0); for (ResourceInformation ri : stepFactor.getResources()) { ri.setValue(1);
/** * Get resource by multiplying the cluster resource and the percentage of * each resource respectively. Return the absolute resource if either * {@code percentages} or {@code clusterResource} is null. * * @param clusterResource the cluster resource * @return resource the resulting resource */ public Resource getResource(Resource clusterResource) { if (percentages != null && clusterResource != null) { long memory = (long) (clusterResource.getMemorySize() * percentages[0]); int vcore = (int) (clusterResource.getVirtualCores() * percentages[1]); Resource res = Resource.newInstance(memory, vcore); ResourceInformation[] clusterInfo = clusterResource.getResources(); for (int i = 2; i < clusterInfo.length; i++) { res.setResourceValue(i, (long)(clusterInfo[i].getValue() * percentages[i])); } return res; } else { return resource; } }
@Override public boolean equals(Object obj) { if (this == obj) { return true; } if (obj == null || !(obj instanceof Resource)) { return false; } Resource other = (Resource) obj; if (getMemorySize() != other.getMemorySize() || getVirtualCores() != other.getVirtualCores()) { return false; } if (resources.length > 2) { ResourceInformation[] otherVectors = other.getResources(); if (resources.length != otherVectors.length) { return false; } for (int i = 2; i < resources.length; i++) { ResourceInformation a = resources[i]; ResourceInformation b = otherVectors[i]; if ((a != b) && ((a == null) || !a.equals(b))) { return false; } } } return true; }
static ResourceProto getProto(Resource r) { final ResourcePBImpl pb; if (r instanceof ResourcePBImpl) { pb = (ResourcePBImpl) r; } else { pb = new ResourcePBImpl(); pb.setMemorySize(r.getMemorySize()); pb.setVirtualCores(r.getVirtualCores()); for(ResourceInformation res : r.getResources()) { pb.setResourceInformation(res.getName(), res); } } return pb.getProto(); }
@Test public void testCalculateMinShareRatios2() { ResourceUtils.resetResourceTypes(new Configuration()); Resource used = Resources.createResource(10, 5); Resource minShares = Resources.createResource(5, 10); DominantResourceFairnessComparator2 comparator = new DominantResourceFairnessComparator2(); double[] ratios = comparator.calculateMinShareRatios(used.getResources(), minShares.getResources()); assertEquals("Calculated min share ratio for memory (10MB out of 5MB) is " + "incorrect", 2.0, ratios[Resource.MEMORY_INDEX], .00001f); assertEquals("Calculated min share ratio for vcores (5 out of 10) is " + "incorrect", 0.5, ratios[Resource.VCORES_INDEX], .00001f); }
@Test public void testGetResourceInformation() throws Exception { Configuration conf = new YarnConfiguration(); Map<String, Resource> testRun = new HashMap<>(); setupResourceTypes(conf, "resource-types-4.xml"); // testRun.put("node-resources-1.xml", Resource.newInstance(1024, 1)); Resource test3Resources = Resource.newInstance(0, 0); test3Resources.setResourceInformation("resource1", ResourceInformation.newInstance("resource1", "Gi", 5L)); test3Resources.setResourceInformation("resource2", ResourceInformation.newInstance("resource2", "m", 2L)); test3Resources.setResourceInformation("yarn.io/gpu", ResourceInformation.newInstance("yarn.io/gpu", "", 1)); testRun.put("node-resources-2.xml", test3Resources); for (Map.Entry<String, Resource> entry : testRun.entrySet()) { String resourceFile = entry.getKey(); ResourceUtils.resetNodeResources(); File source = new File( conf.getClassLoader().getResource(resourceFile).getFile()); nodeResourcesFile = new File(source.getParent(), "node-resources.xml"); FileUtils.copyFile(source, nodeResourcesFile); Map<String, ResourceInformation> actual = ResourceUtils .getNodeResourceInformation(conf); Assert.assertEquals(actual.size(), entry.getValue().getResources().length); for (ResourceInformation resInfo : entry.getValue().getResources()) { Assert.assertEquals(resInfo, actual.get(resInfo.getName())); } } }
.getNodeResourceInformation(conf); Assert.assertEquals(actual.size(), entry.getValue().getResources().length); for (ResourceInformation resInfo : entry.getValue().getResources()) { Assert.assertEquals(resInfo, actual.get(resInfo.getName()));
@InterfaceAudience.Private @InterfaceStability.Unstable public static Resource newInstance(Resource resource) { Resource ret; int numberOfKnownResourceTypes = ResourceUtils .getNumberOfKnownResourceTypes(); if (numberOfKnownResourceTypes > 2) { ret = new LightWeightResource(resource.getMemorySize(), resource.getVirtualCores(), resource.getResources()); } else { ret = new LightWeightResource(resource.getMemorySize(), resource.getVirtualCores()); } return ret; }
@Test public void testCalculateClusterAndFairRatios2() { ResourceUtils.resetResourceTypes(new Configuration()); Resource used = Resources.createResource(10, 5); Resource capacity = Resources.createResource(100, 10); double[] shares = new double[2]; DominantResourceFairnessComparator2 comparator = new DominantResourceFairnessComparator2(); int dominant = comparator.calculateClusterAndFairRatios(used.getResources(), 1.0f, capacity.getResources(), shares); assertEquals("Calculated usage ratio for memory (10MB out of 100MB) is " + "incorrect", 0.1, shares[Resource.MEMORY_INDEX], .00001); assertEquals("Calculated usage ratio for vcores (5 out of 10) is " + "incorrect", 0.5, shares[Resource.VCORES_INDEX], .00001); assertEquals("The wrong dominant resource index was returned", Resource.VCORES_INDEX, dominant); }