public static void setCapabilityGPU(Resource resource, int gpuCount) { // short-circuit when the GPU count is 0. if (gpuCount <= 0) { return; } resource.setResourceValue(GPU_URI, gpuCount); }
/** * Set the value of the wrapped resource if this object isn't setup to use * percentages. If this object is set to use percentages, this method has * no effect. * * @param name the name of the resource * @param value the value */ void setValue(String name, long value) { if (resource != null) { resource.setResourceValue(name, value); } }
private Resource newResource(long memory, int cpu, int test) { Resource res = newResource(memory, cpu); res.setResourceValue("test", test); return res; }
/** * Get a Resource object with for the maximum allocation possible. * @return a Resource object with the maximum allocation for the scheduler */ public static Resource getResourceTypesMaximumAllocation() { Resource ret = Resource.newInstance(0, 0); for (ResourceInformation entry : resourceTypesArray) { ret.setResourceValue(entry.getName(), entry.getMaximumAllocation()); } return ret; }
@Override public void updateConfiguredResource(Resource res) throws YarnException { LOG.info("Initializing configured FPGA resources for the NodeManager."); List<FpgaResourceAllocator.FpgaDevice> list = FpgaDiscoverer.getInstance().getCurrentFpgaInfo(); List<Integer> minors = new LinkedList<>(); for (FpgaResourceAllocator.FpgaDevice device : list) { minors.add(device.getMinor()); } if (minors.isEmpty()) { LOG.info("Didn't find any usable FPGAs on the NodeManager."); return; } long count = minors.size(); Map<String, ResourceInformation> configuredResourceTypes = ResourceUtils.getResourceTypes(); if (!configuredResourceTypes.containsKey(FPGA_URI)) { throw new YarnException("Wrong configurations, found " + count + " usable FPGAs, however " + FPGA_URI + " resource-type is not configured inside" + " resource-types.xml, please configure it to enable FPGA feature or" + " remove " + FPGA_URI + " from " + YarnConfiguration.NM_RESOURCE_PLUGINS); } res.setResourceValue(FPGA_URI, count); } }
public static Resource multiplyTo(Resource lhs, double by) { int maxLength = ResourceUtils.getNumberOfKnownResourceTypes(); for (int i = 0; i < maxLength; i++) { try { ResourceInformation lhsValue = lhs.getResourceInformation(i); lhs.setResourceValue(i, (long) (lhsValue.getValue() * by)); } catch (ResourceNotFoundException ye) { LOG.warn("Resource is missing:" + ye.getMessage()); continue; } } return lhs; }
@Override public void updateConfiguredResource(Resource res) throws YarnException { LOG.info("Initializing configured GPU resources for the NodeManager."); List<GpuDevice> usableGpus = GpuDiscoverer.getInstance().getGpusUsableByYarn(); if (null == usableGpus || usableGpus.isEmpty()) { String message = "GPU is enabled, but couldn't find any usable GPUs on the " + "NodeManager."; LOG.error(message); // No gpu can be used by YARN. throw new YarnException(message); } long nUsableGpus = usableGpus.size(); Map<String, ResourceInformation> configuredResourceTypes = ResourceUtils.getResourceTypes(); if (!configuredResourceTypes.containsKey(GPU_URI)) { throw new YarnException("Found " + nUsableGpus + " usable GPUs, however " + GPU_URI + " resource-type is not configured inside" + " resource-types.xml, please configure it to enable GPU feature or" + " remove " + GPU_URI + " from " + YarnConfiguration.NM_RESOURCE_PLUGINS); } res.setResourceValue(GPU_URI, nUsableGpus); } }
/** * Helper method to handle Schedulabes with fixed fairshares. * Returns the resources taken by fixed fairshare schedulables, * and adds the remaining to the passed nonFixedSchedulables. */ private static int handleFixedFairShares( Collection<? extends Schedulable> schedulables, Collection<Schedulable> nonFixedSchedulables, boolean isSteadyShare, String type) { int totalResource = 0; for (Schedulable sched : schedulables) { long fixedShare = getFairShareIfFixed(sched, isSteadyShare, type); if (fixedShare < 0) { nonFixedSchedulables.add(sched); } else { Resource target; if (isSteadyShare) { target = ((FSQueue)sched).getSteadyFairShare(); } else { target = sched.getFairShare(); } target.setResourceValue(type, fixedShare); totalResource = (int) Math.min((long)totalResource + (long)fixedShare, Integer.MAX_VALUE); } } return totalResource; }
/** * Get maximum allocation from config, *THIS WILL NOT UPDATE INTERNAL DATA* * @param conf config * @return maximum allocation */ public static Resource fetchMaximumAllocationFromConfig(Configuration conf) { Map<String, ResourceInformation> resourceInformationMap = getResourceInformationMapFromConfig(conf); Resource ret = Resource.newInstance(0, 0); for (ResourceInformation entry : resourceInformationMap.values()) { ret.setResourceValue(entry.getName(), entry.getMaximumAllocation()); } return ret; }
private Resource getMinResourceNormalized(String name, Map<String, Float> effectiveMinRatio, Resource minResource) { Resource ret = Resource.newInstance(minResource); int maxLength = ResourceUtils.getNumberOfKnownResourceTypes(); for (int i = 0; i < maxLength; i++) { ResourceInformation nResourceInformation = minResource .getResourceInformation(i); Float ratio = effectiveMinRatio.get(nResourceInformation.getName()); if (ratio != null) { ret.setResourceValue(i, (long) (nResourceInformation.getValue() * ratio.floatValue())); if (LOG.isDebugEnabled()) { LOG.debug("Updating min resource for Queue: " + name + " as " + ret.getResourceInformation(i) + ", Actual resource: " + nResourceInformation.getValue() + ", ratio: " + ratio.floatValue()); } } } return ret; }
public static Resource multiplyAndRoundDown(Resource lhs, double by) { Resource out = clone(lhs); int maxLength = ResourceUtils.getNumberOfKnownResourceTypes(); for (int i = 0; i < maxLength; i++) { try { ResourceInformation lhsValue = lhs.getResourceInformation(i); out.setResourceValue(i, (long) (lhsValue.getValue() * by)); } catch (ResourceNotFoundException ye) { LOG.warn("Resource is missing:" + ye.getMessage()); continue; } } return out; }
private Resource getTaskResourceCapability() throws YarnRuntimeException { if (containerMemory < -1 || containerMemory == 0) { throw new YarnRuntimeException("Value of AM memory '" + containerMemory + "' has to be greater than 0"); } if (containerVirtualCores < -1 || containerVirtualCores == 0) { throw new YarnRuntimeException( "Value of AM vcores '" + containerVirtualCores + "' has to be greater than 0"); } Resource resourceCapability = Resource.newInstance(containerMemory, containerVirtualCores); containerMemory = containerMemory == -1 ? DEFAULT_CONTAINER_MEMORY : containerMemory; containerVirtualCores = containerVirtualCores == -1 ? DEFAULT_CONTAINER_VCORES : containerVirtualCores; resourceCapability.setMemorySize(containerMemory); resourceCapability.setVirtualCores(containerVirtualCores); for (Map.Entry<String, Long> entry : containerResources.entrySet()) { resourceCapability.setResourceValue(entry.getKey(), entry.getValue()); } return resourceCapability; } }
public static Resource addTo(Resource lhs, Resource rhs) { int maxLength = ResourceUtils.getNumberOfKnownResourceTypes(); for (int i = 0; i < maxLength; i++) { try { ResourceInformation rhsValue = rhs.getResourceInformation(i); ResourceInformation lhsValue = lhs.getResourceInformation(i); lhs.setResourceValue(i, lhsValue.getValue() + rhsValue.getValue()); } catch (ResourceNotFoundException ye) { LOG.warn("Resource is missing:" + ye.getMessage()); continue; } } return lhs; }
public static Resource subtractFrom(Resource lhs, Resource rhs) { int maxLength = ResourceUtils.getNumberOfKnownResourceTypes(); for (int i = 0; i < maxLength; i++) { try { ResourceInformation rhsValue = rhs.getResourceInformation(i); ResourceInformation lhsValue = lhs.getResourceInformation(i); lhs.setResourceValue(i, lhsValue.getValue() - rhsValue.getValue()); } catch (ResourceNotFoundException ye) { LOG.warn("Resource is missing:" + ye.getMessage()); continue; } } return lhs; }
/** * Multiply {@code rhs} by {@code by}, and add the result to {@code lhs} * without creating any new {@link Resource} object */ public static Resource multiplyAndAddTo( Resource lhs, Resource rhs, double by) { int maxLength = ResourceUtils.getNumberOfKnownResourceTypes(); for (int i = 0; i < maxLength; i++) { try { ResourceInformation rhsValue = rhs.getResourceInformation(i); ResourceInformation lhsValue = lhs.getResourceInformation(i); long convertedRhs = (long) (rhsValue.getValue() * by); lhs.setResourceValue(i, lhsValue.getValue() + convertedRhs); } catch (ResourceNotFoundException ye) { LOG.warn("Resource is missing:" + ye.getMessage()); continue; } } return lhs; }
public static Resource getResourceTypesMinimumAllocation() { Resource ret = Resource.newInstance(0, 0); for (ResourceInformation entry : resourceTypesArray) { String name = entry.getName(); if (name.equals(ResourceInformation.MEMORY_MB.getName())) { ret.setMemorySize(entry.getMinimumAllocation()); } else if (name.equals(ResourceInformation.VCORES.getName())) { Long tmp = entry.getMinimumAllocation(); if (tmp > Integer.MAX_VALUE) { tmp = (long) Integer.MAX_VALUE; } ret.setVirtualCores(tmp.intValue()); } else { ret.setResourceValue(name, entry.getMinimumAllocation()); } } return ret; }
@Override public Resource multiplyAndNormalizeUp(Resource r, double[] by, Resource stepFactor) { Resource ret = Resource.newInstance(r); int maxLength = ResourceUtils.getNumberOfKnownResourceTypes(); for (int i = 0; i < maxLength; i++) { ResourceInformation rResourceInformation = r.getResourceInformation(i); ResourceInformation stepFactorResourceInformation = stepFactor .getResourceInformation(i); long rValue = rResourceInformation.getValue(); long stepFactorValue = stepFactorResourceInformation.getValue(); ret.setResourceValue(i, ResourceCalculator .roundUp((long) Math.ceil(rValue * by[i]), stepFactorValue)); } return ret; }
/** * Get resource by multiplying the cluster resource and the percentage of * each resource respectively. Return the absolute resource if either * {@code percentages} or {@code clusterResource} is null. * * @param clusterResource the cluster resource * @return resource the resulting resource */ public Resource getResource(Resource clusterResource) { if (percentages != null && clusterResource != null) { long memory = (long) (clusterResource.getMemorySize() * percentages[0]); int vcore = (int) (clusterResource.getVirtualCores() * percentages[1]); Resource res = Resource.newInstance(memory, vcore); ResourceInformation[] clusterInfo = clusterResource.getResources(); for (int i = 2; i < clusterInfo.length; i++) { res.setResourceValue(i, (long)(clusterInfo[i].getValue() * percentages[i])); } return res; } else { return resource; } }
@Test public void testCalculateClusterAndFairRatios() { Map<String, Integer> index = ResourceUtils.getResourceTypeIndex(); Resource used = Resources.createResource(10, 5); Resource capacity = Resources.createResource(100, 10); float[][] shares = new float[3][2]; DominantResourceFairnessComparatorN comparator = new DominantResourceFairnessComparatorN(); used.setResourceValue("test", 2L); capacity.setResourceValue("test", 5L); int dominant = comparator.calculateClusterAndFairRatios(used, capacity, shares, 1.0f); assertEquals("Calculated usage ratio for memory (10MB out of 100MB) is " + "incorrect", 0.1, shares[index.get(ResourceInformation.MEMORY_MB.getName())][0], .00001); assertEquals("Calculated usage ratio for vcores (5 out of 10) is " + "incorrect", 0.5, shares[index.get(ResourceInformation.VCORES.getName())][0], .00001); assertEquals("Calculated usage ratio for test resource (2 out of 5) is " + "incorrect", 0.4, shares[index.get("test")][0], .00001); assertEquals("The wrong dominant resource index was returned", index.get(ResourceInformation.VCORES.getName()).intValue(), dominant); }
@Test public void testCalculateMinShareRatios() { Map<String, Integer> index = ResourceUtils.getResourceTypeIndex(); Resource used = Resources.createResource(10, 5); Resource minShares = Resources.createResource(5, 10); float[][] ratios = new float[3][3]; DominantResourceFairnessComparatorN comparator = new DominantResourceFairnessComparatorN(); used.setResourceValue("test", 2L); minShares.setResourceValue("test", 0L); comparator.calculateMinShareRatios(used, minShares, ratios); assertEquals("Calculated min share ratio for memory (10MB out of 5MB) is " + "incorrect", 2.0, ratios[index.get(ResourceInformation.MEMORY_MB.getName())][2], .00001f); assertEquals("Calculated min share ratio for vcores (5 out of 10) is " + "incorrect", 0.5, ratios[index.get(ResourceInformation.VCORES.getName())][2], .00001f); assertEquals("Calculated min share ratio for test resource (0 out of 5) is " + "incorrect", Float.POSITIVE_INFINITY, ratios[index.get("test")][2], 0.00001f); }