private Resource parseResource(Map profileInfo) throws IOException { Resource resource = Resource.newInstance(0, 0); Iterator iterator = profileInfo.entrySet().iterator(); Map<String, ResourceInformation> resourceTypes = ResourceUtils .getResourceTypes(); while (iterator.hasNext()) { Map.Entry resourceEntry = (Map.Entry) iterator.next(); String resourceName = resourceEntry.getKey().toString(); ResourceInformation resourceValue = fromString(resourceName, resourceEntry.getValue().toString()); if (resourceName.equals(MEMORY)) { resource.setMemorySize(resourceValue.getValue()); continue; } if (resourceName.equals(VCORES)) { resource .setVirtualCores(Long.valueOf(resourceValue.getValue()).intValue()); continue; } if (resourceTypes.containsKey(resourceName)) { resource.setResourceInformation(resourceName, resourceValue); } else { throw new IOException("Unrecognized resource type '" + resourceName + "'. Recognized resource types are '" + resourceTypes.keySet() + "'"); } } return resource; }
resource.setResourceInformation(resName, ri); } else { throw new IllegalArgumentException("Invalid resource value: " +
private void updateResourceValuesFromConfig(Set<String> resourceTypes, Resource resource, String[] splits) { // If key is not a valid type, skip it. if (!resourceTypes.contains(splits[0])) { return; } String units = getUnits(splits[1]); Long resourceValue = Long .valueOf(splits[1].substring(0, splits[1].length() - units.length())); // Convert all incoming units to MB if units is configured. if (!units.isEmpty()) { resourceValue = UnitsConversionUtil.convert(units, "Mi", resourceValue); } // map it based on key. AbsoluteResourceType resType = AbsoluteResourceType .valueOf(StringUtils.toUpperCase(splits[0].trim())); switch (resType) { case MEMORY : resource.setMemorySize(resourceValue); break; case VCORES : resource.setVirtualCores(resourceValue.intValue()); break; default : resource.setResourceInformation(splits[0].trim(), ResourceInformation .newInstance(splits[0].trim(), units, resourceValue)); break; } }
@InterfaceAudience.Private @InterfaceStability.Unstable public static void copy(Resource source, Resource dest) { for (ResourceInformation entry : source.getResources()) { dest.setResourceInformation(entry.getName(), entry); } }
private Resource checkAndGetResourceProfile(String profile, Resource overrideResource) { Resource returnResource = overrideResource; // if application requested a non-empty/null profile, and the if (profile != null && !profile.isEmpty()) { if (resourceProfilesMap == null || (!resourceProfilesMap.containsKey( profile))) { throw new InvalidContainerRequestException( "Invalid profile name specified=" + profile + ( resourceProfilesMap == null ? "" : (", valid profile names are " + resourceProfilesMap .keySet()))); } returnResource = Resources.clone(resourceProfilesMap.get(profile)); for (ResourceInformation info : overrideResource .getAllResourcesListCopy()) { if (info.getValue() > 0) { returnResource.setResourceInformation(info.getName(), info); } } } return returnResource; }
for (Map.Entry<String, ResourceInformation> entry : resourceInformation .entrySet()) { ret.setResourceInformation(entry.getKey(), entry.getValue()); LOG.debug("Setting key " + entry.getKey() + " to " + entry.getValue());
/** * An easy way to create resources other than memory and vcores for tests. * @param memory memory * @param vcores vcores * @param nameToValues resource types other than memory and vcores. * @return created resource */ public static Resource createResource(long memory, int vcores, Map<String, Integer> nameToValues) { Resource res = Resource.newInstance(memory, vcores); if (nameToValues != null) { for (Map.Entry<String, Integer> entry : nameToValues.entrySet()) { res.setResourceInformation(entry.getKey(), ResourceInformation .newInstance(entry.getKey(), "", entry.getValue())); } } return res; } }
public Resource createResource(long memory, int vCores, long resource2) { Resource ret = Resource.newInstance(memory, vCores); ret.setResourceInformation(EXTRA_RESOURCE_TYPE, ResourceInformation.newInstance(EXTRA_RESOURCE_TYPE, resource2)); return ret; }
public static Resource componentwiseMax(Resource lhs, Resource rhs) { Resource ret = createResource(0); int maxLength = ResourceUtils.getNumberOfKnownResourceTypes(); for (int i = 0; i < maxLength; i++) { try { ResourceInformation rhsValue = rhs.getResourceInformation(i); ResourceInformation lhsValue = lhs.getResourceInformation(i); ResourceInformation outInfo = lhsValue.getValue() > rhsValue.getValue() ? lhsValue : rhsValue; ret.setResourceInformation(i, outInfo); } catch (ResourceNotFoundException ye) { LOG.warn("Resource is missing:" + ye.getMessage()); continue; } } return ret; }
public static Resource newResource(long memory, int vCores, Map<String, String> customResources) { Resource resource = RECORD_FACTORY.newRecordInstance(Resource.class); resource.setMemorySize(memory); resource.setVirtualCores(vCores); for (Map.Entry<String, String> customResource : customResources.entrySet()) { String resourceName = customResource.getKey(); ResourceInformation resourceInformation = createResourceInformation(resourceName, customResource.getValue()); resource.setResourceInformation(resourceName, resourceInformation); } return resource; }
public static Resource componentwiseMin(Resource lhs, Resource rhs) { Resource ret = createResource(0); int maxLength = ResourceUtils.getNumberOfKnownResourceTypes(); for (int i = 0; i < maxLength; i++) { try { ResourceInformation rhsValue = rhs.getResourceInformation(i); ResourceInformation lhsValue = lhs.getResourceInformation(i); ResourceInformation outInfo = lhsValue.getValue() < rhsValue.getValue() ? lhsValue : rhsValue; ret.setResourceInformation(i, outInfo); } catch (ResourceNotFoundException ye) { LOG.warn("Resource is missing:" + ye.getMessage()); continue; } } return ret; }
resourceInformation.setUnits(resourceReq.getUnits()); resourceInformation.setValue(resourceReq.getValue()); capability.setResourceInformation(resourceName, resourceInformation);
test3Resources.setResourceInformation("resource1", ResourceInformation.newInstance("resource1", "T", 5L)); test3Resources.setResourceInformation("resource2", ResourceInformation.newInstance("resource2", "M", 2L)); test3Resources.setResourceInformation("yarn.io/gpu", ResourceInformation.newInstance("yarn.io/gpu", "", 1)); testRun.put("node-resources-3.xml", test3Resources);
@Test public void testGetResourceInformation() throws Exception { Configuration conf = new YarnConfiguration(); Map<String, Resource> testRun = new HashMap<>(); setupResourceTypes(conf, "resource-types-4.xml"); // testRun.put("node-resources-1.xml", Resource.newInstance(1024, 1)); Resource test3Resources = Resource.newInstance(0, 0); test3Resources.setResourceInformation("resource1", ResourceInformation.newInstance("resource1", "Gi", 5L)); test3Resources.setResourceInformation("resource2", ResourceInformation.newInstance("resource2", "m", 2L)); test3Resources.setResourceInformation("yarn.io/gpu", ResourceInformation.newInstance("yarn.io/gpu", "", 1)); testRun.put("node-resources-2.xml", test3Resources); for (Map.Entry<String, Resource> entry : testRun.entrySet()) { String resourceFile = entry.getKey(); ResourceUtils.resetNodeResources(); File source = new File( conf.getClassLoader().getResource(resourceFile).getFile()); nodeResourcesFile = new File(source.getParent(), "node-resources.xml"); FileUtils.copyFile(source, nodeResourcesFile); Map<String, ResourceInformation> actual = ResourceUtils .getNodeResourceInformation(conf); Assert.assertEquals(actual.size(), entry.getValue().getResources().length); for (ResourceInformation resInfo : entry.getValue().getResources()) { Assert.assertEquals(resInfo, actual.get(resInfo.getName())); } } }
private Resource parseResourceFromString(String p) { String[] resource = p.split(":"); Resource res; if (resource.length == 1) { res = Resources.createResource(Integer.valueOf(resource[0])); } else { res = Resources.createResource(Integer.valueOf(resource[0]), Integer.valueOf(resource[1])); if (resource.length > 2) { // Using the same order of resources from ResourceUtils, set resource // informations. ResourceInformation[] storedResourceInfo = ResourceUtils .getResourceTypesArray(); for (int i = 2; i < resource.length; i++) { res.setResourceInformation(storedResourceInfo[i].getName(), ResourceInformation.newInstance(storedResourceInfo[i].getName(), storedResourceInfo[i].getUnits(), Integer.valueOf(resource[i]))); } } } return res; }
resourceInformation.setUnits(resourceRequest.getUnits()); resourceInformation.setValue(resourceRequest.getValue()); this.resourceCapability.setResourceInformation(resourceName, resourceInformation);
@Override public Resource normalize(Resource r, Resource minimumResource, Resource maximumResource, Resource stepFactor) { Resource ret = Resource.newInstance(r); int maxLength = ResourceUtils.getNumberOfKnownResourceTypes(); for (int i = 0; i < maxLength; i++) { ResourceInformation rResourceInformation = r.getResourceInformation(i); ResourceInformation minimumResourceInformation = minimumResource .getResourceInformation(i); ResourceInformation maximumResourceInformation = maximumResource .getResourceInformation(i); ResourceInformation stepFactorResourceInformation = stepFactor .getResourceInformation(i); ResourceInformation tmp = ret.getResourceInformation(i); long rValue = rResourceInformation.getValue(); long value = Math.max(rValue, minimumResourceInformation.getValue()); if (stepFactorResourceInformation.getValue() != 0) { value = roundUp(value, stepFactorResourceInformation.getValue()); } tmp.setValue(Math.min(value, maximumResourceInformation.getValue())); ret.setResourceInformation(i, tmp); } return ret; }
resource.setResourceInformation("memory-mb", ResourceInformation.newInstance("memory-mb", "G", 1024)); resource.setResourceInformation("resource1", ResourceInformation.newInstance("resource1", "T", 1)); resource.setResourceInformation("resource2", ResourceInformation.newInstance("resource2", "M", 1));