@Override public void recover(InputStream is, NodeAttributesManager mgr) throws IOException { NodesToAttributesMappingRequest request = new NodesToAttributesMappingRequestPBImpl( YarnServerResourceManagerServiceProtos .NodesToAttributesMappingRequestProto .parseDelimitedFrom(is)); mgr.addNodeAttributes(getNodeToAttributesMap(request)); }
@Override public void recover(InputStream is, NodeAttributesManager mgr) throws IOException { NodesToAttributesMappingRequest request = new NodesToAttributesMappingRequestPBImpl( YarnServerResourceManagerServiceProtos .NodesToAttributesMappingRequestProto .parseDelimitedFrom(is)); mgr.removeNodeAttributes(getNodeToAttributesMap(request)); }
@Override public void recover(InputStream is, NodeAttributesManager mgr) throws IOException { NodesToAttributesMappingRequest request = new NodesToAttributesMappingRequestPBImpl( YarnServerResourceManagerServiceProtos .NodesToAttributesMappingRequestProto .parseDelimitedFrom(is)); mgr.replaceNodeAttributes(NodeAttribute.PREFIX_CENTRALIZED, getNodeToAttributesMap(request)); }
switch (request.getOperation()) { case ADD: nodeAttributesManager.addNodeAttributes(nodeAttributeMapping); break; case REMOVE: nodeAttributesManager.removeNodeAttributes(nodeAttributeMapping); break; case REPLACE: nodeAttributesManager.replaceNodeAttributes( NodeAttribute.PREFIX_CENTRALIZED, nodeAttributeMapping); break;
attributesManager.addNodeAttributes(toAddAttributes); nodeAttributes = attributesManager.getAttributesForNode(HOSTNAMES[0]); Assert.assertEquals(8, nodeAttributes.size()); nodeAttributes = attributesManager.getAttributesForNode(HOSTNAMES[1]); Assert.assertEquals(4, nodeAttributes.size()); .getClusterNodeAttributes(Sets.newHashSet(PREFIXES[0])); Assert.assertEquals(3, allAttributesPerPrefix.size()); allAttributesPerPrefix = attributesManager .getClusterNodeAttributes(Sets.newHashSet(PREFIXES[1])); Assert.assertEquals(5, allAttributesPerPrefix.size()); allAttributesPerPrefix = attributesManager .getClusterNodeAttributes(Sets.newHashSet(PREFIXES[2])); Assert.assertEquals(2, allAttributesPerPrefix.size()); NodeAttributeType.STRING, "anyValue")); toRemoveAttributes.put(HOSTNAMES[0], attributes2rm1); attributesManager.removeNodeAttributes(toRemoveAttributes); nodeAttributes = attributesManager.getAttributesForNode(HOSTNAMES[0]); Assert.assertEquals(7, nodeAttributes.size()); NodeAttributeType.STRING, "anyValue")); toRemoveAttributes.put(HOSTNAMES[0], attributes2rm1); attributesManager.removeNodeAttributes(toRemoveAttributes); nodeAttributes = attributesManager.getAttributesForNode(HOSTNAMES[0]);
attributesManager.addNodeAttributes(toAddAttributes); nodeAttributes = attributesManager.getAttributesForNode(HOSTNAMES[0]); Assert.assertEquals(3, nodeAttributes.size()); createAttributesForTest(NodeAttribute.PREFIX_DISTRIBUTED, 10, "dist-node-attribute", "dist_v1")); attributesManager.addNodeAttributes(toAddAttributes); nodeAttributes = attributesManager.getAttributesForNode(HOSTNAMES[0]); Assert.assertEquals(13, nodeAttributes.size()); clusterAttributes = attributesManager.getClusterNodeAttributes( Sets.newHashSet(NodeAttribute.PREFIX_DISTRIBUTED, PREFIXES[0])); Assert.assertEquals(13, clusterAttributes.size()); attributesManager.replaceNodeAttributes(NodeAttribute.PREFIX_DISTRIBUTED, ImmutableMap.of(HOSTNAMES[0], toReplaceAttributes)); nodeAttributes = attributesManager.getAttributesForNode(HOSTNAMES[0]); Assert.assertEquals(8, nodeAttributes.size()); clusterAttributes = attributesManager.getClusterNodeAttributes( Sets.newHashSet(NodeAttribute.PREFIX_DISTRIBUTED, PREFIXES[0])); Assert.assertEquals(8, clusterAttributes.size()); createAttributesForTest(NodeAttribute.PREFIX_DISTRIBUTED, 1, "dist-node-attribute-v2", "dist_v3"); attributesManager.replaceNodeAttributes(NodeAttribute.PREFIX_DISTRIBUTED, ImmutableMap.of(HOSTNAMES[0], toReplaceAttributes)); nodeAttributes = attributesManager.getAttributesForNode(HOSTNAMES[0]); Assert.assertEquals(4, nodeAttributes.size()); clusterAttributes = attributesManager.getClusterNodeAttributes( Sets.newHashSet(NodeAttribute.PREFIX_DISTRIBUTED));
attributesManager.addNodeAttributes(toAddAttributes); nodeAttributes = attributesManager.getAttributesForNode(HOSTNAMES[0]); toAddAttributes.put(HOSTNAMES[1], createAttributesForTest(PREFIXES[0], 2, "A", "host2_v1")); attributesManager.addNodeAttributes(toAddAttributes); nodeAttributes = attributesManager.getAttributesForNode(HOSTNAMES[0]); Assert.assertEquals(3, nodeAttributes.size()); nodeAttributes = attributesManager.getAttributesForNode(HOSTNAMES[1]); Assert.assertEquals(2, nodeAttributes.size()); Assert.assertTrue(sameAttributeSet(toAddAttributes.get(HOSTNAMES[1]), .getClusterNodeAttributes(Sets.newHashSet(PREFIXES[0])); Assert.assertEquals(3, clusterAttributes.size()); .getClusterNodeAttributes(Sets.newHashSet("non_exist_prefix")); Assert.assertEquals(0, clusterAttributes.size()); clusterAttributes = attributesManager.getClusterNodeAttributes(null); Assert.assertEquals(3, clusterAttributes.size()); attributesManager.addNodeAttributes(toAddAttributes); nodeAttributes = attributesManager.getAttributesForNode(HOSTNAMES[0]); Assert.assertEquals(13, nodeAttributes.size()); nodeAttributes = attributesManager.getAttributesForNode(HOSTNAMES[1]);
@Override public Set<NodeAttribute> getAllNodeAttributes() { NodeAttributesManager attrMgr = context.getNodeAttributesManager(); Map<NodeAttribute, AttributeValue> nodeattrs = attrMgr.getAttributesForNode(hostName); return nodeattrs.keySet(); } }
@Override public GetClusterNodeAttributesResponse getClusterNodeAttributes( GetClusterNodeAttributesRequest request) throws YarnException, IOException { NodeAttributesManager attributesManager = rmContext.getNodeAttributesManager(); Set<NodeAttribute> attributes = attributesManager.getClusterNodeAttributes(null); GetClusterNodeAttributesResponse response = GetClusterNodeAttributesResponse.newInstance( attributes.stream().map(attr -> NodeAttributeInfo.newInstance(attr)) .collect(Collectors.toSet())); return response; }
@Before public void init() throws IOException { Configuration conf = new Configuration(); attributesManager = new NodeAttributesManagerImpl(); conf.setClass(YarnConfiguration.FS_NODE_ATTRIBUTE_STORE_IMPL_CLASS, FileSystemNodeAttributeStore.class, NodeAttributeStore.class); conf = NodeAttributeTestUtils.getRandomDirConf(conf); attributesManager.init(conf); attributesManager.start(); }
@Override public void write(OutputStream os, NodeAttributesManager mgr) throws IOException { ((NodesToAttributesMappingRequestPBImpl) NodesToAttributesMappingRequest .newInstance(AttributeMappingOperationType.REPLACE, mgr.getNodeToAttributes( ImmutableSet.of(NodeAttribute.PREFIX_CENTRALIZED)), false)) .getProto().writeDelimitedTo(os); }
@Override public GetAttributesToNodesResponse getAttributesToNodes( GetAttributesToNodesRequest request) throws YarnException, IOException { NodeAttributesManager attributesManager = rmContext.getNodeAttributesManager(); Map<NodeAttributeKey, List<NodeToAttributeValue>> attrToNodesWithStrVal = new HashMap<>(); Map<NodeAttributeKey, Map<String, AttributeValue>> attributesToNodes = attributesManager.getAttributesToNodes(request.getNodeAttributes()); for (Map.Entry<NodeAttributeKey, Map<String, AttributeValue>> attrib : attributesToNodes.entrySet()) { Map<String, AttributeValue> nodesToVal = attrib.getValue(); List<NodeToAttributeValue> nodeToAttrValList = new ArrayList<>(); for (Map.Entry<String, AttributeValue> nodeToVal : nodesToVal .entrySet()) { nodeToAttrValList.add(NodeToAttributeValue .newInstance(nodeToVal.getKey(), nodeToVal.getValue().getValue())); } attrToNodesWithStrVal.put(attrib.getKey(), nodeToAttrValList); } GetAttributesToNodesResponse response = GetAttributesToNodesResponse.newInstance(attrToNodesWithStrVal); return response; }
@Override public GetNodesToAttributesResponse getNodesToAttributes( GetNodesToAttributesRequest request) throws YarnException, IOException { NodeAttributesManager attributesManager = rmContext.getNodeAttributesManager(); GetNodesToAttributesResponse response = GetNodesToAttributesResponse .newInstance( attributesManager.getNodesToAttributes(request.getHostNames())); return response; }
.addNodeAttributes(Mockito.anyMap()); .removeNodeAttributes(Mockito.anyMap()); .replaceNodeAttributes(Mockito.eq(NodeAttribute.PREFIX_CENTRALIZED), Mockito.anyMap());
rm.getRMContext().getNodeAttributesManager(); Map<NodeAttribute, AttributeValue> attrs = attributeManager .getAttributesForNode(nodeId.getHost()); Assert.assertEquals(1, attrs.size()); NodeAttribute na = attrs.keySet().iterator().next(); attrs = attributeManager.getAttributesForNode(nodeId.getHost()); Assert.assertEquals(1, attrs.size()); na = attrs.keySet().iterator().next();
nodes.put(node1, ImmutableSet.of(gpu, os, dist)); nodes.put(node2, ImmutableSet.of(docker, dist)); mgr.addNodeAttributes(nodes);
@Override public void recover(InputStream is, NodeAttributesManager mgr) throws IOException { NodesToAttributesMappingRequest request = new NodesToAttributesMappingRequestPBImpl( YarnServerResourceManagerServiceProtos .NodesToAttributesMappingRequestProto .parseDelimitedFrom(is)); //Only CENTRALIZED is stored to FS system mgr.replaceNodeAttributes(NodeAttribute.PREFIX_CENTRALIZED, getNodeToAttributesMap(request)); }
nodes.put(host1.getHost(), ImmutableSet.of(gpu, os)); nodes.put(host2.getHost(), ImmutableSet.of(docker)); mgr.addNodeAttributes(nodes);
.replaceNodeAttributes(NodeAttribute.PREFIX_DISTRIBUTED, ImmutableMap.of(nodeId.getHost(), nodeAttributes));
nodes.put(node1, ImmutableSet.of(gpu, os, dist)); nodes.put(node2, ImmutableSet.of(docker, dist)); mgr.addNodeAttributes(nodes);