/** * resolving the network topology. * @param hostName the hostname of this node. * @return the resolved {@link Node} for this nodemanager. */ public static Node resolve(String hostName) { return RackResolver.resolve(hostName); }
/** * resolving the network topology. * @param hostName the hostname of this node. * @return the resolved {@link Node} for this nodemanager. */ public static Node resolve(String hostName) { return RackResolver.resolve(hostName); }
/** * resolving the network topology. * @param hostName the hostname of this node. * @return the resolved {@link Node} for this nodemanager. */ public static Node resolve(String hostName) { return RackResolver.resolve(hostName); }
private Set<String> resolveRacks(List<String> nodes) { Set<String> racks = new HashSet<String>(); if (nodes != null) { for (String node : nodes) { // Ensure node requests are accompanied by requests for // corresponding rack String rack = RackResolver.resolve(node).getNetworkLocation(); if (rack == null) { LOG.warn("Failed to resolve rack for node " + node + "."); } else { racks.add(rack); } } } return racks; }
private Set<String> resolveRacks(List<String> nodes) { Set<String> racks = new HashSet<String>(); if (nodes != null) { for (String node : nodes) { // Ensure node requests are accompanied by requests for // corresponding rack String rack = RackResolver.resolve(node).getNetworkLocation(); if (rack == null) { LOG.warn("Failed to resolve rack for node " + node + "."); } else { racks.add(rack); } } } return racks; }
private Set<String> resolveRacks(List<String> nodes) { Set<String> racks = new HashSet<String>(); if (nodes != null) { for (String node : nodes) { // Ensure node requests are accompanied by requests for // corresponding rack String rack = RackResolver.resolve(node).getNetworkLocation(); if (rack == null) { LOG.warn("Failed to resolve rack for node " + node + "."); } else { racks.add(rack); } } } return racks; }
private Set<String> resolveRacks(List<String> nodes) { Set<String> racks = new HashSet<String>(); if (nodes != null) { for (String node : nodes) { // Ensure node requests are accompanied by requests for // corresponding rack String rack = RackResolver.resolve(node).getNetworkLocation(); if (rack == null) { LOG.warn("Failed to resolve rack for node " + node + "."); } else { racks.add(rack); } } } return racks; }
/** * Resolve rack for host. * * @param configuration the hadoop configuration * @param node the node * @return the resolved rack, null if failure */ private static String resolveRack(Configuration configuration, String node) { try { if (node != null) { String rack = RackResolver.resolve(configuration, node).getNetworkLocation(); if (rack == null) { log.warn("Failed to resolve rack for node " + node + "."); return null; } else { return rack; } } } catch (Exception e) { log.warn("Failure in RackResolver", e); } return null; }
private void computeRackAndLocality() { NodeId containerNodeId = container.getNodeId(); nodeRackName = RackResolver.resolve( containerNodeId.getHost()).getNetworkLocation(); locality = Locality.OFF_SWITCH; if (dataLocalHosts.size() > 0) { String cHost = resolveHost(containerNodeId.getHost()); if (dataLocalHosts.contains(cHost)) { locality = Locality.NODE_LOCAL; } } if (locality == Locality.OFF_SWITCH) { if (dataLocalRacks.contains(nodeRackName)) { locality = Locality.RACK_LOCAL; } } }
private void computeRackAndLocality() { NodeId containerNodeId = container.getNodeId(); nodeRackName = RackResolver.resolve( containerNodeId.getHost()).getNetworkLocation(); locality = Locality.OFF_SWITCH; if (dataLocalHosts.size() > 0) { String cHost = resolveHost(containerNodeId.getHost()); if (dataLocalHosts.contains(cHost)) { locality = Locality.NODE_LOCAL; } } if (locality == Locality.OFF_SWITCH) { if (dataLocalRacks.contains(nodeRackName)) { locality = Locality.RACK_LOCAL; } } }
private void computeRackAndLocality() { NodeId containerNodeId = container.getNodeId(); nodeRackName = RackResolver.resolve( containerNodeId.getHost()).getNetworkLocation(); locality = Locality.OFF_SWITCH; if (dataLocalHosts.size() > 0) { String cHost = resolveHost(containerNodeId.getHost()); if (dataLocalHosts.contains(cHost)) { locality = Locality.NODE_LOCAL; } } if (locality == Locality.OFF_SWITCH) { if (dataLocalRacks.contains(nodeRackName)) { locality = Locality.RACK_LOCAL; } } }
private String resolveRack(GridMember member) { if (getConfiguration() != null) { String host = member.getContainer().getNodeId().getHost(); String rack = RackResolver.resolve(getConfiguration(), host).getNetworkLocation(); if (rack == null) { log.warn("Failed to resolve rack for node " + host + "."); } else { log.info("Resolve rack for node " + host + " into " + rack); } return rack; } else { return null; } }
@Override public CookieContainerRequest assignNewContainer(Container container) { String location = RackResolver.resolve(container.getNodeId().getHost()) .getNetworkLocation(); CookieContainerRequest assigned = getMatchingRequestWithPriority(container, location); doBookKeepingForAssignedContainer(assigned, container, location, false); return assigned; }
private void addLeafTask(TaskAttemptToSchedulerEvent event) { TaskAttempt taskAttempt = event.getTaskAttempt(); List<DataLocation> locations = taskAttempt.getTask().getDataLocations(); for (DataLocation location : locations) { String host = location.getHost(); leafTaskHosts.add(host); HostVolumeMapping hostVolumeMapping = leafTaskHostMapping.get(host); if (hostVolumeMapping == null) { String rack = RackResolver.resolve(host).getNetworkLocation(); hostVolumeMapping = new HostVolumeMapping(host, rack); leafTaskHostMapping.put(host, hostVolumeMapping); } hostVolumeMapping.addTaskAttempt(location.getVolumeId(), taskAttempt); if (LOG.isDebugEnabled()) { LOG.debug("Added attempt req to host " + host); } HashSet<TaskAttemptId> list = leafTasksRackMapping.get(hostVolumeMapping.getRack()); if (list == null) { list = new HashSet<TaskAttemptId>(); leafTasksRackMapping.put(hostVolumeMapping.getRack(), list); } list.add(taskAttempt.getId()); if (LOG.isDebugEnabled()) { LOG.debug("Added attempt req to rack " + hostVolumeMapping.getRack()); } } leafTasks.add(taskAttempt.getId()); }
private boolean tryRackAccept(GridMember member) { if (getConfiguration() != null) { String host = member.getContainer().getNodeId().getHost(); String rack = RackResolver.resolve(getConfiguration(), host).getNetworkLocation(); if (rack == null) { log.warn("Failed to resolve rack for node " + host + "."); } else { log.info("Resolve rack for node " + host + " into " + rack); Integer target = getProjectionData().getRacks().get(rack); Collection<GridMember> rackCountMembers = getRackCountMembers(rack); if (rackCountMembers != null && target != null && rackCountMembers.size() < target) { if (addRackMember(member)) { return true; } } } } else { log.warn("Failed to resolve rack for node - no configuration"); } return false; }
HeldContainer(Container container, long nextScheduleTime, long containerExpiryTime, CookieContainerRequest firstTaskInfo, ContainerSignatureMatcher signatureMatcher) { this.container = container; this.nextScheduleTime = nextScheduleTime; if (firstTaskInfo != null) { this.lastTaskInfo = firstTaskInfo; this.lastAssignedContainerSignature = firstTaskInfo.getCookie().getContainerSignature(); } this.localityMatchLevel = LocalityMatchLevel.NODE; this.containerExpiryTime = containerExpiryTime; this.rack = RackResolver.resolve(container.getNodeId().getHost()) .getNetworkLocation(); this.signatureMatcher = signatureMatcher; }
@Test public void testCaching() { Configuration conf = new Configuration(); conf.setClass( CommonConfigurationKeysPublic.NET_TOPOLOGY_NODE_SWITCH_MAPPING_IMPL_KEY, MyResolver.class, DNSToSwitchMapping.class); RackResolver.init(conf); try { InetAddress iaddr = InetAddress.getByName("host1"); MyResolver.resolvedHost1 = iaddr.getHostAddress(); } catch (UnknownHostException e) { // Ignore if not found } Node node = RackResolver.resolve("host1"); Assert.assertEquals("/rack1", node.getNetworkLocation()); node = RackResolver.resolve("host1"); Assert.assertEquals("/rack1", node.getNetworkLocation()); node = RackResolver.resolve(invalidHost); Assert.assertEquals(NetworkTopology.DEFAULT_RACK, node.getNetworkLocation()); }
@Test public void testCaching() { Configuration conf = new Configuration(); conf.setClass( CommonConfigurationKeysPublic.NET_TOPOLOGY_NODE_SWITCH_MAPPING_IMPL_KEY, MyResolver.class, DNSToSwitchMapping.class); RackResolver.init(conf); try { InetAddress iaddr = InetAddress.getByName("host1"); MyResolver.resolvedHost1 = iaddr.getHostAddress(); } catch (UnknownHostException e) { // Ignore if not found } Node node = RackResolver.resolve("host1"); Assert.assertEquals("/rack1", node.getNetworkLocation()); node = RackResolver.resolve("host1"); Assert.assertEquals("/rack1", node.getNetworkLocation()); node = RackResolver.resolve(invalidHost); Assert.assertEquals(NetworkTopology.DEFAULT_RACK, node.getNetworkLocation()); }
@Test public void testCaching() { Configuration conf = new Configuration(); conf.setClass( CommonConfigurationKeysPublic.NET_TOPOLOGY_NODE_SWITCH_MAPPING_IMPL_KEY, MyResolver.class, DNSToSwitchMapping.class); RackResolver.init(conf); try { InetAddress iaddr = InetAddress.getByName("host1"); MyResolver.resolvedHost1 = iaddr.getHostAddress(); } catch (UnknownHostException e) { // Ignore if not found } Node node = RackResolver.resolve("host1"); Assert.assertEquals("/rack1", node.getNetworkLocation()); node = RackResolver.resolve("host1"); Assert.assertEquals("/rack1", node.getNetworkLocation()); node = RackResolver.resolve(invalidHost); Assert.assertEquals(NetworkTopology.DEFAULT_RACK, node.getNetworkLocation()); }
private RMNodeImpl getNewRMNode(String host, int port, int memory) { NodeId nodeId = NodeId.newInstance(host, port); RMNodeImpl nodeImpl = new RMNodeImpl(nodeId, rm.getRMContext(), nodeId.getHost(), nodeId.getPort(), nodeId.getPort() + 1, RackResolver.resolve(nodeId.getHost()), Resource.newInstance(memory, 4), YarnVersionInfo.getVersion()); rm.getRMContext().getRMNodes().put(nodeId, nodeImpl); return nodeImpl; }