@Override protected void serviceInit(Configuration conf) throws Exception { RackResolver.init(conf); super.serviceInit(conf); }
@Override protected void serviceInit(Configuration conf) throws Exception { RackResolver.init(conf); super.serviceInit(conf); }
/** * Utility method for getting a hostname resolved to a node in the * network topology. This method initializes the class with the * right resolver implementation. * @param conf * @param hostName * @return node {@link Node} after resolving the hostname */ public static Node resolve(Configuration conf, String hostName) { init(conf); return coreResolve(hostName); }
/** * Utility method for getting a hostname resolved to a node in the * network topology. This method initializes the class with the * right resolver implementation. * @param conf * @param hostName * @return node {@link Node} after resolving the hostname */ public static Node resolve(Configuration conf, String hostName) { init(conf); return coreResolve(hostName); }
@Override protected void serviceInit(Configuration conf) throws Exception { RackResolver.init(conf); super.serviceInit(conf); }
/** * Utility method for getting a hostname resolved to a node in the * network topology. This method initializes the class with the * right resolver implementation. * @param conf * @param hostName * @return node {@link Node} after resolving the hostname */ public static Node resolve(Configuration conf, String hostName) { init(conf); return coreResolve(hostName); }
@Override protected void serviceInit(Configuration conf) throws Exception { RackResolver.init(conf); super.serviceInit(conf); }
@Override protected void serviceInit(Configuration conf) throws Exception { resourceTrackerAddress = conf.getSocketAddr( YarnConfiguration.RM_BIND_HOST, YarnConfiguration.RM_RESOURCE_TRACKER_ADDRESS, YarnConfiguration.DEFAULT_RM_RESOURCE_TRACKER_ADDRESS, YarnConfiguration.DEFAULT_RM_RESOURCE_TRACKER_PORT); RackResolver.init(conf); nextHeartBeatInterval = conf.getLong(YarnConfiguration.RM_NM_HEARTBEAT_INTERVAL_MS, YarnConfiguration.DEFAULT_RM_NM_HEARTBEAT_INTERVAL_MS); if (nextHeartBeatInterval <= 0) { throw new YarnRuntimeException("Invalid Configuration. " + YarnConfiguration.RM_NM_HEARTBEAT_INTERVAL_MS + " should be larger than 0."); } minAllocMb = conf.getInt( YarnConfiguration.RM_SCHEDULER_MINIMUM_ALLOCATION_MB, YarnConfiguration.DEFAULT_RM_SCHEDULER_MINIMUM_ALLOCATION_MB); minAllocVcores = conf.getInt( YarnConfiguration.RM_SCHEDULER_MINIMUM_ALLOCATION_VCORES, YarnConfiguration.DEFAULT_RM_SCHEDULER_MINIMUM_ALLOCATION_VCORES); minimumNodeManagerVersion = conf.get( YarnConfiguration.RM_NODEMANAGER_MINIMUM_VERSION, YarnConfiguration.DEFAULT_RM_NODEMANAGER_MINIMUM_VERSION); super.serviceInit(conf); }
@Override protected void serviceInit(Configuration conf) throws Exception { resourceTrackerAddress = conf.getSocketAddr( YarnConfiguration.RM_BIND_HOST, YarnConfiguration.RM_RESOURCE_TRACKER_ADDRESS, YarnConfiguration.DEFAULT_RM_RESOURCE_TRACKER_ADDRESS, YarnConfiguration.DEFAULT_RM_RESOURCE_TRACKER_PORT); RackResolver.init(conf); nextHeartBeatInterval = conf.getLong(YarnConfiguration.RM_NM_HEARTBEAT_INTERVAL_MS, YarnConfiguration.DEFAULT_RM_NM_HEARTBEAT_INTERVAL_MS); if (nextHeartBeatInterval <= 0) { throw new YarnRuntimeException("Invalid Configuration. " + YarnConfiguration.RM_NM_HEARTBEAT_INTERVAL_MS + " should be larger than 0."); } minAllocMb = conf.getInt( YarnConfiguration.RM_SCHEDULER_MINIMUM_ALLOCATION_MB, YarnConfiguration.DEFAULT_RM_SCHEDULER_MINIMUM_ALLOCATION_MB); minAllocVcores = conf.getInt( YarnConfiguration.RM_SCHEDULER_MINIMUM_ALLOCATION_VCORES, YarnConfiguration.DEFAULT_RM_SCHEDULER_MINIMUM_ALLOCATION_VCORES); minimumNodeManagerVersion = conf.get( YarnConfiguration.RM_NODEMANAGER_MINIMUM_VERSION, YarnConfiguration.DEFAULT_RM_NODEMANAGER_MINIMUM_VERSION); super.serviceInit(conf); }
maxRunningReduces = conf.getInt(MRJobConfig.JOB_RUNNING_REDUCE_LIMIT, MRJobConfig.DEFAULT_JOB_RUNNING_REDUCE_LIMIT); RackResolver.init(conf); retryInterval = getConfig().getLong(MRJobConfig.MR_AM_TO_RM_WAIT_INTERVAL_MS, MRJobConfig.DEFAULT_MR_AM_TO_RM_WAIT_INTERVAL_MS);
@Override protected void serviceInit(Configuration conf) throws Exception { super.serviceInit(conf); reduceSlowStart = conf.getFloat( MRJobConfig.COMPLETED_MAPS_FOR_REDUCE_SLOWSTART, DEFAULT_COMPLETED_MAPS_PERCENT_FOR_REDUCE_SLOWSTART); maxReduceRampupLimit = conf.getFloat( MRJobConfig.MR_AM_JOB_REDUCE_RAMPUP_UP_LIMIT, MRJobConfig.DEFAULT_MR_AM_JOB_REDUCE_RAMP_UP_LIMIT); maxReducePreemptionLimit = conf.getFloat( MRJobConfig.MR_AM_JOB_REDUCE_PREEMPTION_LIMIT, MRJobConfig.DEFAULT_MR_AM_JOB_REDUCE_PREEMPTION_LIMIT); reducerUnconditionalPreemptionDelayMs = 1000 * conf.getInt( MRJobConfig.MR_JOB_REDUCER_UNCONDITIONAL_PREEMPT_DELAY_SEC, MRJobConfig.DEFAULT_MR_JOB_REDUCER_UNCONDITIONAL_PREEMPT_DELAY_SEC); reducerNoHeadroomPreemptionDelayMs = conf.getInt( MRJobConfig.MR_JOB_REDUCER_PREEMPT_DELAY_SEC, MRJobConfig.DEFAULT_MR_JOB_REDUCER_PREEMPT_DELAY_SEC) * 1000;//sec -> ms maxRunningMaps = conf.getInt(MRJobConfig.JOB_RUNNING_MAP_LIMIT, MRJobConfig.DEFAULT_JOB_RUNNING_MAP_LIMIT); maxRunningReduces = conf.getInt(MRJobConfig.JOB_RUNNING_REDUCE_LIMIT, MRJobConfig.DEFAULT_JOB_RUNNING_REDUCE_LIMIT); RackResolver.init(conf); retryInterval = getConfig().getLong(MRJobConfig.MR_AM_TO_RM_WAIT_INTERVAL_MS, MRJobConfig.DEFAULT_MR_AM_TO_RM_WAIT_INTERVAL_MS); // Init startTime to current time. If all goes well, it will be reset after // first attempt to contact RM. retrystartTime = System.currentTimeMillis(); }
maxRunningReduces = conf.getInt(MRJobConfig.JOB_RUNNING_REDUCE_LIMIT, MRJobConfig.DEFAULT_JOB_RUNNING_REDUCE_LIMIT); RackResolver.init(conf); retryInterval = getConfig().getLong(MRJobConfig.MR_AM_TO_RM_WAIT_INTERVAL_MS, MRJobConfig.DEFAULT_MR_AM_TO_RM_WAIT_INTERVAL_MS);
YarnConfiguration.DEFAULT_RM_RESOURCE_TRACKER_PORT); RackResolver.init(conf); nextHeartBeatInterval = conf.getLong(YarnConfiguration.RM_NM_HEARTBEAT_INTERVAL_MS,
@Test public void testScriptName() { Configuration conf = new Configuration(); conf .setClass( CommonConfigurationKeysPublic. NET_TOPOLOGY_NODE_SWITCH_MAPPING_IMPL_KEY, ScriptBasedMapping.class, DNSToSwitchMapping.class); conf.set(CommonConfigurationKeysPublic.NET_TOPOLOGY_SCRIPT_FILE_NAME_KEY, "testScript"); RackResolver.init(conf); Assert.assertEquals(RackResolver.getDnsToSwitchMapping().toString(), "script-based mapping with script testScript"); } }
@Test public void testScriptName() { Configuration conf = new Configuration(); conf .setClass( CommonConfigurationKeysPublic. NET_TOPOLOGY_NODE_SWITCH_MAPPING_IMPL_KEY, ScriptBasedMapping.class, DNSToSwitchMapping.class); conf.set(CommonConfigurationKeysPublic.NET_TOPOLOGY_SCRIPT_FILE_NAME_KEY, "testScript"); RackResolver.init(conf); Assert.assertEquals(RackResolver.getDnsToSwitchMapping().toString(), "script-based mapping with script testScript"); } }
@Test public void testScriptName() { Configuration conf = new Configuration(); conf .setClass( CommonConfigurationKeysPublic. NET_TOPOLOGY_NODE_SWITCH_MAPPING_IMPL_KEY, ScriptBasedMapping.class, DNSToSwitchMapping.class); conf.set(CommonConfigurationKeysPublic.NET_TOPOLOGY_SCRIPT_FILE_NAME_KEY, "testScript"); RackResolver.init(conf); Assert.assertEquals(RackResolver.getDnsToSwitchMapping().toString(), "script-based mapping with script testScript"); } }
RackResolver.init(conf); this.dataLocalRacks = new HashSet<String>(); for (String host : this.dataLocalHosts) {
@Test public void testCaching() { Configuration conf = new Configuration(); conf.setClass( CommonConfigurationKeysPublic.NET_TOPOLOGY_NODE_SWITCH_MAPPING_IMPL_KEY, MyResolver.class, DNSToSwitchMapping.class); RackResolver.init(conf); try { InetAddress iaddr = InetAddress.getByName("host1"); MyResolver.resolvedHost1 = iaddr.getHostAddress(); } catch (UnknownHostException e) { // Ignore if not found } Node node = RackResolver.resolve("host1"); Assert.assertEquals("/rack1", node.getNetworkLocation()); node = RackResolver.resolve("host1"); Assert.assertEquals("/rack1", node.getNetworkLocation()); node = RackResolver.resolve(invalidHost); Assert.assertEquals(NetworkTopology.DEFAULT_RACK, node.getNetworkLocation()); }
@Test public void testCaching() { Configuration conf = new Configuration(); conf.setClass( CommonConfigurationKeysPublic.NET_TOPOLOGY_NODE_SWITCH_MAPPING_IMPL_KEY, MyResolver.class, DNSToSwitchMapping.class); RackResolver.init(conf); try { InetAddress iaddr = InetAddress.getByName("host1"); MyResolver.resolvedHost1 = iaddr.getHostAddress(); } catch (UnknownHostException e) { // Ignore if not found } Node node = RackResolver.resolve("host1"); Assert.assertEquals("/rack1", node.getNetworkLocation()); node = RackResolver.resolve("host1"); Assert.assertEquals("/rack1", node.getNetworkLocation()); node = RackResolver.resolve(invalidHost); Assert.assertEquals(NetworkTopology.DEFAULT_RACK, node.getNetworkLocation()); }
@Test public void testCaching() { Configuration conf = new Configuration(); conf.setClass( CommonConfigurationKeysPublic.NET_TOPOLOGY_NODE_SWITCH_MAPPING_IMPL_KEY, MyResolver.class, DNSToSwitchMapping.class); RackResolver.init(conf); try { InetAddress iaddr = InetAddress.getByName("host1"); MyResolver.resolvedHost1 = iaddr.getHostAddress(); } catch (UnknownHostException e) { // Ignore if not found } Node node = RackResolver.resolve("host1"); Assert.assertEquals("/rack1", node.getNetworkLocation()); node = RackResolver.resolve("host1"); Assert.assertEquals("/rack1", node.getNetworkLocation()); node = RackResolver.resolve(invalidHost); Assert.assertEquals(NetworkTopology.DEFAULT_RACK, node.getNetworkLocation()); }