protected ResourceTrackerService createResourceTrackerService() { return new ResourceTrackerService(this.rmContext, this.nodesListManager, this.nmLivelinessMonitor, this.rmContext.getContainerTokenSecretManager(), this.rmContext.getNMTokenSecretManager()); }
!isNodeInDecommissioning(nodeId)) { String message = "Disallowed NodeManager from " + host Resource dynamicLoadCapability = loadNodeResourceFromDRConfiguration(nid); if (dynamicLoadCapability != null) { if (LOG.isDebugEnabled()) { resolve(host), capability, nodeManagerVersion, physicalResource); + request.getNMContainerStatuses()); for (NMContainerStatus status : request.getNMContainerStatuses()) { handleNMContainerStatus(status, nodeId); if (isDistributedNodeLabelsConf && nodeLabels != null) { try { updateNodeLabelsFromNMReport(nodeLabels, nodeId); response.setAreNodeLabelsAcceptedByRM(true); } catch (IOException ex) {
&& !isNodeInDecommissioning(nodeId)) { String message = "Disallowed NodeManager nodeId: " + nodeId + " hostname: " if (getNextResponseId( remoteNodeStatus.getResponseId()) == lastNodeHeartbeatResponse .getResponseId()) { YarnConfiguration.timelineServiceV2Enabled(getConfig()); if (timelineV2Enabled) { updateAppCollectorsMap(request); getNextResponseId(lastNodeHeartbeatResponse.getResponseId()), NodeAction.NORMAL, null, null, null, null, nextHeartBeatInterval); rmNode.setAndUpdateNodeHeartbeatResponse(nodeHeartBeatResponse); populateKeys(request, nodeHeartBeatResponse); setAppCollectorsMapToResponse(rmNode.getRunningApps(), nodeHeartBeatResponse); updateNodeLabelsFromNMReport( NodeLabelsUtils.convertToStringSet(request.getNodeLabels()), nodeId); Resource capability = loadNodeResourceFromDRConfiguration(nid);
@Override protected void serviceStart() throws Exception { super.serviceStart(); // ResourceTrackerServer authenticates NodeManager via Kerberos if // security is enabled, so no secretManager. Configuration conf = getConfig(); YarnRPC rpc = YarnRPC.create(conf); this.server = rpc.getServer( ResourceTracker.class, this, resourceTrackerAddress, conf, null, conf.getInt(YarnConfiguration.RM_RESOURCE_TRACKER_CLIENT_THREAD_COUNT, YarnConfiguration.DEFAULT_RM_RESOURCE_TRACKER_CLIENT_THREAD_COUNT)); // Enable service authorization? if (conf.getBoolean( CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHORIZATION, false)) { InputStream inputStream = this.rmContext.getConfigurationProvider() .getConfigurationInputStream(conf, YarnConfiguration.HADOOP_POLICY_CONFIGURATION_FILE); if (inputStream != null) { conf.addResource(inputStream); } refreshServiceAcls(conf, RMPolicyProvider.getInstance()); } this.server.start(); conf.updateConnectAddr(YarnConfiguration.RM_BIND_HOST, YarnConfiguration.RM_RESOURCE_TRACKER_ADDRESS, YarnConfiguration.DEFAULT_RM_RESOURCE_TRACKER_ADDRESS, server.getListenerAddress()); }
@Before public void setUp() { Configuration conf = new Configuration(); // Dispatcher that processes events inline Dispatcher dispatcher = new InlineDispatcher(); RMContext context = new RMContextImpl(dispatcher, null, null, null, null, null, null, null, null, null); dispatcher.register(SchedulerEventType.class, new InlineDispatcher.EmptyEventHandler()); dispatcher.register(RMNodeEventType.class, new NodeEventDispatcher(context)); NMLivelinessMonitor nmLivelinessMonitor = new TestNmLivelinessMonitor( dispatcher); nmLivelinessMonitor.init(conf); nmLivelinessMonitor.start(); NodesListManager nodesListManager = new NodesListManager(context); nodesListManager.init(conf); RMContainerTokenSecretManager containerTokenSecretManager = new RMContainerTokenSecretManager(conf); containerTokenSecretManager.start(); NMTokenSecretManagerInRM nmTokenSecretManager = new NMTokenSecretManagerInRM(conf); nmTokenSecretManager.start(); resourceTrackerService = new ResourceTrackerService(context, nodesListManager, nmLivelinessMonitor, containerTokenSecretManager, nmTokenSecretManager); resourceTrackerService.init(conf); resourceTrackerService.start(); }
public void init(String nodeIdStr, int memory, int cores, int dispatchTime, int heartBeatInterval, ResourceManager rm) throws IOException, YarnException { super.init(dispatchTime, dispatchTime + 1000000L * heartBeatInterval, heartBeatInterval); // create resource String rackHostName[] = SLSUtils.getRackHostName(nodeIdStr); this.node = NodeInfo.newNodeInfo(rackHostName[0], rackHostName[1], BuilderUtils.newResource(memory, cores)); this.rm = rm; // init data structures completedContainerList = Collections.synchronizedList(new ArrayList<ContainerId>()); releasedContainerList = Collections.synchronizedList(new ArrayList<ContainerId>()); containerQueue = new DelayQueue<ContainerSimulator>(); amContainerList = Collections.synchronizedList(new ArrayList<ContainerId>()); runningContainers = new ConcurrentHashMap<ContainerId, ContainerSimulator>(); // register NM with RM RegisterNodeManagerRequest req = Records.newRecord(RegisterNodeManagerRequest.class); req.setNodeId(node.getNodeID()); req.setResource(node.getTotalCapability()); req.setHttpPort(80); RegisterNodeManagerResponse response = rm.getResourceTrackerService() .registerNodeManager(req); masterKey = response.getNMTokenMasterKey(); }
registerReq.setNMVersion(YarnVersionInfo.getVersion()); RegisterNodeManagerResponse registerResponse = resourceTrackerService.registerNodeManager(registerReq); .getContainerTokenMasterKey()); heartbeatReq.setNodeAttributes(nodeAttributes); resourceTrackerService.nodeHeartbeat(heartbeatReq); heartbeatReq.setNodeStatus(nodeStatusObject); heartbeatReq.setNodeAttributes(nodeAttributes); resourceTrackerService.nodeHeartbeat(heartbeatReq);
resolve(host), capability, nodeManagerVersion); + request.getNMContainerStatuses()); for (NMContainerStatus status : request.getNMContainerStatuses()) { handleNMContainerStatus(status, nodeId);
@Before public void setUp() { Configuration conf = new Configuration(); // Dispatcher that processes events inline Dispatcher dispatcher = new InlineDispatcher(); dispatcher.register(SchedulerEventType.class, new EventHandler<Event>() { @Override public void handle(Event event) { ; // ignore } }); RMContext context = new RMContextImpl(dispatcher, null, null, null, null, null, new RMContainerTokenSecretManager(conf), new NMTokenSecretManagerInRM(conf), null, null); dispatcher.register(RMNodeEventType.class, new ResourceManager.NodeEventDispatcher(context)); NodesListManager nodesListManager = new NodesListManager(context); nodesListManager.init(conf); context.getContainerTokenSecretManager().rollMasterKey(); context.getNMTokenSecretManager().rollMasterKey(); resourceTrackerService = new ResourceTrackerService(context, nodesListManager, new NMLivelinessMonitor(dispatcher), context.getContainerTokenSecretManager(), context.getNMTokenSecretManager()); resourceTrackerService.init(conf); }
beatRequest.setNodeStatus(ns); NodeHeartbeatResponse beatResponse = rm.getResourceTrackerService().nodeHeartbeat(beatRequest); if (! beatResponse.getContainersToCleanup().isEmpty()) {
request1.setHttpPort(0); request1.setResource(capability); privateResourceTrackerService.registerNodeManager(request1); privateDispatcher.await(); Resource clusterResource = privateResourceTrackerService.registerNodeManager(request2); privateDispatcher.await(); Assert.assertEquals("Cluster resources don't match", newCapability, rm.getResourceScheduler().getClusterResource()); privateResourceTrackerService.stop(); } finally { rm.stop();
ContainerState.COMPLETE, Resource.newInstance(1024, 1), "Dummy Completed", 0, Priority.newInstance(10), 1234); rm.getResourceTrackerService().handleNMContainerStatus(report, null); verify(handler, never()).handle((Event) any()); ContainerState.COMPLETE, Resource.newInstance(1024, 1), "Dummy Completed", 0, Priority.newInstance(10), 1234); rm.getResourceTrackerService().handleNMContainerStatus(report, null); verify(handler, never()).handle((Event)any()); "Dummy Completed", 0, Priority.newInstance(10), 1234); try { rm.getResourceTrackerService().handleNMContainerStatus(report, null); } catch (Exception e) { "Dummy Completed", 0, Priority.newInstance(10), 1234); try { rm.getResourceTrackerService().handleNMContainerStatus(report, null); } catch (Exception e) {
private void refreshActiveServicesAcls() throws IOException, YarnException { PolicyProvider policyProvider = RMPolicyProvider.getInstance(); Configuration conf = getConfiguration(new Configuration(false), YarnConfiguration.HADOOP_POLICY_CONFIGURATION_FILE); rmContext.getClientRMService().refreshServiceAcls(conf, policyProvider); rmContext.getApplicationMasterService().refreshServiceAcls( conf, policyProvider); rmContext.getResourceTrackerService().refreshServiceAcls( conf, policyProvider); }
rmNode.updateNodeHeartbeatResponseForCleanup(nodeHeartBeatResponse); populateKeys(request, nodeHeartBeatResponse);
.getServer().getServiceAuthorizationManager(); verifyServiceACLsRefresh(RTService, org.apache.hadoop.yarn.api.ApplicationClientProtocolPB.class,
@Before public void setUp() { Configuration conf = new Configuration(); // Dispatcher that processes events inline Dispatcher dispatcher = new InlineDispatcher(); RMContext context = new RMContextImpl(dispatcher, null, null, null, null, null, null, null, null, null); dispatcher.register(SchedulerEventType.class, new InlineDispatcher.EmptyEventHandler()); dispatcher.register(RMNodeEventType.class, new NodeEventDispatcher(context)); NMLivelinessMonitor nmLivelinessMonitor = new TestNmLivelinessMonitor( dispatcher); nmLivelinessMonitor.init(conf); nmLivelinessMonitor.start(); NodesListManager nodesListManager = new NodesListManager(context); nodesListManager.init(conf); RMContainerTokenSecretManager containerTokenSecretManager = new RMContainerTokenSecretManager(conf); containerTokenSecretManager.start(); NMTokenSecretManagerInRM nmTokenSecretManager = new NMTokenSecretManagerInRM(conf); nmTokenSecretManager.start(); resourceTrackerService = new ResourceTrackerService(context, nodesListManager, nmLivelinessMonitor, containerTokenSecretManager, nmTokenSecretManager); resourceTrackerService.init(conf); resourceTrackerService.start(); }
@Test public void testReconnect() throws Exception { String hostname1 = "localhost1"; Resource capability = BuilderUtils.newResource(1024, 1); RegisterNodeManagerRequest request1 = recordFactory .newRecordInstance(RegisterNodeManagerRequest.class); NodeId nodeId1 = NodeId.newInstance(hostname1, 0); request1.setNodeId(nodeId1); request1.setHttpPort(0); request1.setResource(capability); resourceTrackerService.registerNodeManager(request1); Assert.assertEquals(RMNodeEventType.STARTED, rmNodeEvents.get(0).getType()); rmNodeEvents.clear(); resourceTrackerService.registerNodeManager(request1); Assert.assertEquals(RMNodeEventType.RECONNECTED, rmNodeEvents.get(0).getType()); rmNodeEvents.clear(); resourceTrackerService.registerNodeManager(request1); capability = BuilderUtils.newResource(1024, 2); request1.setResource(capability); Assert.assertEquals(RMNodeEventType.RECONNECTED, rmNodeEvents.get(0).getType()); }
request1.setHttpPort(0); request1.setResource(capability); resourceTrackerService.registerNodeManager(request1); NodeHeartbeatResponse response = resourceTrackerService.nodeHeartbeat( nodeHeartBeatRequest); Assert.assertTrue(response.getResponseId() == 1); response = resourceTrackerService.nodeHeartbeat(nodeHeartBeatRequest); Assert.assertTrue(response.getResponseId() == 2); response = resourceTrackerService.nodeHeartbeat(nodeHeartBeatRequest); Assert.assertTrue(response.getResponseId() == 2); response = resourceTrackerService.nodeHeartbeat(nodeHeartBeatRequest); Assert.assertTrue(NodeAction.RESYNC.equals(response.getNodeAction())); Assert.assertEquals("Too far behind rm response id:2 nm response id:0",
resolve(host), capability, nodeManagerVersion); + request.getNMContainerStatuses()); for (NMContainerStatus status : request.getNMContainerStatuses()) { handleNMContainerStatus(status, nodeId);
Configuration conf = getConfig(); YarnRPC rpc = YarnRPC.create(conf); this.server = conf.addResource(inputStream); refreshServiceAcls(conf, RMPolicyProvider.getInstance());