@Override protected void serviceStart() throws Exception { // NodeId is only available during start, the following cannot be moved // anywhere else. this.nodeId = this.context.getNodeId(); super.serviceStart(); }
@Override protected void serviceStart() throws Exception { // NodeId is only available during start, the following cannot be moved // anywhere else. this.nodeId = this.context.getNodeId(); super.serviceStart(); }
@Override protected void serviceStart() throws Exception { // NodeId is only available during start, the following cannot be moved // anywhere else. this.nodeId = this.context.getNodeId(); super.serviceStart(); }
@Override protected void serviceStart() throws Exception { // NodeId is only available during start, the following cannot be moved // anywhere else. this.nodeId = this.context.getNodeId(); super.serviceStart(); }
@Override protected void serviceStart() throws Exception { super.serviceStart(); // context will be updated after containerManagerImpl is started // hence NMMetricsPublisher is added subservice of containerManagerImpl this.nodeId = context.getNodeId(); this.httpAddress = nodeId.getHost() + ":" + httpPort; }
protected ContainerTokenIdentifier verifyAndGetContainerTokenIdentifier( org.apache.hadoop.yarn.api.records.Token token, ContainerTokenIdentifier containerTokenIdentifier) throws YarnException, InvalidToken { byte[] password = context.getContainerTokenSecretManager().retrievePassword( containerTokenIdentifier); byte[] tokenPass = token.getPassword().array(); if (password == null || tokenPass == null || !Arrays.equals(password, tokenPass)) { throw new InvalidToken( "Invalid container token used for starting container on : " + context.getNodeId().toString()); } return containerTokenIdentifier; }
protected ContainerTokenIdentifier verifyAndGetContainerTokenIdentifier( org.apache.hadoop.yarn.api.records.Token token, ContainerTokenIdentifier containerTokenIdentifier) throws YarnException, InvalidToken { byte[] password = context.getContainerTokenSecretManager().retrievePassword( containerTokenIdentifier); byte[] tokenPass = token.getPassword().array(); if (password == null || tokenPass == null || !Arrays.equals(password, tokenPass)) { throw new InvalidToken( "Invalid container token used for starting container on : " + context.getNodeId().toString()); } return containerTokenIdentifier; }
protected ContainerTokenIdentifier verifyAndGetContainerTokenIdentifier( org.apache.hadoop.yarn.api.records.Token token, ContainerTokenIdentifier containerTokenIdentifier) throws YarnException, InvalidToken { byte[] password = context.getContainerTokenSecretManager().retrievePassword( containerTokenIdentifier); byte[] tokenPass = token.getPassword().array(); if (password == null || tokenPass == null || !Arrays.equals(password, tokenPass)) { throw new InvalidToken( "Invalid container token used for starting container on : " + context.getNodeId().toString()); } return containerTokenIdentifier; }
protected ContainerTokenIdentifier verifyAndGetContainerTokenIdentifier( org.apache.hadoop.yarn.api.records.Token token, ContainerTokenIdentifier containerTokenIdentifier) throws YarnException, InvalidToken { byte[] password = context.getContainerTokenSecretManager().retrievePassword( containerTokenIdentifier); byte[] tokenPass = token.getPassword().array(); if (password == null || tokenPass == null || !Arrays.equals(password, tokenPass)) { throw new InvalidToken( "Invalid container token used for starting container on : " + context.getNodeId().toString()); } return containerTokenIdentifier; }
public NMContainerLogsInfo(final Context nmContext, final ContainerId containerId, String remoteUser, ContainerLogAggregationType logType) throws YarnException { this.logType = logType.toString(); this.containerId = containerId.toString(); this.nodeId = nmContext.getNodeId().toString(); this.containerLogsInfo = getContainerLogsInfo( containerId, remoteUser, nmContext); }
@Override protected void serviceStart() throws Exception { // NodeManager is the last service to start, so NodeId is available. this.nodeId = this.context.getNodeId(); LOG.info("Node ID assigned is : " + this.nodeId); this.httpPort = this.context.getHttpPort(); this.nodeManagerVersionId = YarnVersionInfo.getVersion(); try { // Registration has to be in start so that ContainerManager can get the // perNM tokens needed to authenticate ContainerTokens. this.resourceTracker = getRMClient(); registerWithRM(); super.serviceStart(); startStatusUpdater(); } catch (Exception e) { String errorMessage = "Unexpected error starting NodeStatusUpdater"; LOG.error(errorMessage, e); throw new YarnRuntimeException(e); } }
@Override protected void serviceStart() throws Exception { // NodeManager is the last service to start, so NodeId is available. this.nodeId = this.context.getNodeId(); LOG.info("Node ID assigned is : " + this.nodeId); this.httpPort = this.context.getHttpPort(); this.nodeManagerVersionId = YarnVersionInfo.getVersion(); try { // Registration has to be in start so that ContainerManager can get the // perNM tokens needed to authenticate ContainerTokens. this.resourceTracker = getRMClient(); registerWithRM(); super.serviceStart(); startStatusUpdater(); } catch (Exception e) { String errorMessage = "Unexpected error starting NodeStatusUpdater"; LOG.error(errorMessage, e); throw new YarnRuntimeException(e); } }
@Override protected void serviceStart() throws Exception { // NodeManager is the last service to start, so NodeId is available. this.nodeId = this.context.getNodeId(); this.httpPort = this.context.getHttpPort(); this.nodeManagerVersionId = YarnVersionInfo.getVersion(); try { // Registration has to be in start so that ContainerManager can get the // perNM tokens needed to authenticate ContainerTokens. this.resourceTracker = getRMClient(); registerWithRM(); super.serviceStart(); startStatusUpdater(); } catch (Exception e) { String errorMessage = "Unexpected error starting NodeStatusUpdater"; LOG.error(errorMessage, e); throw new YarnRuntimeException(e); } }
@Override protected void serviceStart() throws Exception { // NodeManager is the last service to start, so NodeId is available. this.nodeId = this.context.getNodeId(); this.httpPort = this.context.getHttpPort(); this.nodeManagerVersionId = YarnVersionInfo.getVersion(); try { // Registration has to be in start so that ContainerManager can get the // perNM tokens needed to authenticate ContainerTokens. this.resourceTracker = getRMClient(); registerWithRM(); super.serviceStart(); startStatusUpdater(); } catch (Exception e) { String errorMessage = "Unexpected error starting NodeStatusUpdater"; LOG.error(errorMessage, e); throw new YarnRuntimeException(e); } }
@Before public void setup() { conf = new YarnConfiguration(); conf.set(YarnConfiguration.RM_WEBAPP_ADDRESS, "localhost:0"); conf.setInt(YarnConfiguration.RM_NM_HEARTBEAT_INTERVAL_MS, 100); String name = TestMiniYarnClusterNodeUtilization.class.getName(); cluster = new MiniYARNCluster(name, NUM_RM, NUM_NM, 1, 1); cluster.init(conf); cluster.start(); assertFalse("RM never turned active", -1 == cluster.getActiveRMIndex()); nm = (CustomNodeManager)cluster.getNodeManager(0); nodeStatus = createNodeStatus(nm.getNMContext().getNodeId(), 0, CONTAINER_PMEM_1, CONTAINER_VMEM_1, CONTAINER_CPU_1, NODE_PMEM_1, NODE_VMEM_1, NODE_CPU_1); nm.setNodeStatus(nodeStatus); }
public NodeInfo(final Context context, final ResourceView resourceView) { this.id = context.getNodeId().toString(); this.nodeHostName = context.getNodeId().getHost(); this.totalVmemAllocatedContainersMB = resourceView .getVmemAllocatedForContainers() / BYTES_IN_MB; this.vmemCheckEnabled = resourceView.isVmemCheckEnabled(); this.totalPmemAllocatedContainersMB = resourceView .getPmemAllocatedForContainers() / BYTES_IN_MB; this.pmemCheckEnabled = resourceView.isPmemCheckEnabled(); this.totalVCoresAllocatedContainers = resourceView .getVCoresAllocatedForContainers(); this.nodeHealthy = context.getNodeHealthStatus().getIsNodeHealthy(); this.lastNodeUpdateTime = context.getNodeHealthStatus() .getLastHealthReportTime(); this.healthReport = context.getNodeHealthStatus().getHealthReport(); this.nodeManagerVersion = YarnVersionInfo.getVersion(); this.nodeManagerBuildVersion = YarnVersionInfo.getBuildVersion(); this.nodeManagerVersionBuiltOn = YarnVersionInfo.getDate(); this.hadoopVersion = VersionInfo.getVersion(); this.hadoopBuildVersion = VersionInfo.getBuildVersion(); this.hadoopVersionBuiltOn = VersionInfo.getDate(); }
public NodeInfo(final Context context, final ResourceView resourceView) { this.id = context.getNodeId().toString(); this.nodeHostName = context.getNodeId().getHost(); this.totalVmemAllocatedContainersMB = resourceView .getVmemAllocatedForContainers() / BYTES_IN_MB; this.vmemCheckEnabled = resourceView.isVmemCheckEnabled(); this.totalPmemAllocatedContainersMB = resourceView .getPmemAllocatedForContainers() / BYTES_IN_MB; this.pmemCheckEnabled = resourceView.isPmemCheckEnabled(); this.totalVCoresAllocatedContainers = resourceView .getVCoresAllocatedForContainers(); this.nodeHealthy = context.getNodeHealthStatus().getIsNodeHealthy(); this.lastNodeUpdateTime = context.getNodeHealthStatus() .getLastHealthReportTime(); this.healthReport = context.getNodeHealthStatus().getHealthReport(); this.nodeManagerVersion = YarnVersionInfo.getVersion(); this.nodeManagerBuildVersion = YarnVersionInfo.getBuildVersion(); this.nodeManagerVersionBuiltOn = YarnVersionInfo.getDate(); this.hadoopVersion = VersionInfo.getVersion(); this.hadoopBuildVersion = VersionInfo.getBuildVersion(); this.hadoopVersionBuiltOn = VersionInfo.getDate(); }
public NodeInfo(final Context context, final ResourceView resourceView) { this.id = context.getNodeId().toString(); this.nodeHostName = context.getNodeId().getHost(); this.totalVmemAllocatedContainersMB = resourceView .getVmemAllocatedForContainers() / BYTES_IN_MB; this.vmemCheckEnabled = resourceView.isVmemCheckEnabled(); this.totalPmemAllocatedContainersMB = resourceView .getPmemAllocatedForContainers() / BYTES_IN_MB; this.pmemCheckEnabled = resourceView.isPmemCheckEnabled(); this.totalVCoresAllocatedContainers = resourceView .getVCoresAllocatedForContainers(); this.resourceTypes = StringUtils.join(", ", ResourceUtils.getResourcesTypeInfo()); this.nodeHealthy = context.getNodeHealthStatus().getIsNodeHealthy(); this.lastNodeUpdateTime = context.getNodeHealthStatus() .getLastHealthReportTime(); this.healthReport = context.getNodeHealthStatus().getHealthReport(); this.nodeManagerVersion = YarnVersionInfo.getVersion(); this.nodeManagerBuildVersion = YarnVersionInfo.getBuildVersion(); this.nodeManagerVersionBuiltOn = YarnVersionInfo.getDate(); this.hadoopVersion = VersionInfo.getVersion(); this.hadoopBuildVersion = VersionInfo.getBuildVersion(); this.hadoopVersionBuiltOn = VersionInfo.getDate(); this.nmStartupTime = NodeManager.getNMStartupTime(); }
/** * Simulates a NM heartbeat using the simulated NodeStatus fixture. Verify * both the RMNode and SchedulerNode have been updated with the new * utilization. */ @Test(timeout=60000) public void testUpdateNodeUtilization() throws InterruptedException, IOException, YarnException { assertTrue("NMs fail to connect to the RM", cluster.waitForNodeManagersToConnect(10000)); // Give the heartbeat time to propagate to the RM verifySimulatedUtilization(); // Alter utilization nodeStatus = createNodeStatus(nm.getNMContext().getNodeId(), 0, CONTAINER_PMEM_2, CONTAINER_VMEM_2, CONTAINER_CPU_2, NODE_PMEM_2, NODE_VMEM_2, NODE_CPU_2); nm.setNodeStatus(nodeStatus); // Give the heartbeat time to propagate to the RM verifySimulatedUtilization(); }
/** * Trigger the NM to send a heartbeat using the simulated NodeStatus fixture. * Verify both the RMNode and SchedulerNode have been updated with the new * utilization. */ @Test(timeout=60000) public void testMockNodeStatusHeartbeat() throws InterruptedException, YarnException { assertTrue("NMs fail to connect to the RM", cluster.waitForNodeManagersToConnect(10000)); NodeStatusUpdater updater = nm.getNodeStatusUpdater(); updater.sendOutofBandHeartBeat(); // Give the heartbeat time to propagate to the RM verifySimulatedUtilization(); // Alter utilization nodeStatus = createNodeStatus(nm.getNMContext().getNodeId(), 0, CONTAINER_PMEM_2, CONTAINER_VMEM_2, CONTAINER_CPU_2, NODE_PMEM_2, NODE_VMEM_2, NODE_CPU_2); nm.setNodeStatus(nodeStatus); updater.sendOutofBandHeartBeat(); verifySimulatedUtilization(); }