@Override public ReservationId getNewReservationId() { writeLock.lock(); try { ReservationId resId = ReservationId.newInstance(ResourceManager.getClusterTimeStamp(), resCounter.incrementAndGet()); LOG.info("Allocated new reservationId: " + resId); return resId; } finally { writeLock.unlock(); } }
@Override public ReservationId getNewReservationId() { writeLock.lock(); try { ReservationId resId = ReservationId.newInstance( ResourceManager.getClusterTimeStamp(), resCounter.incrementAndGet()); LOG.info("Allocated new reservationId: " + resId); return resId; } finally { writeLock.unlock(); } }
@Override public ReservationId getNewReservationId() { writeLock.lock(); try { ReservationId resId = ReservationId.newInstance(ResourceManager.getClusterTimeStamp(), resCounter.incrementAndGet()); LOG.info("Allocated new reservationId: " + resId); return resId; } finally { writeLock.unlock(); } }
ApplicationId getNewApplicationId() { ApplicationId applicationId = org.apache.hadoop.yarn.server.utils.BuilderUtils .newApplicationId(recordFactory, ResourceManager.getClusterTimeStamp(), applicationCounter.incrementAndGet()); LOG.info("Allocated new applicationId: " + applicationId.getId()); return applicationId; }
ApplicationId getNewApplicationId() { ApplicationId applicationId = org.apache.hadoop.yarn.server.utils.BuilderUtils .newApplicationId(recordFactory, ResourceManager.getClusterTimeStamp(), applicationCounter.incrementAndGet()); LOG.info("Allocated new applicationId: " + applicationId.getId()); return applicationId; }
ApplicationId getNewApplicationId() { ApplicationId applicationId = org.apache.hadoop.yarn.server.utils.BuilderUtils .newApplicationId(recordFactory, ResourceManager.getClusterTimeStamp(), applicationCounter.incrementAndGet()); LOG.info("Allocated new applicationId: " + applicationId.getId()); return applicationId; }
@Override public Resource getMaximumResourceCapability() { Resource maxResource; maxAllocReadLock.lock(); try { if (useConfiguredMaximumAllocationOnly) { if (System.currentTimeMillis() - ResourceManager.getClusterTimeStamp() > configuredMaximumAllocationWaitTime) { useConfiguredMaximumAllocationOnly = false; } maxResource = Resources.clone(configuredMaximumAllocation); } else { maxResource = Resources.clone(maximumAllocation); } } finally { maxAllocReadLock.unlock(); } return maxResource; }
@Override public Resource getMaximumResourceCapability() { Resource maxResource; maxAllocReadLock.lock(); try { if (useConfiguredMaximumAllocationOnly) { if (System.currentTimeMillis() - ResourceManager.getClusterTimeStamp() > configuredMaximumAllocationWaitTime) { useConfiguredMaximumAllocationOnly = false; } maxResource = Resources.clone(configuredMaximumAllocation); } else { maxResource = Resources.clone(maximumAllocation); } } finally { maxAllocReadLock.unlock(); } return maxResource; }
private class NextKeyActivator extends TimerTask { @Override public void run() { // Activation will happen after an absolute time interval. It will be good // if we can force activation after an NM updates and acknowledges a // roll-over. But that is only possible when we move to per-NM keys. TODO: activateNextMasterKey(); } }
public Resource getMaxAllowedAllocation() { readLock.lock(); try { if (forceConfiguredMaxAllocation && System.currentTimeMillis() - ResourceManager.getClusterTimeStamp() > configuredMaxAllocationWaitTime) { forceConfiguredMaxAllocation = false; } if (forceConfiguredMaxAllocation || !reportedMaxAllocation) { return configuredMaxAllocation; } Resource ret = Resources.clone(configuredMaxAllocation); for (int i = 0; i < maxAllocation.length; i++) { ResourceInformation info = ret.getResourceInformation(i); if (info.getValue() > maxAllocation[i]) { info.setValue(maxAllocation[i]); } } return ret; } finally { readLock.unlock(); } }
public FederationStateStoreHeartbeat(SubClusterId subClusterId, FederationStateStore stateStoreClient, ResourceScheduler scheduler) { this.stateStoreService = stateStoreClient; this.subClusterId = subClusterId; this.rs = scheduler; // Initialize the JAXB Marshaller this.currentClusterState = new StringWriter(); try { this.jc = new JSONJAXBContext( JSONConfiguration.mapped().rootUnwrapping(false).build(), ClusterMetricsInfo.class); marshaller = jc.createJSONMarshaller(); } catch (JAXBException e) { LOG.warn("Exception while trying to initialize JAXB context.", e); } LOG.info("Initialized Federation membership for cluster with timestamp: " + ResourceManager.getClusterTimeStamp()); }
protected void releaseContainers(List<ContainerId> containers, SchedulerApplicationAttempt attempt) { for (ContainerId containerId : containers) { RMContainer rmContainer = getRMContainer(containerId); if (rmContainer == null) { if (System.currentTimeMillis() - ResourceManager.getClusterTimeStamp() < nmExpireInterval) { LOG.info(containerId + " doesn't exist. Add the container" + " to the release request cache as it maybe on recovery."); synchronized (attempt) { attempt.getPendingRelease().add(containerId); } } else { RMAuditLogger.logFailure(attempt.getUser(), AuditConstants.RELEASE_CONTAINER, "Unauthorized access or invalid container", "Scheduler", "Trying to release container not owned by app or with invalid id.", attempt.getApplicationId(), containerId); } } completedContainer(rmContainer, SchedulerUtils.createAbnormalContainerStatus(containerId, SchedulerUtils.RELEASED_CONTAINER), RMContainerEventType.RELEASED); } }
protected void releaseContainers(List<ContainerId> containers, SchedulerApplicationAttempt attempt) { for (ContainerId containerId : containers) { RMContainer rmContainer = getRMContainer(containerId); if (rmContainer == null) { if (System.currentTimeMillis() - ResourceManager.getClusterTimeStamp() < nmExpireInterval) { LOG.info(containerId + " doesn't exist. Add the container" + " to the release request cache as it maybe on recovery."); synchronized (attempt) { attempt.getPendingRelease().add(containerId); } } else { RMAuditLogger.logFailure(attempt.getUser(), AuditConstants.RELEASE_CONTAINER, "Unauthorized access or invalid container", "Scheduler", "Trying to release container not owned by app or with invalid id.", attempt.getApplicationId(), containerId); } } completedContainer(rmContainer, SchedulerUtils.createAbnormalContainerStatus(containerId, SchedulerUtils.RELEASED_CONTAINER), RMContainerEventType.RELEASED); } }
protected void releaseContainers(List<ContainerId> containers, SchedulerApplicationAttempt attempt) { for (ContainerId containerId : containers) { RMContainer rmContainer = getRMContainer(containerId); if (rmContainer == null) { if (System.currentTimeMillis() - ResourceManager.getClusterTimeStamp() < nmExpireInterval) { LOG.info(containerId + " doesn't exist. Add the container" + " to the release request cache as it maybe on recovery."); attempt.getPendingRelease().add(containerId); } else { RMAuditLogger.logFailure(attempt.getUser(), AuditConstants.RELEASE_CONTAINER, "Unauthorized access or invalid container", "Scheduler", "Trying to release container not owned by app or with invalid id.", attempt.getApplicationId(), containerId, null); } } completedContainer(rmContainer, SchedulerUtils.createAbnormalContainerStatus(containerId, SchedulerUtils.RELEASED_CONTAINER), RMContainerEventType.RELEASED); } }
public ClusterInfo(ResourceManager rm) { long ts = ResourceManager.getClusterTimeStamp(); this.id = ts; this.state = rm.getServiceState(); this.haState = rm.getRMContext().getHAServiceState(); this.rmStateStoreName = rm.getRMContext().getStateStore().getClass() .getName(); this.startedOn = ts; this.resourceManagerVersion = YarnVersionInfo.getVersion(); this.resourceManagerBuildVersion = YarnVersionInfo.getBuildVersion(); this.resourceManagerVersionBuiltOn = YarnVersionInfo.getDate(); this.hadoopVersion = VersionInfo.getVersion(); this.hadoopBuildVersion = VersionInfo.getBuildVersion(); this.hadoopVersionBuiltOn = VersionInfo.getDate(); this.haZooKeeperConnectionState = rm.getRMContext().getHAZookeeperConnectionState(); }
public ClusterInfo(ResourceManager rm) { long ts = ResourceManager.getClusterTimeStamp(); this.id = ts; this.state = rm.getServiceState(); this.haState = rm.getRMContext().getHAServiceState(); this.rmStateStoreName = rm.getRMContext().getStateStore().getClass() .getName(); this.startedOn = ts; this.resourceManagerVersion = YarnVersionInfo.getVersion(); this.resourceManagerBuildVersion = YarnVersionInfo.getBuildVersion(); this.resourceManagerVersionBuiltOn = YarnVersionInfo.getDate(); this.hadoopVersion = VersionInfo.getVersion(); this.hadoopBuildVersion = VersionInfo.getBuildVersion(); this.hadoopVersionBuiltOn = VersionInfo.getDate(); this.haZooKeeperConnectionState = rm.getRMContext().getRMAdminService().getHAZookeeperConnectionState(); }
public ClusterInfo(ResourceManager rm) { long ts = ResourceManager.getClusterTimeStamp(); this.id = ts; this.state = rm.getServiceState(); this.haState = rm.getRMContext().getHAServiceState(); this.rmStateStoreName = rm.getRMContext().getStateStore().getClass() .getName(); this.startedOn = ts; this.resourceManagerVersion = YarnVersionInfo.getVersion(); this.resourceManagerBuildVersion = YarnVersionInfo.getBuildVersion(); this.resourceManagerVersionBuiltOn = YarnVersionInfo.getDate(); this.hadoopVersion = VersionInfo.getVersion(); this.hadoopBuildVersion = VersionInfo.getBuildVersion(); this.hadoopVersionBuiltOn = VersionInfo.getDate(); this.haZooKeeperConnectionState = rm.getRMContext().getRMAdminService().getHAZookeeperConnectionState(); }
@Test public void testSetRMIdentifierInRegistration() throws Exception { Configuration conf = new Configuration(); rm = new MockRM(conf); rm.start(); MockNM nm = new MockNM("host1:1234", 5120, rm.getResourceTrackerService()); RegisterNodeManagerResponse response = nm.registerNode(); // Verify the RMIdentifier is correctly set in RegisterNodeManagerResponse Assert.assertEquals(ResourceManager.getClusterTimeStamp(), response.getRMIdentifier()); }
@Test public void testSetRMIdentifierInRegistration() throws Exception { Configuration conf = new Configuration(); rm = new MockRM(conf); rm.start(); MockNM nm = new MockNM("host1:1234", 5120, rm.getResourceTrackerService()); RegisterNodeManagerResponse response = nm.registerNode(); // Verify the RMIdentifier is correctly set in RegisterNodeManagerResponse Assert.assertEquals(ResourceManager.getClusterTimeStamp(), response.getRMIdentifier()); }
public void verifyClusterGeneric(long clusterid, long startedon, String state, String haState, String haZooKeeperConnectionState, String hadoopVersionBuiltOn, String hadoopBuildVersion, String hadoopVersion, String resourceManagerVersionBuiltOn, String resourceManagerBuildVersion, String resourceManagerVersion) { assertEquals("clusterId doesn't match: ", ResourceManager.getClusterTimeStamp(), clusterid); assertEquals("startedOn doesn't match: ", ResourceManager.getClusterTimeStamp(), startedon); assertTrue("stated doesn't match: " + state, state.matches(STATE.INITED.toString())); assertTrue("HA state doesn't match: " + haState, haState.matches("INITIALIZING")); WebServicesTestUtils.checkStringMatch("hadoopVersionBuiltOn", VersionInfo.getDate(), hadoopVersionBuiltOn); WebServicesTestUtils.checkStringEqual("hadoopBuildVersion", VersionInfo.getBuildVersion(), hadoopBuildVersion); WebServicesTestUtils.checkStringMatch("hadoopVersion", VersionInfo.getVersion(), hadoopVersion); WebServicesTestUtils.checkStringMatch("resourceManagerVersionBuiltOn", YarnVersionInfo.getDate(), resourceManagerVersionBuiltOn); WebServicesTestUtils.checkStringEqual("resourceManagerBuildVersion", YarnVersionInfo.getBuildVersion(), resourceManagerBuildVersion); WebServicesTestUtils.checkStringMatch("resourceManagerVersion", YarnVersionInfo.getVersion(), resourceManagerVersion); }