@Override public int compareInputOrderTo(SchedulableEntity other) { if (other instanceof SchedulerApplicationAttempt) { return getApplicationId().compareTo( ((SchedulerApplicationAttempt)other).getApplicationId()); } return 1;//let other types go before this, if any }
@Override public String getId() { return getApplicationId().toString(); }
public synchronized void showRequests() { if (LOG.isDebugEnabled()) { for (Priority priority : getPriorities()) { Map<String, ResourceRequest> requests = getResourceRequests(priority); if (requests != null) { LOG.debug("showRequests:" + " application=" + getApplicationId() + " headRoom=" + getHeadroom() + " currentConsumption=" + currentConsumption.getMemory()); for (ResourceRequest request : requests.values()) { LOG.debug("showRequests:" + " application=" + getApplicationId() + " request=" + request); } } } } }
public synchronized void showRequests() { if (LOG.isDebugEnabled()) { for (Priority priority : getPriorities()) { Map<String, ResourceRequest> requests = getResourceRequests(priority); if (requests != null) { LOG.debug("showRequests:" + " application=" + getApplicationId() + " headRoom=" + getHeadroom() + " currentConsumption=" + currentConsumption.getMemory()); for (ResourceRequest request : requests.values()) { LOG.debug("showRequests:" + " application=" + getApplicationId() + " request=" + request); } } } } }
void startAppAllocationRecording(NodeId nodeID, long currTS, SchedulerApplicationAttempt application) { ApplicationId applicationId = application.getApplicationId(); if (recordingAppActivitiesUntilSpecifiedTime.containsKey(applicationId) && recordingAppActivitiesUntilSpecifiedTime.get(applicationId) > currTS) { appsAllocation.put(applicationId, new AppAllocation(application.getPriority(), nodeID, application.getQueueName())); } if (recordingAppActivitiesUntilSpecifiedTime.containsKey(applicationId) && recordingAppActivitiesUntilSpecifiedTime.get(applicationId) <= currTS) { turnOffActivityMonitoringForApp(applicationId); } }
public static boolean isBlacklisted(SchedulerApplicationAttempt application, SchedulerNode node, Log LOG) { if (application.isBlacklisted(node.getNodeName())) { if (LOG.isDebugEnabled()) { LOG.debug("Skipping 'host' " + node.getNodeName() + " for " + application.getApplicationId() + " since it has been blacklisted"); } return true; } if (application.isBlacklisted(node.getRackName())) { if (LOG.isDebugEnabled()) { LOG.debug("Skipping 'rack' " + node.getRackName() + " for " + application.getApplicationId() + " since it has been blacklisted"); } return true; } return false; }
public static boolean isBlacklisted(SchedulerApplicationAttempt application, SchedulerNode node, Log LOG) { if (application.isBlacklisted(node.getNodeName())) { if (LOG.isDebugEnabled()) { LOG.debug("Skipping 'host' " + node.getNodeName() + " for " + application.getApplicationId() + " since it has been blacklisted"); } return true; } if (application.isBlacklisted(node.getRackName())) { if (LOG.isDebugEnabled()) { LOG.debug("Skipping 'rack' " + node.getRackName() + " for " + application.getApplicationId() + " since it has been blacklisted"); } return true; } return false; }
public static boolean isPlaceBlacklisted( SchedulerApplicationAttempt application, SchedulerNode node, Log log) { if (application.isPlaceBlacklisted(node.getNodeName())) { if (log.isDebugEnabled()) { log.debug("Skipping 'host' " + node.getNodeName() + " for " + application.getApplicationId() + " since it has been blacklisted"); } return true; } if (application.isPlaceBlacklisted(node.getRackName())) { if (log.isDebugEnabled()) { log.debug("Skipping 'rack' " + node.getRackName() + " for " + application.getApplicationId() + " since it has been blacklisted"); } return true; } return false; }
public static void recordRejectedAppActivityFromLeafQueue( ActivitiesManager activitiesManager, SchedulerNode node, SchedulerApplicationAttempt application, Priority priority, String diagnostic) { String type = "app"; if (node == null || activitiesManager == null) { return; } if (activitiesManager.shouldRecordThisNode(node.getNodeID())) { recordActivity(activitiesManager, node, application.getQueueName(), application.getApplicationId().toString(), priority, ActivityState.REJECTED, diagnostic, type); } finishSkippedAppAllocationRecording(activitiesManager, application.getApplicationId(), ActivityState.REJECTED, diagnostic); }
@VisibleForTesting public void clearPendingContainerCache() { for (SchedulerApplication<T> app : applications.values()) { T attempt = app.getCurrentAppAttempt(); if (attempt != null) { for (ContainerId containerId : attempt.getPendingRelease()) { RMAuditLogger.logFailure(app.getUser(), AuditConstants.RELEASE_CONTAINER, "Unauthorized access or invalid container", "Scheduler", "Trying to release container not owned by app " + "or with invalid id.", attempt.getApplicationId(), containerId, null); } attempt.getPendingRelease().clear(); } } }
@Override public void run() { for (SchedulerApplication<T> app : applications.values()) { T attempt = app.getCurrentAppAttempt(); synchronized (attempt) { for (ContainerId containerId : attempt.getPendingRelease()) { RMAuditLogger.logFailure( app.getUser(), AuditConstants.RELEASE_CONTAINER, "Unauthorized access or invalid container", "Scheduler", "Trying to release container not owned by app or with invalid id.", attempt.getApplicationId(), containerId); } attempt.getPendingRelease().clear(); } } LOG.info("Release request cache is cleaned up"); } }, nmExpireInterval);
@Override public void run() { for (SchedulerApplication<T> app : applications.values()) { T attempt = app.getCurrentAppAttempt(); synchronized (attempt) { for (ContainerId containerId : attempt.getPendingRelease()) { RMAuditLogger.logFailure( app.getUser(), AuditConstants.RELEASE_CONTAINER, "Unauthorized access or invalid container", "Scheduler", "Trying to release container not owned by app or with invalid id.", attempt.getApplicationId(), containerId); } attempt.getPendingRelease().clear(); } } LOG.info("Release request cache is cleaned up"); } }, nmExpireInterval);
public void showRequests() { if (LOG.isDebugEnabled()) { try { readLock.lock(); for (SchedulerRequestKey schedulerKey : getSchedulerKeys()) { AppPlacementAllocator ap = getAppPlacementAllocator(schedulerKey); if (ap != null && ap.getOutstandingAsksCount(ResourceRequest.ANY) > 0) { LOG.debug("showRequests:" + " application=" + getApplicationId() + " headRoom=" + getHeadroom() + " currentConsumption=" + attemptResourceUsage.getUsed().getMemorySize()); ap.showRequests(); } } } finally { readLock.unlock(); } } }
protected void releaseContainers(List<ContainerId> containers, SchedulerApplicationAttempt attempt) { for (ContainerId containerId : containers) { RMContainer rmContainer = getRMContainer(containerId); if (rmContainer == null) { if (System.currentTimeMillis() - ResourceManager.getClusterTimeStamp() < nmExpireInterval) { LOG.info(containerId + " doesn't exist. Add the container" + " to the release request cache as it maybe on recovery."); synchronized (attempt) { attempt.getPendingRelease().add(containerId); } } else { RMAuditLogger.logFailure(attempt.getUser(), AuditConstants.RELEASE_CONTAINER, "Unauthorized access or invalid container", "Scheduler", "Trying to release container not owned by app or with invalid id.", attempt.getApplicationId(), containerId); } } completedContainer(rmContainer, SchedulerUtils.createAbnormalContainerStatus(containerId, SchedulerUtils.RELEASED_CONTAINER), RMContainerEventType.RELEASED); } }
protected void releaseContainers(List<ContainerId> containers, SchedulerApplicationAttempt attempt) { for (ContainerId containerId : containers) { RMContainer rmContainer = getRMContainer(containerId); if (rmContainer == null) { if (System.currentTimeMillis() - ResourceManager.getClusterTimeStamp() < nmExpireInterval) { LOG.info(containerId + " doesn't exist. Add the container" + " to the release request cache as it maybe on recovery."); synchronized (attempt) { attempt.getPendingRelease().add(containerId); } } else { RMAuditLogger.logFailure(attempt.getUser(), AuditConstants.RELEASE_CONTAINER, "Unauthorized access or invalid container", "Scheduler", "Trying to release container not owned by app or with invalid id.", attempt.getApplicationId(), containerId); } } completedContainer(rmContainer, SchedulerUtils.createAbnormalContainerStatus(containerId, SchedulerUtils.RELEASED_CONTAINER), RMContainerEventType.RELEASED); } }
protected void releaseContainers(List<ContainerId> containers, SchedulerApplicationAttempt attempt) { for (ContainerId containerId : containers) { RMContainer rmContainer = getRMContainer(containerId); if (rmContainer == null) { if (System.currentTimeMillis() - ResourceManager.getClusterTimeStamp() < nmExpireInterval) { LOG.info(containerId + " doesn't exist. Add the container" + " to the release request cache as it maybe on recovery."); attempt.getPendingRelease().add(containerId); } else { RMAuditLogger.logFailure(attempt.getUser(), AuditConstants.RELEASE_CONTAINER, "Unauthorized access or invalid container", "Scheduler", "Trying to release container not owned by app or with invalid id.", attempt.getApplicationId(), containerId, null); } } completedContainer(rmContainer, SchedulerUtils.createAbnormalContainerStatus(containerId, SchedulerUtils.RELEASED_CONTAINER), RMContainerEventType.RELEASED); } }
application.getApplicationId().toString(), null, priority.toString(), ActivityState.SKIPPED, diagnostic, type); type = "app"; application.getApplicationId().toString(), application.getPriority().toString(), ActivityState.SKIPPED, ActivityDiagnosticConstant.EMPTY, type); application.getApplicationId())) { String type = "container"; activitiesManager.addSchedulingActivityForApp( application.getApplicationId(), null, priority.toString(), appState, diagnostic, type);
@Override public synchronized void unreserveResource( SchedulerApplicationAttempt application) { // Cannot unreserve for wrong application... ApplicationAttemptId reservedApplication = getReservedContainer().getContainer().getId() .getApplicationAttemptId(); if (!reservedApplication.equals( application.getApplicationAttemptId())) { throw new IllegalStateException("Trying to unreserve " + " for application " + application.getApplicationId() + " when currently reserved " + " for application " + reservedApplication.getApplicationId() + " on node " + this); } setReservedContainer(null); this.reservedAppSchedulable = null; }
@Override public synchronized void unreserveResource( SchedulerApplicationAttempt application) { // Cannot unreserve for wrong application... ApplicationAttemptId reservedApplication = getReservedContainer().getContainer().getId().getApplicationAttemptId(); if (!reservedApplication.equals( application.getApplicationAttemptId())) { throw new IllegalStateException("Trying to unreserve " + " for application " + application.getApplicationId() + " when currently reserved " + " for application " + reservedApplication.getApplicationId() + " on node " + this); } setReservedContainer(null); this.reservedAppSchedulable = null; }
@Override public synchronized void unreserveResource( SchedulerApplicationAttempt application) { // Cannot unreserve for wrong application... ApplicationAttemptId reservedApplication = getReservedContainer().getContainer().getId().getApplicationAttemptId(); if (!reservedApplication.equals( application.getApplicationAttemptId())) { throw new IllegalStateException("Trying to unreserve " + " for application " + application.getApplicationId() + " when currently reserved " + " for application " + reservedApplication.getApplicationId() + " on node " + this); } setReservedContainer(null); this.reservedAppSchedulable = null; }