private boolean tooManyTasksPerOfferHostForRequest(Map<String, Integer> tasksPerOffer, String hostname, SingularityTaskRequest taskRequest) { if (!tasksPerOffer.containsKey(hostname)) { return false; } int maxPerOfferPerRequest = taskRequest.getRequest().getMaxTasksPerOffer().or(configuration.getMaxTasksPerOfferPerRequest()); return maxPerOfferPerRequest > 0 && tasksPerOffer.get(hostname) > maxPerOfferPerRequest; }
private boolean isTooManyInstancesForRequest(SingularityTaskRequest taskRequest, List<SingularityTaskId> activeTaskIdsForRequest) { if (taskRequest.getRequest().getRequestType() == RequestType.ON_DEMAND) { int maxActiveOnDemandTasks = taskRequest.getRequest().getInstances().or(configuration.getMaxActiveOnDemandTasksPerRequest()); if (maxActiveOnDemandTasks > 0) { int activeTasksForRequest = activeTaskIdsForRequest.size(); LOG.debug("Running {} instances for request {}. Max is {}", activeTasksForRequest, taskRequest.getRequest().getId(), maxActiveOnDemandTasks); if (activeTasksForRequest >= maxActiveOnDemandTasks) { return true; } } } return false; } }
@Override public String apply(@Nonnull SingularityTask input) { return input.getTaskRequest().getRequest().getId(); } };
private boolean areSlaveAttributeMinimumsFeasible(SingularityOfferHolder offerHolder, SingularityTaskRequest taskRequest, List<SingularityTaskId> activeTaskIdsForRequest) { if (!taskRequest.getRequest().getSlaveAttributeMinimums().isPresent()) { return true; } Map<String, String> offerAttributes = slaveManager.getSlave(offerHolder.getSlaveId()).get().getAttributes(); Integer numDesiredInstances = taskRequest.getRequest().getInstancesSafe(); Integer numActiveInstances = activeTaskIdsForRequest.size(); for (Entry<String, Map<String, Integer>> keyEntry : taskRequest.getRequest().getSlaveAttributeMinimums().get().entrySet()) { String attrKey = keyEntry.getKey(); for (Entry<String, Integer> valueEntry : keyEntry.getValue().entrySet()) { Integer percentInstancesWithAttr = valueEntry.getValue(); Integer minInstancesWithAttr = Math.max(1, (int) ((percentInstancesWithAttr / 100.0) * numDesiredInstances)); if (offerAttributes.containsKey(attrKey) && offerAttributes.get(attrKey).equals(valueEntry.getKey())) { // Accepting this offer would add an instance of the needed attribute, so it's okay. continue; } // Would accepting this offer prevent meeting the necessary attribute in the future? long numInstancesWithAttr = getNumInstancesWithAttribute(activeTaskIdsForRequest, attrKey, valueEntry.getKey()); long numInstancesWithoutAttr = numActiveInstances - numInstancesWithAttr + 1; long maxPotentialInstancesWithAttr = numDesiredInstances - numInstancesWithoutAttr; if (maxPotentialInstancesWithAttr < minInstancesWithAttr) { return false; } } } return true; }
private boolean isPreemptibleTask(SingularityTaskRequest taskRequest) { // A long running task can be replaced + killed easily if (taskRequest.getRequest().getRequestType().isLongRunning()) { return true; } // A short, non-long-running task Optional<SingularityDeployStatistics> deployStatistics = deployManager.getDeployStatistics(taskRequest.getRequest().getId(), taskRequest.getDeploy().getId()); return deployStatistics.isPresent() && deployStatistics.get().getAverageRuntimeMillis().isPresent() && deployStatistics.get().getAverageRuntimeMillis().get() < configuration.getPreemptibleTaskMaxExpectedRuntimeMs(); }
public void removeTasksAffectedByPriorityFreeze(List<SingularityTaskRequest> taskRequests) { final Optional<SingularityPriorityFreezeParent> maybePriorityFreeze = priorityManager.getActivePriorityFreeze(); if (maybePriorityFreeze.isPresent()) { final ListIterator<SingularityTaskRequest> iterator = taskRequests.listIterator(); while (iterator.hasNext()) { final SingularityTaskRequest taskRequest = iterator.next(); final double taskPriorityLevel = priorityManager.getTaskPriorityLevelForRequest(taskRequest.getRequest()); if (taskPriorityLevel < maybePriorityFreeze.get().getPriorityFreeze().getMinimumPriorityLevel()) { LOG.trace("Skipping scheduled task {} because taskPriorityLevel ({}) is less than active priority freeze ({})", taskRequest.getPendingTask().getPendingTaskId(), taskPriorityLevel, maybePriorityFreeze.get().getPriorityFreeze().getMinimumPriorityLevel()); iterator.remove(); } } } }
Map<String, String> reservedSlaveAttributes = offer.getReservedSlaveAttributes(); if ((taskRequest.getRequest().getRequiredSlaveAttributes().isPresent() && !taskRequest.getRequest().getRequiredSlaveAttributes().get().isEmpty()) || (taskRequest.getRequest().getAllowedSlaveAttributes().isPresent() && !taskRequest.getRequest().getAllowedSlaveAttributes().get().isEmpty())) { Map<String, String> mergedAttributes = new HashMap<>(); mergedAttributes.putAll(taskRequest.getRequest().getRequiredSlaveAttributes().or(new HashMap<>())); mergedAttributes.putAll(taskRequest.getRequest().getAllowedSlaveAttributes().or(new HashMap<>())); if (!slaveAndRackHelper.hasRequiredAttributes(mergedAttributes, reservedSlaveAttributes)) { LOG.trace("Slaves with attributes {} are reserved for matching tasks. Task with attributes {} does not match", reservedSlaveAttributes, taskRequest.getRequest().getRequiredSlaveAttributes().or(Collections.emptyMap())); return false; if (taskRequest.getRequest().getRequiredSlaveAttributes().isPresent() && !slaveAndRackHelper.hasRequiredAttributes(offer.getTextAttributes(), taskRequest.getRequest().getRequiredSlaveAttributes().get())) { LOG.trace("Task requires slave with attributes {}, (slave attributes are {})", taskRequest.getRequest().getRequiredSlaveAttributes().get(), offer.getTextAttributes()); return false;
@GET @PropertyFiltering @Path("/scheduled/task/{pendingTaskId}") @Operation(summary = "Retrieve information about a pending task") public SingularityTaskRequest getPendingTask( @Parameter(hidden = true) @Auth SingularityUser user, @Parameter(description = "The unique id of the pending task") @PathParam("pendingTaskId") String pendingTaskIdStr) { Optional<SingularityPendingTask> pendingTask = taskManager.getPendingTask(getPendingTaskIdFromStr(pendingTaskIdStr)); checkNotFound(pendingTask.isPresent(), "Couldn't find %s", pendingTaskIdStr); List<SingularityTaskRequest> taskRequestList = taskRequestManager.getTaskRequests(Collections.singletonList(pendingTask.get())); checkNotFound(!taskRequestList.isEmpty(), "Couldn't find: " + pendingTaskIdStr); authorizationHelper.checkForAuthorization(taskRequestList.get(0).getRequest(), user, SingularityAuthorizationScope.READ); return taskRequestList.get(0); }
private double getWeightedPriority(SingularityTaskRequest taskRequest, long now) { Long overdueMillis = Math.max(now - taskRequest.getPendingTask().getPendingTaskId().getNextRunAt(), 1); Double requestPriority = priorityManager.getTaskPriorityLevelForRequest(taskRequest.getRequest()); return overdueMillis * Math.pow(requestPriority, configuration.getSchedulerPriorityWeightFactor()); }
private void cleanupTaskDueToDecomission(final Map<String, Optional<String>> requestIdsToUserToReschedule, final Set<SingularityTaskId> matchingTaskIds, SingularityTask task, SingularityMachineAbstraction<?> decommissioningObject) { requestIdsToUserToReschedule.put(task.getTaskRequest().getRequest().getId(), decommissioningObject.getCurrentState().getUser()); matchingTaskIds.add(task.getTaskId()); LOG.trace("Scheduling a cleanup task for {} due to decomissioning {}", task.getTaskId(), decommissioningObject); taskManager.createTaskCleanup(new SingularityTaskCleanup(decommissioningObject.getCurrentState().getUser(), TaskCleanupType.DECOMISSIONING, System.currentTimeMillis(), task.getTaskId(), Optional.of(String.format("%s %s is decomissioning", decommissioningObject.getTypeName(), decommissioningObject.getName())), Optional.<String>absent(), Optional.<SingularityTaskShellCommandRequestId>absent())); }
private Optional<String> getRequestGroupForTask(final SingularityTaskId taskId, SingularityUser user) { Optional<SingularityTaskHistory> maybeTaskHistory = getTaskHistory(taskId, user); if (maybeTaskHistory.isPresent()) { SingularityRequest request = maybeTaskHistory.get().getTask().getTaskRequest().getRequest(); authorizationHelper.checkForAuthorization(request, user, SingularityAuthorizationScope.READ); return request.getGroup(); } else { return getRequestGroup(taskId.getRequestId(), user); } }
private List<SingularityTaskRequest> checkForStaleScheduledTasks(List<SingularityPendingTask> pendingTasks, List<SingularityTaskRequest> taskRequests) { final Set<String> foundPendingTaskId = Sets.newHashSetWithExpectedSize(taskRequests.size()); final Set<String> requestIds = Sets.newHashSetWithExpectedSize(taskRequests.size()); for (SingularityTaskRequest taskRequest : taskRequests) { foundPendingTaskId.add(taskRequest.getPendingTask().getPendingTaskId().getId()); requestIds.add(taskRequest.getRequest().getId()); } for (SingularityPendingTask pendingTask : pendingTasks) { if (!foundPendingTaskId.contains(pendingTask.getPendingTaskId().getId())) { LOG.info("Removing stale pending task {}", pendingTask.getPendingTaskId()); taskManager.deletePendingTask(pendingTask.getPendingTaskId()); } } // TODO this check isn't necessary if we keep track better during deploys final Map<String, SingularityRequestDeployState> deployStates = deployManager.getRequestDeployStatesByRequestIds(requestIds); final List<SingularityTaskRequest> taskRequestsWithValidDeploys = Lists.newArrayListWithCapacity(taskRequests.size()); for (SingularityTaskRequest taskRequest : taskRequests) { SingularityRequestDeployState requestDeployState = deployStates.get(taskRequest.getRequest().getId()); if (!matchesDeploy(requestDeployState, taskRequest) && !(taskRequest.getRequest().getRequestType() == RequestType.RUN_ONCE)) { LOG.info("Removing stale pending task {} because the deployId did not match active/pending deploys {}", taskRequest.getPendingTask().getPendingTaskId(), requestDeployState); taskManager.deletePendingTask(taskRequest.getPendingTask().getPendingTaskId()); } else { taskRequestsWithValidDeploys.add(taskRequest); } } return taskRequestsWithValidDeploys; }
@Before public void setup() { Mockito.when(taskRequest.getRequest()).thenReturn(request); Mockito.when(request.getId()).thenReturn("requestId"); Mockito.when(taskRequest.getDeploy()).thenReturn(deploy); Mockito.when(deploy.getId()).thenReturn("deployId"); Mockito.when(taskRequest.getPendingTask()).thenReturn(task); Mockito.when(task.getPendingTaskId()).thenReturn(taskId); }
private int getDelaySeconds(SingularityTask task, Optional<SingularityRequestWithState> requestWithState) { int delaySeconds = configuration.getNewTaskCheckerBaseDelaySeconds(); if (hasHealthcheck(task, requestWithState)) { Optional<Integer> maybeStartupDelay = task.getTaskRequest().getDeploy().getHealthcheck().get().getStartupDelaySeconds().or(configuration.getStartupDelaySeconds()); if (maybeStartupDelay.isPresent()) { return maybeStartupDelay.get(); } } else if (task.getTaskRequest().getRequest().isLoadBalanced()) { return delaySeconds; } delaySeconds += task.getTaskRequest().getDeploy().getDeployHealthTimeoutSeconds().or(configuration.getDeployHealthyBySeconds()); return delaySeconds; }
private boolean shouldHealthcheck(final SingularityTask task, final Optional<SingularityRequestWithState> request, Optional<SingularityPendingDeploy> pendingDeploy) { if (disasterManager.isDisabled(SingularityAction.RUN_HEALTH_CHECKS)) { return false; } if (!task.getTaskRequest().getRequest().isLongRunning() || !task.getTaskRequest().getDeploy().getHealthcheck().isPresent() || task.getTaskRequest().getDeploy().getHealthcheck().get().getHealthcheckResultFilePath().isPresent()) { return false; } if (task.getTaskRequest().getPendingTask().getSkipHealthchecks().or(false)) { return false; } if (pendingDeploy.isPresent() && pendingDeploy.get().getDeployMarker().getDeployId().equals(task.getTaskId().getDeployId()) && task.getTaskRequest().getDeploy().getSkipHealthchecksOnDeploy().or(false)) { return false; } if (request.isPresent() && request.get().getRequest().getSkipHealthchecks().or(false)) { return false; } Optional<SingularityTaskHealthcheckResult> lastHealthcheck = taskManager.getLastHealthcheck(task.getTaskId()); if (lastHealthcheck.isPresent() && !lastHealthcheck.get().isFailed()) { LOG.debug("Not submitting a new healthcheck for {} because it already passed a healthcheck", task.getTaskId()); return false; } return true; }
private SingularityMesosTaskHolder acceptTask(SingularityOfferHolder offerHolder, Map<String, Integer> tasksPerOffer, SingularityTaskRequestHolder taskRequestHolder) { final SingularityTaskRequest taskRequest = taskRequestHolder.getTaskRequest(); final SingularityMesosTaskHolder taskHolder = mesosTaskBuilder.buildTask(offerHolder, offerHolder.getCurrentResources(), taskRequest, taskRequestHolder.getTaskResources(), taskRequestHolder.getExecutorResources()); final SingularityTask zkTask = taskSizeOptimizer.getSizeOptimizedTask(taskHolder); LOG.trace("Accepted and built task {}", zkTask); LOG.info("Launching task {} slot on slave {} ({})", taskHolder.getTask().getTaskId(), offerHolder.getSlaveId(), offerHolder.getHostname()); taskManager.createTaskAndDeletePendingTask(zkTask); addRequestToMapByOfferHost(tasksPerOffer, offerHolder.getHostname(), taskRequest.getRequest().getId()); return taskHolder; }
private void updateSlaveUsageScores(SingularityTaskRequestHolder taskHolder, Map<String, SingularitySlaveUsageWithCalculatedScores> currentSlaveUsagesBySlaveId, String slaveId, Map<String, RequestUtilization> requestUtilizations) { Optional<SingularitySlaveUsageWithCalculatedScores> maybeUsage = Optional.fromNullable(currentSlaveUsagesBySlaveId.get(slaveId)); if (maybeUsage.isPresent() && !maybeUsage.get().isMissingUsageData()) { SingularitySlaveUsageWithCalculatedScores usage = maybeUsage.get(); usage.addEstimatedCpuReserved(taskHolder.getTotalResources().getCpus()); usage.addEstimatedMemoryReserved(taskHolder.getTotalResources().getMemoryMb()); usage.addEstimatedDiskReserved(taskHolder.getTotalResources().getDiskMb()); if (requestUtilizations.containsKey(taskHolder.getTaskRequest().getRequest().getId())) { RequestUtilization requestUtilization = requestUtilizations.get(taskHolder.getTaskRequest().getRequest().getId()); usage.addEstimatedCpuUsage(requestUtilization.getMaxCpuUsed()); usage.addEstimatedMemoryBytesUsage(requestUtilization.getMaxMemBytesUsed()); usage.addEstimatedDiskBytesUsage(requestUtilization.getMaxDiskBytesUsed()); } else { usage.addEstimatedCpuUsage(taskHolder.getTotalResources().getCpus()); usage.addEstimatedMemoryBytesUsage(taskHolder.getTotalResources().getMemoryMb() * SingularitySlaveUsage.BYTES_PER_MEGABYTE); usage.addEstimatedDiskBytesUsage(taskHolder.getTotalResources().getDiskMb() * SingularitySlaveUsage.BYTES_PER_MEGABYTE); } usage.recalculateScores(); } }
SingularityTask getSizeOptimizedTask(SingularityMesosTaskHolder taskHolder) { if (configuration.isStoreAllMesosTaskInfoForDebugging()) { return taskHolder.getTask(); } SingularityTask task = taskHolder.getTask(); TaskInfo.Builder mesosTask = taskHolder.getMesosTask().toBuilder(); mesosTask.clearData(); List<MesosOfferObject> offers = task.getOffers() .stream() .map(MesosOfferObject::sizeOptimized) .collect(Collectors.toList()); SingularityTaskRequest taskRequest = task.getTaskRequest(); if (task.getTaskRequest().getDeploy().getExecutorData().isPresent()) { SingularityDeployBuilder deploy = task.getTaskRequest().getDeploy().toBuilder(); deploy.setExecutorData(Optional.absent()); taskRequest = new SingularityTaskRequest(task.getTaskRequest().getRequest(), deploy.build(), task.getTaskRequest().getPendingTask()); } return new SingularityTask(taskRequest, task.getTaskId(), offers, mesosProtosUtils.taskFromProtos(mesosTask.build()), task.getRackId()); }
Assert.assertEquals(sortedRequestsByPriority.get(0).getRequest().getId(), highPriorityRequest.getId()); Assert.assertEquals(sortedRequestsByPriority.get(1).getRequest().getId(), mediumPriorityRequest.getId()); Assert.assertEquals(sortedRequestsByPriority.get(2).getRequest().getId(), lowPriorityRequest.getId()); Assert.assertEquals(sortedRequestsByOverdueAndPriority.get(0).getRequest().getId(), lowPriorityRequest.getId()); Assert.assertEquals(sortedRequestsByOverdueAndPriority.get(1).getRequest().getId(), mediumPriorityRequest.getId()); Assert.assertEquals(sortedRequestsByOverdueAndPriority.get(2).getRequest().getId(), highPriorityRequest.getId());
private void relaunchTask(SingularityTask task) { SingularityPendingTask pendingTask = task.getTaskRequest().getPendingTask(); SingularityPendingRequest pendingRequest = new SingularityPendingRequestBuilder() .setRequestId(task.getTaskRequest().getRequest().getId()) .setDeployId(task.getTaskRequest().getDeploy().getId()) .setPendingType(PendingType.RETRY) .setUser(pendingTask.getUser()) .setRunId(pendingTask.getRunId()) .setCmdLineArgsList(pendingTask.getCmdLineArgsList()) .setSkipHealthchecks(pendingTask.getSkipHealthchecks()) .setMessage(pendingTask.getMessage()) .setResources(pendingTask.getResources()) .setS3UploaderAdditionalFiles(pendingTask.getS3UploaderAdditionalFiles()) .setRunAsUserOverride(pendingTask.getRunAsUserOverride()) .setEnvOverrides(pendingTask.getEnvOverrides()) .setExtraArtifacts(pendingTask.getExtraArtifacts()) .setActionId(pendingTask.getActionId()) .setRunAt(pendingTask.getPendingTaskId().getNextRunAt()) .setTimestamp(System.currentTimeMillis()) .build(); requestManager.addToPendingQueue(pendingRequest); }