@Before public void setup() { Mockito.when(taskRequest.getRequest()).thenReturn(request); Mockito.when(request.getId()).thenReturn("requestId"); Mockito.when(taskRequest.getDeploy()).thenReturn(deploy); Mockito.when(deploy.getId()).thenReturn("deployId"); Mockito.when(taskRequest.getPendingTask()).thenReturn(task); Mockito.when(task.getPendingTaskId()).thenReturn(taskId); }
SingularityTask getSizeOptimizedTask(SingularityMesosTaskHolder taskHolder) { if (configuration.isStoreAllMesosTaskInfoForDebugging()) { return taskHolder.getTask(); } SingularityTask task = taskHolder.getTask(); TaskInfo.Builder mesosTask = taskHolder.getMesosTask().toBuilder(); mesosTask.clearData(); List<MesosOfferObject> offers = task.getOffers() .stream() .map(MesosOfferObject::sizeOptimized) .collect(Collectors.toList()); SingularityTaskRequest taskRequest = task.getTaskRequest(); if (task.getTaskRequest().getDeploy().getExecutorData().isPresent()) { SingularityDeployBuilder deploy = task.getTaskRequest().getDeploy().toBuilder(); deploy.setExecutorData(Optional.absent()); taskRequest = new SingularityTaskRequest(task.getTaskRequest().getRequest(), deploy.build(), task.getTaskRequest().getPendingTask()); } return new SingularityTask(taskRequest, task.getTaskId(), offers, mesosProtosUtils.taskFromProtos(mesosTask.build()), task.getRackId()); }
public SingularityTaskRequestHolder(SingularityTaskRequest taskRequest, Resources defaultResources, Resources defaultCustomExecutorResources) { this.taskRequest = taskRequest; this.executorResources = taskRequest.getDeploy().getCustomExecutorCmd().isPresent() ? taskRequest.getDeploy().getCustomExecutorResources().or(defaultCustomExecutorResources) : Resources.EMPTY_RESOURCES;; this.taskResources = taskRequest.getPendingTask().getResources().or(taskRequest.getDeploy().getResources()).or(defaultResources); this.totalResources = Resources.add(taskResources, executorResources); this.requestedPorts = new ArrayList<>(); if (taskRequest.getDeploy().getContainerInfo().isPresent() && taskRequest.getDeploy().getContainerInfo().get().getDocker().isPresent()) { requestedPorts.addAll(taskRequest.getDeploy().getContainerInfo().get().getDocker().get().getLiteralHostPorts()); } }
private double getWeightedPriority(SingularityTaskRequest taskRequest, long now) { Long overdueMillis = Math.max(now - taskRequest.getPendingTask().getPendingTaskId().getNextRunAt(), 1); Double requestPriority = priorityManager.getTaskPriorityLevelForRequest(taskRequest.getRequest()); return overdueMillis * Math.pow(requestPriority, configuration.getSchedulerPriorityWeightFactor()); }
private boolean isPreemptibleTask(SingularityTaskRequest taskRequest) { // A long running task can be replaced + killed easily if (taskRequest.getRequest().getRequestType().isLongRunning()) { return true; } // A short, non-long-running task Optional<SingularityDeployStatistics> deployStatistics = deployManager.getDeployStatistics(taskRequest.getRequest().getId(), taskRequest.getDeploy().getId()); return deployStatistics.isPresent() && deployStatistics.get().getAverageRuntimeMillis().isPresent() && deployStatistics.get().getAverageRuntimeMillis().get() < configuration.getPreemptibleTaskMaxExpectedRuntimeMs(); }
@Override public int compareTo(SingularityTaskRequest o) { return this.getPendingTask().getPendingTaskId().compareTo(o.getPendingTask().getPendingTaskId()); }
private boolean tooManyTasksPerOfferHostForRequest(Map<String, Integer> tasksPerOffer, String hostname, SingularityTaskRequest taskRequest) { if (!tasksPerOffer.containsKey(hostname)) { return false; } int maxPerOfferPerRequest = taskRequest.getRequest().getMaxTasksPerOffer().or(configuration.getMaxTasksPerOfferPerRequest()); return maxPerOfferPerRequest > 0 && tasksPerOffer.get(hostname) > maxPerOfferPerRequest; }
public SingularityHealthcheckAsyncHandler(SingularityExceptionNotifier exceptionNotifier, SingularityConfiguration configuration, SingularityHealthchecker healthchecker, SingularityNewTaskChecker newTaskChecker, TaskManager taskManager, SingularityTask task) { this.exceptionNotifier = exceptionNotifier; this.taskManager = taskManager; this.newTaskChecker = newTaskChecker; this.healthchecker = healthchecker; this.task = task; this.maxHealthcheckResponseBodyBytes = configuration.getMaxHealthcheckResponseBodyBytes(); this.failureStatusCodes = task.getTaskRequest().getDeploy().getHealthcheck().isPresent() ? task.getTaskRequest().getDeploy().getHealthcheck().get().getFailureStatusCodes().or(configuration.getHealthcheckFailureStatusCodes()) : configuration.getHealthcheckFailureStatusCodes(); startTime = System.currentTimeMillis(); }
protected SingularityTaskRequest buildTaskRequest(SingularityRequest request, SingularityDeploy deploy, long launchTime) { return new SingularityTaskRequest(request, deploy, buildPendingTask(request, deploy, launchTime, 100, Optional.<String>absent())); }
.setArguments(Optional.of(Collections.singletonList("wat"))) .build(); final SingularityTaskRequest taskRequest = new SingularityTaskRequest(request, deploy, pendingTask); final SingularityMesosTaskHolder task = builder.buildTask(offerHolder, Collections.singletonList(portsResource), taskRequest, taskResources, executorResources); assertTrue(task.getMesosTask().getContainer().getDocker().getParametersList().contains(envParameter)); assertEquals(String.format("/container/%s/%s", task.getTask().getTaskRequest().getDeploy().getRequestId(), task.getTask().getTaskRequest().getDeploy().getId()), task.getMesosTask().getContainer().getVolumes(1).getContainerPath()); assertEquals(String.format("/host/%s", task.getMesosTask().getTaskId().getValue()), task.getMesosTask().getContainer().getVolumes(1).getHostPath()); assertEquals(Mode.RO, task.getMesosTask().getContainer().getVolumes(1).getMode());
private Optional<Integer> getCpuHardLimit(SingularityTaskRequest task) { if (configuration.getCpuHardLimit().isPresent()) { Optional<Resources> maybeResources = task.getPendingTask().getResources().or(task.getDeploy().getResources()); if (maybeResources.isPresent()) { double requestedCpus = maybeResources.get().getCpus(); int scaledLimit = (int) Math.ceil(requestedCpus * configuration.getCpuHardLimitScaleFactor()); return Optional.of(Math.max(scaledLimit, configuration.getCpuHardLimit().get())); } } return Optional.absent(); }
private int getDelaySeconds(SingularityTask task, Optional<SingularityRequestWithState> requestWithState) { int delaySeconds = configuration.getNewTaskCheckerBaseDelaySeconds(); if (hasHealthcheck(task, requestWithState)) { Optional<Integer> maybeStartupDelay = task.getTaskRequest().getDeploy().getHealthcheck().get().getStartupDelaySeconds().or(configuration.getStartupDelaySeconds()); if (maybeStartupDelay.isPresent()) { return maybeStartupDelay.get(); } } else if (task.getTaskRequest().getRequest().isLoadBalanced()) { return delaySeconds; } delaySeconds += task.getTaskRequest().getDeploy().getDeployHealthTimeoutSeconds().or(configuration.getDeployHealthyBySeconds()); return delaySeconds; }
public void removeTasksAffectedByPriorityFreeze(List<SingularityTaskRequest> taskRequests) { final Optional<SingularityPriorityFreezeParent> maybePriorityFreeze = priorityManager.getActivePriorityFreeze(); if (maybePriorityFreeze.isPresent()) { final ListIterator<SingularityTaskRequest> iterator = taskRequests.listIterator(); while (iterator.hasNext()) { final SingularityTaskRequest taskRequest = iterator.next(); final double taskPriorityLevel = priorityManager.getTaskPriorityLevelForRequest(taskRequest.getRequest()); if (taskPriorityLevel < maybePriorityFreeze.get().getPriorityFreeze().getMinimumPriorityLevel()) { LOG.trace("Skipping scheduled task {} because taskPriorityLevel ({}) is less than active priority freeze ({})", taskRequest.getPendingTask().getPendingTaskId(), taskPriorityLevel, maybePriorityFreeze.get().getPriorityFreeze().getMinimumPriorityLevel()); iterator.remove(); } } } }
public Optional<SingularityTaskId> getTaskByRunId(String requestId, String runId) { Map<SingularityTaskId, SingularityTask> activeTasks = getTasks(getActiveTaskIdsForRequest(requestId)); for (Map.Entry<SingularityTaskId, SingularityTask> entry : activeTasks.entrySet()) { if (entry.getValue().getTaskRequest().getPendingTask().getRunId().isPresent() && entry.getValue().getTaskRequest().getPendingTask().getRunId().get().equals(runId)) { return Optional.of(entry.getKey()); } } Map<SingularityTaskId, SingularityTask> inactiveTasks = getTasks(getInactiveTaskIdsForRequest(requestId)); for (Map.Entry<SingularityTaskId, SingularityTask> entry : inactiveTasks.entrySet()) { if (entry.getValue().getTaskRequest().getPendingTask().getRunId().isPresent() && entry.getValue().getTaskRequest().getPendingTask().getRunId().get().equals(runId)) { return Optional.of(entry.getKey()); } } return Optional.absent(); }
private boolean isTooManyInstancesForRequest(SingularityTaskRequest taskRequest, List<SingularityTaskId> activeTaskIdsForRequest) { if (taskRequest.getRequest().getRequestType() == RequestType.ON_DEMAND) { int maxActiveOnDemandTasks = taskRequest.getRequest().getInstances().or(configuration.getMaxActiveOnDemandTasksPerRequest()); if (maxActiveOnDemandTasks > 0) { int activeTasksForRequest = activeTaskIdsForRequest.size(); LOG.debug("Running {} instances for request {}. Max is {}", activeTasksForRequest, taskRequest.getRequest().getId(), maxActiveOnDemandTasks); if (activeTasksForRequest >= maxActiveOnDemandTasks) { return true; } } } return false; } }
private Optional<String> getStatusMessage(Protos.TaskStatus status, Optional<SingularityTask> task) { if (status.hasMessage() && !Strings.isNullOrEmpty(status.getMessage())) { return Optional.of(status.getMessage()); } else if (status.hasReason() && status.getReason() == Reason.REASON_CONTAINER_LIMITATION_MEMORY) { if (task.isPresent() && task.get().getTaskRequest().getDeploy().getResources().isPresent()) { if (task.get().getTaskRequest().getDeploy().getResources().get().getDiskMb() > 0) { return Optional.of(String.format("Task exceeded one or more memory limits (%s MB mem, %s MB disk).", task.get().getTaskRequest().getDeploy().getResources().get().getMemoryMb(), task.get().getTaskRequest().getDeploy().getResources().get().getDiskMb())); } else { return Optional.of(String.format("Task exceeded memory limit (%s MB mem).", task.get().getTaskRequest().getDeploy().getResources().get().getMemoryMb())); } } return Optional.of("Task exceeded memory limit."); } else if (status.hasReason() && status.getReason() == Reason.REASON_CONTAINER_LIMITATION_DISK) { if (task.isPresent() && task.get().getTaskRequest().getDeploy().getResources().isPresent()) { return Optional.of(String.format("Task exceeded disk limit (%s MB disk).", task.get().getTaskRequest().getDeploy().getResources().get().getDiskMb())); } else { return Optional.of("Task exceeded disk limit."); } } return Optional.absent(); }
public List<SingularityTaskRequest> getTaskRequests(List<SingularityPendingTask> tasks) { final Multimap<String, SingularityPendingTask> requestIdToPendingTaskId = ArrayListMultimap.create(tasks.size(), 1); for (SingularityPendingTask task : tasks) { requestIdToPendingTaskId.put(task.getPendingTaskId().getRequestId(), task); } final List<SingularityRequestWithState> matchingRequests = requestManager.getRequests(requestIdToPendingTaskId.keySet()); final Map<SingularityPendingTask, SingularityDeployKey> deployKeys = SingularityDeployKey.fromPendingTasks(requestIdToPendingTaskId.values()); final Map<SingularityDeployKey, SingularityDeploy> matchingDeploys = deployManager.getDeploysForKeys(Sets.newHashSet(deployKeys.values())); final List<SingularityTaskRequest> taskRequests = Lists.newArrayListWithCapacity(matchingRequests.size()); for (SingularityRequestWithState request : matchingRequests) { Optional<SingularityPendingDeploy> maybePendingDeploy = deployManager.getPendingDeploy(request.getRequest().getId()); for (SingularityPendingTask task : requestIdToPendingTaskId.get(request.getRequest().getId())) { SingularityDeploy foundDeploy = matchingDeploys.get(deployKeys.get(task)); if (foundDeploy == null) { LOG.warn("Couldn't find a matching deploy for pending task {}", task); continue; } if (!request.getState().isRunnable()) { LOG.warn("Request was in state {} for pending task {}", request.getState(), task); continue; } Optional<SingularityRequest> updatedRequest = maybePendingDeploy.isPresent() && maybePendingDeploy.get().getDeployMarker().getDeployId().equals(task.getPendingTaskId().getDeployId()) ? maybePendingDeploy.get().getUpdatedRequest() : Optional.<SingularityRequest>absent(); taskRequests.add(new SingularityTaskRequest(updatedRequest.or(request.getRequest()), foundDeploy, task)); } } return taskRequests; }
.setArguments(Optional.of(Collections.singletonList("wat"))) .build(); final SingularityTaskRequest taskRequest = new SingularityTaskRequest(request, deploy, pendingTask); final SingularityMesosTaskHolder task = builder.buildTask(offerHolder, Collections.singletonList(portsResource), taskRequest, taskResources, executorResources); assertTrue(task.getMesosTask().getContainer().getDocker().getParametersList().contains(envParameter)); assertEquals(String.format("/container/%s/%s", task.getTask().getTaskRequest().getDeploy().getRequestId(), task.getTask().getTaskRequest().getDeploy().getId()), task.getMesosTask().getContainer().getVolumes(1).getContainerPath()); assertEquals(String.format("/host/%s", task.getMesosTask().getTaskId().getValue()), task.getMesosTask().getContainer().getVolumes(1).getHostPath()); assertEquals(Mode.RO, task.getMesosTask().getContainer().getVolumes(1).getMode());
private void prepareEnvironment(final SingularityTaskRequest task, SingularityTaskId taskId, CommandInfo.Builder commandBuilder, final SingularityOfferHolder offerHolder, final Optional<long[]> ports) { Map<String, Object> envVars = new HashMap<>(); envVars.put("INSTANCE_NO", task.getPendingTask().getPendingTaskId().getInstanceNo()); envVars.put("TASK_HOST", offerHolder.getHostname()); envVars.put("AVAILABILITY_ZONE", offerHolder.getRackId()); envVars.put("TASK_REQUEST_ID", task.getPendingTask().getPendingTaskId().getRequestId()); envVars.put("TASK_DEPLOY_ID", taskId.getDeployId()); envVars.put("TASK_ID", taskId.getId()); envVars.put("ESTIMATED_INSTANCE_COUNT", task.getRequest().getInstancesSafe()); if (task.getPendingTask().getUser().isPresent()) { envVars.put("STARTED_BY_USER", task.getPendingTask().getUser().get()); for (Entry<String, String> envEntry : task.getDeploy().getEnv().or(Collections.<String, String>emptyMap()).entrySet()) { envVars.put(envEntry.getKey(), fillInTaskIdValues(envEntry.getValue(), offerHolder, taskId)); if (task.getDeploy().getTaskEnv().isPresent() && task.getDeploy().getTaskEnv().get().containsKey(taskId.getInstanceNo()) && !task.getDeploy().getTaskEnv().get().get(taskId.getInstanceNo()).isEmpty()) { for (Entry<String, String> envEntry : task.getDeploy().getTaskEnv().get().get(taskId.getInstanceNo()).entrySet()) { envVars.put(envEntry.getKey(), fillInTaskIdValues(envEntry.getValue(), offerHolder, taskId)); if (task.getPendingTask().getResources().isPresent()) { Resources override = task.getPendingTask().getResources().get(); for (Entry entry : task.getPendingTask().getEnvOverrides().entrySet()) { envVars.put(entry.getKey().toString(), entry.getValue());
private boolean hasHealthcheck(SingularityTask task, Optional<SingularityRequestWithState> requestWithState) { if (disasterManager.isDisabled(SingularityAction.RUN_HEALTH_CHECKS)) { return false; } if (!task.getTaskRequest().getDeploy().getHealthcheck().isPresent()) { return false; } if (task.getTaskRequest().getPendingTask().getSkipHealthchecks().or(Boolean.FALSE)) { return false; } if (requestWithState.isPresent() && requestWithState.get().getRequest().getSkipHealthchecks().or(Boolean.FALSE)) { return false; } return true; }