@Override public int compareTo(SingularityTaskRequest o) { return this.getPendingTask().getPendingTaskId().compareTo(o.getPendingTask().getPendingTaskId()); }
public Optional<SingularityTaskId> getTaskByRunId(String requestId, String runId) { Map<SingularityTaskId, SingularityTask> activeTasks = getTasks(getActiveTaskIdsForRequest(requestId)); for (Map.Entry<SingularityTaskId, SingularityTask> entry : activeTasks.entrySet()) { if (entry.getValue().getTaskRequest().getPendingTask().getRunId().isPresent() && entry.getValue().getTaskRequest().getPendingTask().getRunId().get().equals(runId)) { return Optional.of(entry.getKey()); } } Map<SingularityTaskId, SingularityTask> inactiveTasks = getTasks(getInactiveTaskIdsForRequest(requestId)); for (Map.Entry<SingularityTaskId, SingularityTask> entry : inactiveTasks.entrySet()) { if (entry.getValue().getTaskRequest().getPendingTask().getRunId().isPresent() && entry.getValue().getTaskRequest().getPendingTask().getRunId().get().equals(runId)) { return Optional.of(entry.getKey()); } } return Optional.absent(); }
private Optional<Integer> getCpuHardLimit(SingularityTaskRequest task) { if (configuration.getCpuHardLimit().isPresent()) { Optional<Resources> maybeResources = task.getPendingTask().getResources().or(task.getDeploy().getResources()); if (maybeResources.isPresent()) { double requestedCpus = maybeResources.get().getCpus(); int scaledLimit = (int) Math.ceil(requestedCpus * configuration.getCpuHardLimitScaleFactor()); return Optional.of(Math.max(scaledLimit, configuration.getCpuHardLimit().get())); } } return Optional.absent(); }
private List<SingularityTaskRequestHolder> getSortedDueTaskRequests() { final List<SingularityTaskRequest> taskRequests = taskPrioritizer.getSortedDueTasks(scheduler.getDueTasks()); taskRequests.forEach((taskRequest) -> LOG.trace("Task {} is due", taskRequest.getPendingTask().getPendingTaskId())); taskPrioritizer.removeTasksAffectedByPriorityFreeze(taskRequests); return taskRequests.stream() .map((taskRequest) -> new SingularityTaskRequestHolder(taskRequest, defaultResources, defaultCustomExecutorResources)) .collect(Collectors.toList()); }
public void removeTasksAffectedByPriorityFreeze(List<SingularityTaskRequest> taskRequests) { final Optional<SingularityPriorityFreezeParent> maybePriorityFreeze = priorityManager.getActivePriorityFreeze(); if (maybePriorityFreeze.isPresent()) { final ListIterator<SingularityTaskRequest> iterator = taskRequests.listIterator(); while (iterator.hasNext()) { final SingularityTaskRequest taskRequest = iterator.next(); final double taskPriorityLevel = priorityManager.getTaskPriorityLevelForRequest(taskRequest.getRequest()); if (taskPriorityLevel < maybePriorityFreeze.get().getPriorityFreeze().getMinimumPriorityLevel()) { LOG.trace("Skipping scheduled task {} because taskPriorityLevel ({}) is less than active priority freeze ({})", taskRequest.getPendingTask().getPendingTaskId(), taskPriorityLevel, maybePriorityFreeze.get().getPriorityFreeze().getMinimumPriorityLevel()); iterator.remove(); } } } }
public static SingularityTaskIdHistory fromTaskIdAndTaskAndUpdates(SingularityTaskId taskId, SingularityTask task, List<SingularityTaskHistoryUpdate> updates) { ExtendedTaskState lastTaskState = null; long updatedAt = taskId.getStartedAt(); if (updates != null && !updates.isEmpty()) { SingularityTaskHistoryUpdate lastUpdate = Collections.max(updates); lastTaskState = lastUpdate.getTaskState(); updatedAt = lastUpdate.getTimestamp(); } return new SingularityTaskIdHistory(taskId, updatedAt, Optional.fromNullable(lastTaskState), task.getTaskRequest().getPendingTask().getRunId()); }
private double getWeightedPriority(SingularityTaskRequest taskRequest, long now) { Long overdueMillis = Math.max(now - taskRequest.getPendingTask().getPendingTaskId().getNextRunAt(), 1); Double requestPriority = priorityManager.getTaskPriorityLevelForRequest(taskRequest.getRequest()); return overdueMillis * Math.pow(requestPriority, configuration.getSchedulerPriorityWeightFactor()); }
private void prepareTaskMail(Optional<SingularityTask> task, SingularityTaskId taskId, SingularityRequest request, SingularityEmailType emailType, Map<String, Object> extraProperties, Collection<SingularityTaskHistoryUpdate> taskHistory, ExtendedTaskState taskState, List<SingularityTaskMetadata> taskMetadata) { final Collection<SingularityEmailDestination> emailDestination = getDestination(request, emailType); final Map<String, Object> templateProperties = Maps.newHashMap(); populateRequestEmailProperties(templateProperties, request, emailType); populateTaskEmailProperties(templateProperties, taskId, taskHistory, taskState, taskMetadata, emailType); templateProperties.putAll(extraProperties); final String subject = mailTemplateHelpers.getSubjectForTaskHistory(taskId, taskState, emailType, taskHistory); final String adminEmails = adminJoiner.join(smtpConfiguration.getAdmins()); templateProperties.put("adminEmails", adminEmails); final String body = Jade4J.render(taskTemplate, templateProperties); final Optional<String> user = task.isPresent() ? task.get().getTaskRequest().getPendingTask().getUser() : Optional.<String> absent(); queueMail(emailDestination, request, emailType, user, subject, body); }
private double calculateScore(SingularityOfferHolder offerHolder, Map<String, SingularitySlaveUsageWithCalculatedScores> currentSlaveUsagesBySlaveId, Map<String, Integer> tasksPerOffer, SingularityTaskRequestHolder taskRequestHolder, List<SingularityTaskId> activeTaskIdsForRequest, RequestUtilization requestUtilization) { Optional<SingularitySlaveUsageWithCalculatedScores> maybeSlaveUsage = Optional.fromNullable(currentSlaveUsagesBySlaveId.get(offerHolder.getSlaveId())); double score = score(offerHolder, tasksPerOffer, taskRequestHolder, maybeSlaveUsage, activeTaskIdsForRequest, requestUtilization); if (LOG.isTraceEnabled()) { LOG.trace("Scored {} | Task {} | Offer - mem {} - cpu {} | Slave {} | maybeSlaveUsage - {}", score, taskRequestHolder.getTaskRequest().getPendingTask().getPendingTaskId().getId(), MesosUtils.getMemory(offerHolder.getCurrentResources(), Optional.absent()), MesosUtils.getNumCpus(offerHolder.getCurrentResources(), Optional.absent()), offerHolder.getHostname(), maybeSlaveUsage); } return score; }
private boolean hasHealthcheck(SingularityTask task, Optional<SingularityRequestWithState> requestWithState) { if (disasterManager.isDisabled(SingularityAction.RUN_HEALTH_CHECKS)) { return false; } if (!task.getTaskRequest().getDeploy().getHealthcheck().isPresent()) { return false; } if (task.getTaskRequest().getPendingTask().getSkipHealthchecks().or(Boolean.FALSE)) { return false; } if (requestWithState.isPresent() && requestWithState.get().getRequest().getSkipHealthchecks().or(Boolean.FALSE)) { return false; } return true; }
public SingularityTaskRequestHolder(SingularityTaskRequest taskRequest, Resources defaultResources, Resources defaultCustomExecutorResources) { this.taskRequest = taskRequest; this.executorResources = taskRequest.getDeploy().getCustomExecutorCmd().isPresent() ? taskRequest.getDeploy().getCustomExecutorResources().or(defaultCustomExecutorResources) : Resources.EMPTY_RESOURCES;; this.taskResources = taskRequest.getPendingTask().getResources().or(taskRequest.getDeploy().getResources()).or(defaultResources); this.totalResources = Resources.add(taskResources, executorResources); this.requestedPorts = new ArrayList<>(); if (taskRequest.getDeploy().getContainerInfo().isPresent() && taskRequest.getDeploy().getContainerInfo().get().getDocker().isPresent()) { requestedPorts.addAll(taskRequest.getDeploy().getContainerInfo().get().getDocker().get().getLiteralHostPorts()); } }
private boolean healthchecksSkipped(SingularityTaskId taskId, List<SingularityRequestHistory> requestHistories, SingularityDeploy deploy) { if (deploy.getSkipHealthchecksOnDeploy().or(false)) { return true; } Optional<SingularityTask> maybeTask = taskManager.getTask(taskId); if (maybeTask.isPresent()) { if (maybeTask.get().getTaskRequest().getPendingTask().getSkipHealthchecks().or(false)) { return true; } Optional<Long> runningStartTime = getRunningAt(taskManager.getTaskHistoryUpdates(taskId)); if (runningStartTime.isPresent()) { Optional<SingularityRequestHistory> previousHistory = Optional.absent(); for (SingularityRequestHistory history : requestHistories) { if (history.getCreatedAt() < runningStartTime.get() && (!previousHistory.isPresent() || previousHistory.get().getCreatedAt() < history.getCreatedAt())) { previousHistory = Optional.of(history); } } if (previousHistory.isPresent() && previousHistory.get().getRequest().getSkipHealthchecks().or(false)) { return true; } } } return false; }
public static SingularityTaskState fromTaskHistory(SingularityTaskHistory taskHistory) { return new SingularityTaskState( Optional.of(taskHistory.getTask().getTaskId()), Optional.of(taskHistory.getTask().getTaskRequest().getPendingTask().getPendingTaskId()), taskHistory.getTask().getTaskRequest().getPendingTask().getRunId(), Optional.of(taskHistory.getLastTaskUpdate().get().getTaskState()), taskHistory.getTaskUpdates(), false ); }
private boolean shouldRetryImmediately(SingularityRequest request, SingularityDeployStatistics deployStatistics, Optional<SingularityTask> task) { if (!request.getNumRetriesOnFailure().isPresent()) { return false; } if (task.isPresent() && task.get().getTaskRequest().getPendingTask().getPendingTaskId().getPendingType() == PendingType.IMMEDIATE && request.getRequestType() == RequestType.SCHEDULED) { return false; // don't retry UI triggered scheduled jobs (UI triggered on-demand jobs are okay to retry though) } final int numRetriesInARow = deployStatistics.getNumSequentialRetries(); if (numRetriesInARow >= request.getNumRetriesOnFailure().get()) { LOG.debug("Request {} had {} retries in a row, not retrying again (num retries on failure: {})", request.getId(), numRetriesInARow, request.getNumRetriesOnFailure()); return false; } LOG.debug("Request {} had {} retries in a row - retrying again (num retries on failure: {})", request.getId(), numRetriesInARow, request.getNumRetriesOnFailure()); return true; }
@Before public void setup() { Mockito.when(taskRequest.getRequest()).thenReturn(request); Mockito.when(request.getId()).thenReturn("requestId"); Mockito.when(taskRequest.getDeploy()).thenReturn(deploy); Mockito.when(deploy.getId()).thenReturn("deployId"); Mockito.when(taskRequest.getPendingTask()).thenReturn(task); Mockito.when(task.getPendingTaskId()).thenReturn(taskId); }
private void createTaskAndDeletePendingTaskPrivate(SingularityTask task) throws Exception { // TODO: Should more of the below be done within a transaction? deletePendingTask(task.getTaskRequest().getPendingTask().getPendingTaskId()); final long now = System.currentTimeMillis(); String msg = String.format("Task launched because of %s", task.getTaskRequest().getPendingTask().getPendingTaskId().getPendingType().name()); if (task.getTaskRequest().getPendingTask().getUser().isPresent()) { msg = String.format("%s by %s", msg, task.getTaskRequest().getPendingTask().getUser().get()); } if (task.getTaskRequest().getPendingTask().getMessage().isPresent()) { msg = String.format("%s (%s)", msg, task.getTaskRequest().getPendingTask().getMessage().get()); } saveTaskHistoryUpdate(new SingularityTaskHistoryUpdate(task.getTaskId(), now, ExtendedTaskState.TASK_LAUNCHED, Optional.of(msg), Optional.<String>absent())); saveLastActiveTaskStatus(new SingularityTaskStatusHolder(task.getTaskId(), Optional.absent(), now, serverId, Optional.of(task.getAgentId().getValue()))); try { final String path = getTaskPath(task.getTaskId()); CuratorTransactionFinal transaction = curator.inTransaction().create().forPath(path, taskTranscoder.toBytes(task)).and(); transaction.create().forPath(getActivePath(task.getTaskId().getId())).and().commit(); leaderCache.putActiveTask(task); taskCache.set(path, task); } catch (KeeperException.NodeExistsException nee) { LOG.error("Task or active path already existed for {}", task.getTaskId()); } }
@Override public void saveTaskHistory(SingularityTaskHistory taskHistory) { if (history.getTaskHistoryForTask(taskHistory.getTask().getTaskId().getId()) != null) { if (LOG.isTraceEnabled()) { LOG.trace("saveTaskHistory -- existing taskHistory {}", taskHistory); } return; } SingularityTaskIdHistory taskIdHistory = SingularityTaskIdHistory.fromTaskIdAndTaskAndUpdates(taskHistory.getTask().getTaskId(), taskHistory.getTask(), taskHistory.getTaskUpdates()); String lastTaskStatus = null; if (taskIdHistory.getLastTaskState().isPresent()) { lastTaskStatus = taskIdHistory.getLastTaskState().get().name(); } if (LOG.isTraceEnabled()) { LOG.trace("saveTaskHistory -- will insert taskHistory {}", taskHistory); } history.insertTaskHistory(taskIdHistory.getTaskId().getRequestId(), taskIdHistory.getTaskId().getId(), taskHistoryTranscoder.toBytes(taskHistory), new Date(taskIdHistory.getUpdatedAt()), lastTaskStatus, taskHistory.getTask().getTaskRequest().getPendingTask().getRunId().orNull(), taskIdHistory.getTaskId().getDeployId(), taskIdHistory.getTaskId().getHost(), new Date(taskIdHistory.getTaskId().getStartedAt())); }
SingularityTask getSizeOptimizedTask(SingularityMesosTaskHolder taskHolder) { if (configuration.isStoreAllMesosTaskInfoForDebugging()) { return taskHolder.getTask(); } SingularityTask task = taskHolder.getTask(); TaskInfo.Builder mesosTask = taskHolder.getMesosTask().toBuilder(); mesosTask.clearData(); List<MesosOfferObject> offers = task.getOffers() .stream() .map(MesosOfferObject::sizeOptimized) .collect(Collectors.toList()); SingularityTaskRequest taskRequest = task.getTaskRequest(); if (task.getTaskRequest().getDeploy().getExecutorData().isPresent()) { SingularityDeployBuilder deploy = task.getTaskRequest().getDeploy().toBuilder(); deploy.setExecutorData(Optional.absent()); taskRequest = new SingularityTaskRequest(task.getTaskRequest().getRequest(), deploy.build(), task.getTaskRequest().getPendingTask()); } return new SingularityTask(taskRequest, task.getTaskId(), offers, mesosProtosUtils.taskFromProtos(mesosTask.build()), task.getRackId()); }
@Test public void testRunId() { initScheduledRequest(); initFirstDeploy(); String runId = "my-run-id"; SingularityPendingRequestParent parent = requestResource.scheduleImmediately(singularityUser, requestId, new SingularityRunNowRequestBuilder().setRunId(runId).build()); Assert.assertEquals(runId, parent.getPendingRequest().getRunId().get()); resourceOffers(); Assert.assertEquals(runId, taskManager.getActiveTasks().get(0).getTaskRequest().getPendingTask().getRunId().get()); SingularityTaskId taskId = taskManager.getActiveTaskIds().get(0); statusUpdate(taskManager.getTask(taskId).get(), TaskState.TASK_FINISHED); configuration.setTaskPersistAfterStartupBufferMillis(0); taskMetadataConfiguration.setTaskPersistAfterFinishBufferMillis(0); taskHistoryPersister.runActionOnPoll(); Assert.assertEquals(runId, historyManager.getTaskHistory(taskId.getId()).get().getTask().getTaskRequest().getPendingTask().getRunId().get()); Assert.assertEquals(runId, getTaskHistoryForRequest(requestId, 0, 10).get(0).getRunId().get()); parent = requestResource.scheduleImmediately(singularityUser, requestId, ((SingularityRunNowRequest) null)); Assert.assertTrue(parent.getPendingRequest().getRunId().isPresent()); }
private void relaunchTask(SingularityTask task) { SingularityPendingTask pendingTask = task.getTaskRequest().getPendingTask(); SingularityPendingRequest pendingRequest = new SingularityPendingRequestBuilder() .setRequestId(task.getTaskRequest().getRequest().getId()) .setDeployId(task.getTaskRequest().getDeploy().getId()) .setPendingType(PendingType.RETRY) .setUser(pendingTask.getUser()) .setRunId(pendingTask.getRunId()) .setCmdLineArgsList(pendingTask.getCmdLineArgsList()) .setSkipHealthchecks(pendingTask.getSkipHealthchecks()) .setMessage(pendingTask.getMessage()) .setResources(pendingTask.getResources()) .setS3UploaderAdditionalFiles(pendingTask.getS3UploaderAdditionalFiles()) .setRunAsUserOverride(pendingTask.getRunAsUserOverride()) .setEnvOverrides(pendingTask.getEnvOverrides()) .setExtraArtifacts(pendingTask.getExtraArtifacts()) .setActionId(pendingTask.getActionId()) .setRunAt(pendingTask.getPendingTaskId().getNextRunAt()) .setTimestamp(System.currentTimeMillis()) .build(); requestManager.addToPendingQueue(pendingRequest); }