@Override public String apply(@Nonnull SingularityTask input) { return input.getTaskRequest().getRequest().getId(); } };
public Optional<SingularityTaskId> getTaskByRunId(String requestId, String runId) { Map<SingularityTaskId, SingularityTask> activeTasks = getTasks(getActiveTaskIdsForRequest(requestId)); for (Map.Entry<SingularityTaskId, SingularityTask> entry : activeTasks.entrySet()) { if (entry.getValue().getTaskRequest().getPendingTask().getRunId().isPresent() && entry.getValue().getTaskRequest().getPendingTask().getRunId().get().equals(runId)) { return Optional.of(entry.getKey()); } } Map<SingularityTaskId, SingularityTask> inactiveTasks = getTasks(getInactiveTaskIdsForRequest(requestId)); for (Map.Entry<SingularityTaskId, SingularityTask> entry : inactiveTasks.entrySet()) { if (entry.getValue().getTaskRequest().getPendingTask().getRunId().isPresent() && entry.getValue().getTaskRequest().getPendingTask().getRunId().get().equals(runId)) { return Optional.of(entry.getKey()); } } return Optional.absent(); }
private boolean hasHealthcheck(SingularityTask task, Optional<SingularityRequestWithState> requestWithState) { if (disasterManager.isDisabled(SingularityAction.RUN_HEALTH_CHECKS)) { return false; } if (!task.getTaskRequest().getDeploy().getHealthcheck().isPresent()) { return false; } if (task.getTaskRequest().getPendingTask().getSkipHealthchecks().or(Boolean.FALSE)) { return false; } if (requestWithState.isPresent() && requestWithState.get().getRequest().getSkipHealthchecks().or(Boolean.FALSE)) { return false; } return true; }
public SingularityHealthcheckAsyncHandler(SingularityExceptionNotifier exceptionNotifier, SingularityConfiguration configuration, SingularityHealthchecker healthchecker, SingularityNewTaskChecker newTaskChecker, TaskManager taskManager, SingularityTask task) { this.exceptionNotifier = exceptionNotifier; this.taskManager = taskManager; this.newTaskChecker = newTaskChecker; this.healthchecker = healthchecker; this.task = task; this.maxHealthcheckResponseBodyBytes = configuration.getMaxHealthcheckResponseBodyBytes(); this.failureStatusCodes = task.getTaskRequest().getDeploy().getHealthcheck().isPresent() ? task.getTaskRequest().getDeploy().getHealthcheck().get().getFailureStatusCodes().or(configuration.getHealthcheckFailureStatusCodes()) : configuration.getHealthcheckFailureStatusCodes(); startTime = System.currentTimeMillis(); }
private int getDelaySeconds(SingularityTask task, Optional<SingularityRequestWithState> requestWithState) { int delaySeconds = configuration.getNewTaskCheckerBaseDelaySeconds(); if (hasHealthcheck(task, requestWithState)) { Optional<Integer> maybeStartupDelay = task.getTaskRequest().getDeploy().getHealthcheck().get().getStartupDelaySeconds().or(configuration.getStartupDelaySeconds()); if (maybeStartupDelay.isPresent()) { return maybeStartupDelay.get(); } } else if (task.getTaskRequest().getRequest().isLoadBalanced()) { return delaySeconds; } delaySeconds += task.getTaskRequest().getDeploy().getDeployHealthTimeoutSeconds().or(configuration.getDeployHealthyBySeconds()); return delaySeconds; }
private Optional<String> getHealthcheckUri(SingularityTask task) { if (!task.getTaskRequest().getDeploy().getHealthcheck().isPresent()) { return Optional.absent(); } HealthcheckOptions options = task.getTaskRequest().getDeploy().getHealthcheck().get(); final String hostname = task.getHostname(); Optional<Long> healthcheckPort = options.getPortNumber().or(MesosUtils.getPortByIndex(mesosProtosUtils.toResourceList(task.getMesosTask().getResources()), options.getPortIndex().or(0))); if (!healthcheckPort.isPresent() || healthcheckPort.get() < 1L) { return Optional.absent(); } if (!task.getTaskRequest().getDeploy().getHealthcheck().get().getUri().isPresent()) { return Optional.absent(); } String uri = task.getTaskRequest().getDeploy().getHealthcheck().get().getUri().get(); if (uri.startsWith("/")) { uri = uri.substring(1); } HealthcheckProtocol protocol = options.getProtocol().or(DEFAULT_HEALTH_CHECK_SCHEME); return Optional.of(String.format("%s://%s:%d/%s", protocol.getProtocol(), hostname, healthcheckPort.get(), uri)); }
public static SingularityTaskIdHistory fromTaskIdAndTaskAndUpdates(SingularityTaskId taskId, SingularityTask task, List<SingularityTaskHistoryUpdate> updates) { ExtendedTaskState lastTaskState = null; long updatedAt = taskId.getStartedAt(); if (updates != null && !updates.isEmpty()) { SingularityTaskHistoryUpdate lastUpdate = Collections.max(updates); lastTaskState = lastUpdate.getTaskState(); updatedAt = lastUpdate.getTimestamp(); } return new SingularityTaskIdHistory(taskId, updatedAt, Optional.fromNullable(lastTaskState), task.getTaskRequest().getPendingTask().getRunId()); }
private void prepareTaskMail(Optional<SingularityTask> task, SingularityTaskId taskId, SingularityRequest request, SingularityEmailType emailType, Map<String, Object> extraProperties, Collection<SingularityTaskHistoryUpdate> taskHistory, ExtendedTaskState taskState, List<SingularityTaskMetadata> taskMetadata) { final Collection<SingularityEmailDestination> emailDestination = getDestination(request, emailType); final Map<String, Object> templateProperties = Maps.newHashMap(); populateRequestEmailProperties(templateProperties, request, emailType); populateTaskEmailProperties(templateProperties, taskId, taskHistory, taskState, taskMetadata, emailType); templateProperties.putAll(extraProperties); final String subject = mailTemplateHelpers.getSubjectForTaskHistory(taskId, taskState, emailType, taskHistory); final String adminEmails = adminJoiner.join(smtpConfiguration.getAdmins()); templateProperties.put("adminEmails", adminEmails); final String body = Jade4J.render(taskTemplate, templateProperties); final Optional<String> user = task.isPresent() ? task.get().getTaskRequest().getPendingTask().getUser() : Optional.<String> absent(); queueMail(emailDestination, request, emailType, user, subject, body); }
private boolean shouldCheckHealthchecks(final SingularityRequest request, final Optional<SingularityDeploy> deploy, final Collection<SingularityTaskId> activeTasks, final boolean isDeployPending) { if (disasterManager.isDisabled(SingularityAction.RUN_HEALTH_CHECKS)) { return false; } if (!deploy.isPresent()) { return false; } if (!deploy.get().getHealthcheck().isPresent()) { return false; } if (isDeployPending && deploy.get().getSkipHealthchecksOnDeploy().or(false)) { return false; } if (request.getSkipHealthchecks().or(Boolean.FALSE)) { return false; } for (SingularityTask task : taskManager.getTasks(activeTasks).values()) { if (task.getTaskRequest().getPendingTask().getSkipHealthchecks().or(Boolean.FALSE)) { return false; } } return true; }
private boolean shouldHealthcheck(final SingularityTask task, final Optional<SingularityRequestWithState> request, Optional<SingularityPendingDeploy> pendingDeploy) { if (disasterManager.isDisabled(SingularityAction.RUN_HEALTH_CHECKS)) { return false; } if (!task.getTaskRequest().getRequest().isLongRunning() || !task.getTaskRequest().getDeploy().getHealthcheck().isPresent() || task.getTaskRequest().getDeploy().getHealthcheck().get().getHealthcheckResultFilePath().isPresent()) { return false; } if (task.getTaskRequest().getPendingTask().getSkipHealthchecks().or(false)) { return false; } if (pendingDeploy.isPresent() && pendingDeploy.get().getDeployMarker().getDeployId().equals(task.getTaskId().getDeployId()) && task.getTaskRequest().getDeploy().getSkipHealthchecksOnDeploy().or(false)) { return false; } if (request.isPresent() && request.get().getRequest().getSkipHealthchecks().or(false)) { return false; } Optional<SingularityTaskHealthcheckResult> lastHealthcheck = taskManager.getLastHealthcheck(task.getTaskId()); if (lastHealthcheck.isPresent() && !lastHealthcheck.get().isFailed()) { LOG.debug("Not submitting a new healthcheck for {} because it already passed a healthcheck", task.getTaskId()); return false; } return true; }
private void cleanupTaskDueToDecomission(final Map<String, Optional<String>> requestIdsToUserToReschedule, final Set<SingularityTaskId> matchingTaskIds, SingularityTask task, SingularityMachineAbstraction<?> decommissioningObject) { requestIdsToUserToReschedule.put(task.getTaskRequest().getRequest().getId(), decommissioningObject.getCurrentState().getUser()); matchingTaskIds.add(task.getTaskId()); LOG.trace("Scheduling a cleanup task for {} due to decomissioning {}", task.getTaskId(), decommissioningObject); taskManager.createTaskCleanup(new SingularityTaskCleanup(decommissioningObject.getCurrentState().getUser(), TaskCleanupType.DECOMISSIONING, System.currentTimeMillis(), task.getTaskId(), Optional.of(String.format("%s %s is decomissioning", decommissioningObject.getTypeName(), decommissioningObject.getName())), Optional.<String>absent(), Optional.<SingularityTaskShellCommandRequestId>absent())); }
public static SingularityTaskState fromTaskHistory(SingularityTaskHistory taskHistory) { return new SingularityTaskState( Optional.of(taskHistory.getTask().getTaskId()), Optional.of(taskHistory.getTask().getTaskRequest().getPendingTask().getPendingTaskId()), taskHistory.getTask().getTaskRequest().getPendingTask().getRunId(), Optional.of(taskHistory.getLastTaskUpdate().get().getTaskState()), taskHistory.getTaskUpdates(), false ); }
private Optional<String> getRequestGroupForTask(final SingularityTaskId taskId, SingularityUser user) { Optional<SingularityTaskHistory> maybeTaskHistory = getTaskHistory(taskId, user); if (maybeTaskHistory.isPresent()) { SingularityRequest request = maybeTaskHistory.get().getTask().getTaskRequest().getRequest(); authorizationHelper.checkForAuthorization(request, user, SingularityAuthorizationScope.READ); return request.getGroup(); } else { return getRequestGroup(taskId.getRequestId(), user); } }
SingularityTask getSizeOptimizedTask(SingularityMesosTaskHolder taskHolder) { if (configuration.isStoreAllMesosTaskInfoForDebugging()) { return taskHolder.getTask(); } SingularityTask task = taskHolder.getTask(); TaskInfo.Builder mesosTask = taskHolder.getMesosTask().toBuilder(); mesosTask.clearData(); List<MesosOfferObject> offers = task.getOffers() .stream() .map(MesosOfferObject::sizeOptimized) .collect(Collectors.toList()); SingularityTaskRequest taskRequest = task.getTaskRequest(); if (task.getTaskRequest().getDeploy().getExecutorData().isPresent()) { SingularityDeployBuilder deploy = task.getTaskRequest().getDeploy().toBuilder(); deploy.setExecutorData(Optional.absent()); taskRequest = new SingularityTaskRequest(task.getTaskRequest().getRequest(), deploy.build(), task.getTaskRequest().getPendingTask()); } return new SingularityTask(taskRequest, task.getTaskId(), offers, mesosProtosUtils.taskFromProtos(mesosTask.build()), task.getRackId()); }
private List<UpstreamInfo> tasksToUpstreams(List<SingularityTask> tasks, String requestId, Optional<String> loadBalancerUpstreamGroup) { final List<UpstreamInfo> upstreams = Lists.newArrayListWithCapacity(tasks.size()); for (SingularityTask task : tasks) { final Optional<Long> maybeLoadBalancerPort = MesosUtils.getPortByIndex(mesosProtosUtils.toResourceList(task.getMesosTask().getResources()), task.getTaskRequest().getDeploy().getLoadBalancerPortIndex().or(0)); if (maybeLoadBalancerPort.isPresent()) { String upstream = String.format("%s:%d", task.getHostname(), maybeLoadBalancerPort.get()); Optional<String> group = loadBalancerUpstreamGroup; if (taskLabelForLoadBalancerUpstreamGroup.isPresent()) { for (MesosParameter label : task.getMesosTask().getLabels().getLabels()) { if (label.hasKey() && label.getKey().equals(taskLabelForLoadBalancerUpstreamGroup.get()) && label.hasValue()) { group = Optional.of(label.getValue()); break; } } } upstreams.add(new UpstreamInfo(upstream, Optional.of(requestId), task.getRackId(), Optional.<String>absent(), group)); } else { LOG.warn("Task {} is missing port but is being passed to LB ({})", task.getTaskId(), task); } } return upstreams; }
private boolean shouldRetryImmediately(SingularityRequest request, SingularityDeployStatistics deployStatistics, Optional<SingularityTask> task) { if (!request.getNumRetriesOnFailure().isPresent()) { return false; } if (task.isPresent() && task.get().getTaskRequest().getPendingTask().getPendingTaskId().getPendingType() == PendingType.IMMEDIATE && request.getRequestType() == RequestType.SCHEDULED) { return false; // don't retry UI triggered scheduled jobs (UI triggered on-demand jobs are okay to retry though) } final int numRetriesInARow = deployStatistics.getNumSequentialRetries(); if (numRetriesInARow >= request.getNumRetriesOnFailure().get()) { LOG.debug("Request {} had {} retries in a row, not retrying again (num retries on failure: {})", request.getId(), numRetriesInARow, request.getNumRetriesOnFailure()); return false; } LOG.debug("Request {} had {} retries in a row - retrying again (num retries on failure: {})", request.getId(), numRetriesInARow, request.getNumRetriesOnFailure()); return true; }
private void createTaskAndDeletePendingTaskPrivate(SingularityTask task) throws Exception { // TODO: Should more of the below be done within a transaction? deletePendingTask(task.getTaskRequest().getPendingTask().getPendingTaskId()); final long now = System.currentTimeMillis(); String msg = String.format("Task launched because of %s", task.getTaskRequest().getPendingTask().getPendingTaskId().getPendingType().name()); if (task.getTaskRequest().getPendingTask().getUser().isPresent()) { msg = String.format("%s by %s", msg, task.getTaskRequest().getPendingTask().getUser().get()); } if (task.getTaskRequest().getPendingTask().getMessage().isPresent()) { msg = String.format("%s (%s)", msg, task.getTaskRequest().getPendingTask().getMessage().get()); } saveTaskHistoryUpdate(new SingularityTaskHistoryUpdate(task.getTaskId(), now, ExtendedTaskState.TASK_LAUNCHED, Optional.of(msg), Optional.<String>absent())); saveLastActiveTaskStatus(new SingularityTaskStatusHolder(task.getTaskId(), Optional.absent(), now, serverId, Optional.of(task.getAgentId().getValue()))); try { final String path = getTaskPath(task.getTaskId()); CuratorTransactionFinal transaction = curator.inTransaction().create().forPath(path, taskTranscoder.toBytes(task)).and(); transaction.create().forPath(getActivePath(task.getTaskId().getId())).and().commit(); leaderCache.putActiveTask(task); taskCache.set(path, task); } catch (KeeperException.NodeExistsException nee) { LOG.error("Task or active path already existed for {}", task.getTaskId()); } }
@Override public void saveTaskHistory(SingularityTaskHistory taskHistory) { if (history.getTaskHistoryForTask(taskHistory.getTask().getTaskId().getId()) != null) { if (LOG.isTraceEnabled()) { LOG.trace("saveTaskHistory -- existing taskHistory {}", taskHistory); } return; } SingularityTaskIdHistory taskIdHistory = SingularityTaskIdHistory.fromTaskIdAndTaskAndUpdates(taskHistory.getTask().getTaskId(), taskHistory.getTask(), taskHistory.getTaskUpdates()); String lastTaskStatus = null; if (taskIdHistory.getLastTaskState().isPresent()) { lastTaskStatus = taskIdHistory.getLastTaskState().get().name(); } if (LOG.isTraceEnabled()) { LOG.trace("saveTaskHistory -- will insert taskHistory {}", taskHistory); } history.insertTaskHistory(taskIdHistory.getTaskId().getRequestId(), taskIdHistory.getTaskId().getId(), taskHistoryTranscoder.toBytes(taskHistory), new Date(taskIdHistory.getUpdatedAt()), lastTaskStatus, taskHistory.getTask().getTaskRequest().getPendingTask().getRunId().orNull(), taskIdHistory.getTaskId().getDeployId(), taskIdHistory.getTaskId().getHost(), new Date(taskIdHistory.getTaskId().getStartedAt())); }
private void relaunchTask(SingularityTask task) { SingularityPendingTask pendingTask = task.getTaskRequest().getPendingTask(); SingularityPendingRequest pendingRequest = new SingularityPendingRequestBuilder() .setRequestId(task.getTaskRequest().getRequest().getId()) .setDeployId(task.getTaskRequest().getDeploy().getId()) .setPendingType(PendingType.RETRY) .setUser(pendingTask.getUser()) .setRunId(pendingTask.getRunId()) .setCmdLineArgsList(pendingTask.getCmdLineArgsList()) .setSkipHealthchecks(pendingTask.getSkipHealthchecks()) .setMessage(pendingTask.getMessage()) .setResources(pendingTask.getResources()) .setS3UploaderAdditionalFiles(pendingTask.getS3UploaderAdditionalFiles()) .setRunAsUserOverride(pendingTask.getRunAsUserOverride()) .setEnvOverrides(pendingTask.getEnvOverrides()) .setExtraArtifacts(pendingTask.getExtraArtifacts()) .setActionId(pendingTask.getActionId()) .setRunAt(pendingTask.getPendingTaskId().getNextRunAt()) .setTimestamp(System.currentTimeMillis()) .build(); requestManager.addToPendingQueue(pendingRequest); }
@Test public void testRunId() { initScheduledRequest(); initFirstDeploy(); String runId = "my-run-id"; SingularityPendingRequestParent parent = requestResource.scheduleImmediately(singularityUser, requestId, new SingularityRunNowRequestBuilder().setRunId(runId).build()); Assert.assertEquals(runId, parent.getPendingRequest().getRunId().get()); resourceOffers(); Assert.assertEquals(runId, taskManager.getActiveTasks().get(0).getTaskRequest().getPendingTask().getRunId().get()); SingularityTaskId taskId = taskManager.getActiveTaskIds().get(0); statusUpdate(taskManager.getTask(taskId).get(), TaskState.TASK_FINISHED); configuration.setTaskPersistAfterStartupBufferMillis(0); taskMetadataConfiguration.setTaskPersistAfterFinishBufferMillis(0); taskHistoryPersister.runActionOnPoll(); Assert.assertEquals(runId, historyManager.getTaskHistory(taskId.getId()).get().getTask().getTaskRequest().getPendingTask().getRunId().get()); Assert.assertEquals(runId, getTaskHistoryForRequest(requestId, 0, 10).get(0).getRunId().get()); parent = requestResource.scheduleImmediately(singularityUser, requestId, ((SingularityRunNowRequest) null)); Assert.assertTrue(parent.getPendingRequest().getRunId().isPresent()); }