@Override public boolean apply(SingularityTaskId input) { return input.getDeployId().equals(deployId); } });
private String fillInTaskIdValues(String string, SingularityOfferHolder offerHolder, SingularityTaskId taskId) { if (!Strings.isNullOrEmpty(string)) { string = string.replace("${TASK_REQUEST_ID}", taskId.getRequestId()) .replace("${TASK_DEPLOY_ID}", taskId.getDeployId()) .replace("${TASK_STARTED_AT}", Long.toString(taskId.getStartedAt())) .replace("${TASK_INSTANCE_NO}", Integer.toString(taskId.getInstanceNo())) .replace("${TASK_HOST}", offerHolder.getHostname()) .replace("${TASK_RACK_ID}", offerHolder.getRackId()) .replace("${TASK_ID}", taskId.getId()); } return string; }
public static SingularityDeployKey fromTaskId(SingularityTaskId taskId) { return new SingularityDeployKey(taskId.getRequestId(), taskId.getDeployId()); }
public static String getS3KeyFormat(String s3KeyFormat, SingularityTaskId taskId, Optional<String> loggingTag, String group) { s3KeyFormat = getS3KeyFormat(s3KeyFormat, taskId.getRequestId(), taskId.getDeployId(), loggingTag, group); s3KeyFormat = s3KeyFormat.replace("%host", taskId.getSanitizedHost()); s3KeyFormat = s3KeyFormat.replace("%taskId", taskId.toString()); return s3KeyFormat; }
private String applyPlaceholders(String uri, SingularityTaskHistoryUpdate taskUpdate) { return uri .replaceAll("\\$REQUEST_ID", taskUpdate.getTaskId().getRequestId()) .replaceAll("\\$DEPLOY_ID", taskUpdate.getTaskId().getDeployId()) .replaceAll("\\$TASK_ID", taskUpdate.getTaskId().getId()); }
private List<SingularityTaskId> getMatchingTaskIds(SingularityRequest request, SingularityDeployKey deployKey) { List<SingularityTaskId> activeTaskIdsForRequest = leaderCache.getActiveTaskIdsForRequest(deployKey.getRequestId()); if (request.isLongRunning()) { Set<SingularityTaskId> killedTaskIds = leaderCache.getKilledTasks().stream() .map(SingularityKilledTaskIdRecord::getTaskId) .collect(Collectors.toSet()); List<SingularityTaskId> matchingTaskIds = new ArrayList<>(); for (SingularityTaskId taskId : activeTaskIdsForRequest) { if (!taskId.getDeployId().equals(deployKey.getDeployId())) { continue; } if (leaderCache.getCleanupTaskIds().contains(taskId)) { continue; } if (killedTaskIds.contains(taskId)) { continue; } matchingTaskIds.add(taskId); } return matchingTaskIds; } else { return new ArrayList<>(activeTaskIdsForRequest); } }
private boolean isPartOfPendingDeploy(List<SingularityPendingDeploy> pendingDeploys, SingularityTaskId taskId) { for (SingularityPendingDeploy pendingDeploy : pendingDeploys) { if (pendingDeploy.getDeployMarker().getDeployId().equals(taskId.getDeployId()) && pendingDeploy.getDeployMarker().getRequestId().equals(taskId.getRequestId())) { return true; } } return false; }
if (deployId.isPresent() && !deployId.get().equals(taskId.getDeployId())) { return false;
private void populateTaskEmailProperties(Map<String, Object> templateProperties, SingularityTaskId taskId, Collection<SingularityTaskHistoryUpdate> taskHistory, ExtendedTaskState taskState, List<SingularityTaskMetadata> taskMetadata, SingularityEmailType emailType) { Optional<SingularityTask> task = taskManager.getTask(taskId); Optional<String> directory = taskManager.getDirectory(taskId); templateProperties.put("singularityTaskLink", mailTemplateHelpers.getSingularityTaskLink(taskId.getId())); // Grab the tails of log files from remote mesos slaves. templateProperties.put("logTails", mailTemplateHelpers.getTaskLogs(taskId, task, directory)); templateProperties.put("taskId", taskId.getId()); templateProperties.put("deployId", taskId.getDeployId()); templateProperties.put("taskDirectory", directory.or("directory missing")); templateProperties.put("color", emailType.getColor()); if (task.isPresent()) { templateProperties.put("slaveHostname", task.get().getHostname()); if (task.get().getTaskRequest().getPendingTask().getCmdLineArgsList().isPresent()) { templateProperties.put("extraCmdLineArguments", task.get().getTaskRequest().getPendingTask().getCmdLineArgsList().get()); } } boolean needsBeenPrefix = taskState == ExtendedTaskState.TASK_LOST || taskState == ExtendedTaskState.TASK_KILLED; templateProperties.put("status", String.format("%s%s", needsBeenPrefix ? "has been " : "has ", taskState.getDisplayName())); templateProperties.put("taskStateLost", taskState == ExtendedTaskState.TASK_LOST); templateProperties.put("taskStateFailed", taskState == ExtendedTaskState.TASK_FAILED); templateProperties.put("taskStateFinished", taskState == ExtendedTaskState.TASK_FINISHED); templateProperties.put("taskStateKilled", taskState == ExtendedTaskState.TASK_KILLED); templateProperties.put("taskStateRunning", taskState == ExtendedTaskState.TASK_RUNNING); templateProperties.put("taskHasMetadata", !taskMetadata.isEmpty()); templateProperties.put("taskMetadata", mailTemplateHelpers.getJadeTaskMetadata(taskMetadata)); templateProperties.put("taskUpdates", mailTemplateHelpers.getJadeTaskHistory(taskHistory)); templateProperties.put("taskRan", mailTemplateHelpers.didTaskRun(taskHistory)); }
SingularityDeployKey deployKey = new SingularityDeployKey(taskId.getRequestId(), taskId.getDeployId()); Optional<SingularityPendingDeploy> pendingDeploy = Optional.fromNullable(pendingDeploys.get(deployKey)); Optional<SingularityRequestWithState> request = Optional.fromNullable(idToRequest.get(taskId.getRequestId()));
public Logger buildTaskLogger(String taskId, String executorId, String executorPid, String taskLogFile) { LOG.info("Building a task logger for {} pointing to {}", taskId, taskLogFile); LoggerContext context = new LoggerContext(); context.setName(executorPid); baseLogging.prepareRootLogger(context); String loggerId = taskId; try { SingularityTaskId singularityTaskId = SingularityTaskId.valueOf(taskId); loggerId = String.format("%s.%s.%s.%s.%s", singularityTaskId.getRequestId(), singularityTaskId.getDeployId(), singularityTaskId.getStartedAt(), singularityTaskId.getInstanceNo(), executorId); } catch (InvalidSingularityTaskIdException e) { LOG.info("Handling non-SingularityTaskId %s", taskId); } Logger taskLogger = context.getLogger(loggerId); taskLogger.detachAndStopAllAppenders(); if (baseLogging.getRootLogPath().isPresent()) { taskLogger.addAppender(baseLogging.buildFileAppender(context, baseLogging.getRootLogPath().get())); } taskLogger.addAppender(baseLogging.buildFileAppender(context, taskLogFile)); context.start(); return taskLogger; }
return request.getScheduledExpectedRuntimeMillis(); } else { final Optional<SingularityDeployStatistics> deployStatistics = deployManager.getDeployStatistics(taskId.getRequestId(), taskId.getDeployId());
private void bounce(SingularityRequestCleanup requestCleanup, final List<SingularityTaskId> activeTaskIds) { final long start = System.currentTimeMillis(); final List<SingularityTaskId> matchingTaskIds = new ArrayList<>(); for (SingularityTaskId activeTaskId : activeTaskIds) { if (activeTaskId.getRequestId().equals(requestCleanup.getRequestId()) && activeTaskId.getDeployId().equals(requestCleanup.getDeployId().get())) { matchingTaskIds.add(activeTaskId); } } for (SingularityTaskId matchingTaskId : matchingTaskIds) { LOG.debug("Adding task {} to cleanup (bounce)", matchingTaskId.getId()); Optional<SingularityTaskShellCommandRequestId> runBeforeKillId = Optional.absent(); if (requestCleanup.getRunShellCommandBeforeKill().isPresent()) { SingularityTaskShellCommandRequest shellRequest = new SingularityTaskShellCommandRequest(matchingTaskId, requestCleanup.getUser(), System.currentTimeMillis(), requestCleanup.getRunShellCommandBeforeKill().get()); taskManager.saveTaskShellCommandRequestToQueue(shellRequest); runBeforeKillId = Optional.of(shellRequest.getId()); } taskManager.createTaskCleanup(new SingularityTaskCleanup(requestCleanup.getUser(), requestCleanup.getCleanupType().getTaskCleanupType().get(), start, matchingTaskId, requestCleanup.getMessage(), requestCleanup.getActionId(), runBeforeKillId)); } if (matchingTaskIds.isEmpty() && requestCleanup.getDeployId().isPresent()) { Optional<SingularityExpiringBounce> expiringBounce = requestManager.getExpiringBounce(requestCleanup.getRequestId()); if (expiringBounce.isPresent() && expiringBounce.get().getDeployId().equalsIgnoreCase(requestCleanup.getDeployId().get())) { LOG.info("No running tasks for request {}. Marking bounce {} complete and starting new tasks", expiringBounce.get().getRequestId(), expiringBounce.get()); requestManager.removeExpiringBounce(requestCleanup.getRequestId()); } requestManager.markBounceComplete(requestCleanup.getRequestId()); } requestManager.addToPendingQueue(new SingularityPendingRequest(requestCleanup.getRequestId(), requestCleanup.getDeployId().get(), requestCleanup.getTimestamp(), requestCleanup.getUser(), PendingType.BOUNCE, Optional.absent(), Optional.absent(), requestCleanup.getSkipHealthchecks(), requestCleanup.getMessage(), requestCleanup.getActionId())); LOG.info("Added {} tasks for request {} to cleanup bounce queue in {}", matchingTaskIds.size(), requestCleanup.getRequestId(), JavaUtils.duration(start)); }
private boolean shouldHealthcheck(final SingularityTask task, final Optional<SingularityRequestWithState> request, Optional<SingularityPendingDeploy> pendingDeploy) { if (disasterManager.isDisabled(SingularityAction.RUN_HEALTH_CHECKS)) { return false; } if (!task.getTaskRequest().getRequest().isLongRunning() || !task.getTaskRequest().getDeploy().getHealthcheck().isPresent() || task.getTaskRequest().getDeploy().getHealthcheck().get().getHealthcheckResultFilePath().isPresent()) { return false; } if (task.getTaskRequest().getPendingTask().getSkipHealthchecks().or(false)) { return false; } if (pendingDeploy.isPresent() && pendingDeploy.get().getDeployMarker().getDeployId().equals(task.getTaskId().getDeployId()) && task.getTaskRequest().getDeploy().getSkipHealthchecksOnDeploy().or(false)) { return false; } if (request.isPresent() && request.get().getRequest().getSkipHealthchecks().or(false)) { return false; } Optional<SingularityTaskHealthcheckResult> lastHealthcheck = taskManager.getLastHealthcheck(task.getTaskId()); if (lastHealthcheck.isPresent() && !lastHealthcheck.get().isFailed()) { LOG.debug("Not submitting a new healthcheck for {} because it already passed a healthcheck", task.getTaskId()); return false; } return true; }
@Timed public void handleCompletedTask(Optional<SingularityTask> task, SingularityTaskId taskId, boolean wasActive, long timestamp, ExtendedTaskState state, SingularityCreateResult taskHistoryUpdateCreateResult, Protos.TaskStatus status) { final SingularityDeployStatistics deployStatistics = getDeployStatistics(taskId.getRequestId(), taskId.getDeployId()); Optional<SingularityExpiringBounce> expiringBounce = requestManager.getExpiringBounce(taskId.getRequestId()); if (expiringBounce.isPresent() && expiringBounce.get().getDeployId().equals(taskId.getDeployId())) { requestManager.removeExpiringBounce(taskId.getRequestId());
@Override public void saveTaskHistory(SingularityTaskHistory taskHistory) { if (history.getTaskHistoryForTask(taskHistory.getTask().getTaskId().getId()) != null) { if (LOG.isTraceEnabled()) { LOG.trace("saveTaskHistory -- existing taskHistory {}", taskHistory); } return; } SingularityTaskIdHistory taskIdHistory = SingularityTaskIdHistory.fromTaskIdAndTaskAndUpdates(taskHistory.getTask().getTaskId(), taskHistory.getTask(), taskHistory.getTaskUpdates()); String lastTaskStatus = null; if (taskIdHistory.getLastTaskState().isPresent()) { lastTaskStatus = taskIdHistory.getLastTaskState().get().name(); } if (LOG.isTraceEnabled()) { LOG.trace("saveTaskHistory -- will insert taskHistory {}", taskHistory); } history.insertTaskHistory(taskIdHistory.getTaskId().getRequestId(), taskIdHistory.getTaskId().getId(), taskHistoryTranscoder.toBytes(taskHistory), new Date(taskIdHistory.getUpdatedAt()), lastTaskStatus, taskHistory.getTask().getTaskRequest().getPendingTask().getRunId().orNull(), taskIdHistory.getTaskId().getDeployId(), taskIdHistory.getTaskId().getHost(), new Date(taskIdHistory.getTaskId().getStartedAt())); }
private void assertEquals(SingularityTaskId one, SingularityTaskId two) { Assert.assertEquals(one, two); Assert.assertEquals(one.getDeployId(), two.getDeployId()); Assert.assertEquals(one.getRequestId(), two.getRequestId()); Assert.assertEquals(one.getSanitizedHost(), two.getSanitizedHost()); Assert.assertEquals(one.getSanitizedRackId(), two.getSanitizedRackId()); Assert.assertEquals(one.getStartedAt(), two.getStartedAt()); Assert.assertEquals(one.getInstanceNo(), two.getInstanceNo()); }
@Test public void testRunNowOnDemandJobMayRetryOnFailure() { initRequestWithType(RequestType.ON_DEMAND, false); SingularityRequest request = requestResource.getRequest(requestId, singularityUser).getRequest(); SingularityRequest newRequest = request.toBuilder().setNumRetriesOnFailure(Optional.of(2)).build(); requestResource.postRequest(newRequest, singularityUser); initFirstDeploy(); requestResource.scheduleImmediately(singularityUser, requestId, new SingularityRunNowRequestBuilder().build()); resourceOffers(); SingularityTask task = taskManager.getActiveTasks().get(0); statusUpdate(task, TaskState.TASK_FAILED); scheduler.drainPendingQueue(); SingularityDeployStatistics deployStatistics = deployManager.getDeployStatistics(task.getTaskId().getRequestId(), task.getTaskId().getDeployId()).get(); Assert.assertEquals(MesosTaskState.TASK_FAILED, deployStatistics.getLastTaskState().get().toTaskState().get()); Assert.assertEquals(PendingType.RETRY, taskManager.getPendingTaskIds().get(0).getPendingType()); Assert.assertEquals(1, deployStatistics.getNumFailures()); Assert.assertEquals(1, deployStatistics.getNumSequentialRetries()); }
@Test public void testRunNowScheduledJobDoesNotRetry() { initScheduledRequest(); SingularityRequest request = requestResource.getRequest(requestId, singularityUser).getRequest(); SingularityRequest newRequest = request.toBuilder().setNumRetriesOnFailure(Optional.of(2)).build(); requestResource.postRequest(newRequest, singularityUser); initFirstDeploy(); requestResource.scheduleImmediately(singularityUser, requestId, new SingularityRunNowRequestBuilder().build()); resourceOffers(); SingularityTask task = taskManager.getActiveTasks().get(0); statusUpdate(task, TaskState.TASK_FAILED); scheduler.drainPendingQueue(); SingularityDeployStatistics deployStatistics = deployManager.getDeployStatistics(task.getTaskId().getRequestId(), task.getTaskId().getDeployId()).get(); Assert.assertEquals(MesosTaskState.TASK_FAILED, deployStatistics.getLastTaskState().get().toTaskState().get()); Assert.assertEquals(PendingType.TASK_DONE, taskManager.getPendingTaskIds().get(0).getPendingType()); Assert.assertEquals(1, deployStatistics.getNumFailures()); Assert.assertEquals(0, deployStatistics.getNumSequentialRetries()); Assert.assertEquals(Optional.<Long>absent(), deployStatistics.getAverageRuntimeMillis()); }
List<SingularityTaskId> cleanupTaskIds = taskManager.getCleanupTaskIds(); Assert.assertEquals(1, cleanupTaskIds.size()); Assert.assertEquals(secondDeployId, cleanupTaskIds.get(0).getDeployId());