@Override public boolean apply(@Nonnull SingularityTaskHistoryUpdate input) { return input.getTaskState() == taskState; } });
public static SimplifiedTaskState getCurrentState(Iterable<SingularityTaskHistoryUpdate> updates) { SimplifiedTaskState state = SimplifiedTaskState.UNKNOWN; if (updates == null) { return state; } for (SingularityTaskHistoryUpdate update : updates) { if (update.getTaskState().isDone()) { return SimplifiedTaskState.DONE; } else if (update.getTaskState() == ExtendedTaskState.TASK_RUNNING) { state = SimplifiedTaskState.RUNNING; } else if (state == SimplifiedTaskState.UNKNOWN) { state = SimplifiedTaskState.WAITING; } } return state; }
public void cacheTaskHistoryUpdates(Map<SingularityTaskId, List<SingularityTaskHistoryUpdate>> historyUpdates) { this.historyUpdates = new ConcurrentHashMap<>(historyUpdates.size()); historyUpdates.entrySet().stream().forEach((e) -> this.historyUpdates.put( e.getKey(), e.getValue().stream() .collect(Collectors.toMap((u) -> u.getTaskState(), (u) -> u))) ); }
private Optional<Long> getRunningAt(Collection<SingularityTaskHistoryUpdate> updates) { for (SingularityTaskHistoryUpdate update : updates) { if (update.getTaskState() == ExtendedTaskState.TASK_RUNNING) { return Optional.of(update.getTimestamp()); } } return Optional.absent(); }
private Optional<Long> getRunningAt(Collection<SingularityTaskHistoryUpdate> updates) { for (SingularityTaskHistoryUpdate update : updates) { if (update.getTaskState() == ExtendedTaskState.TASK_RUNNING) { return Optional.of(update.getTimestamp()); } } return Optional.absent(); }
public void saveTaskHistoryUpdate(SingularityTaskHistoryUpdate taskHistoryUpdate, boolean overwrite) { if (!active) { LOG.warn("saveTaskHistoryUpdate {}, but not active", taskHistoryUpdate); return; } historyUpdates.putIfAbsent(taskHistoryUpdate.getTaskId(), new ConcurrentHashMap<>()); if (overwrite) { historyUpdates.get(taskHistoryUpdate.getTaskId()).put(taskHistoryUpdate.getTaskState(), taskHistoryUpdate); } else { historyUpdates.get(taskHistoryUpdate.getTaskId()).putIfAbsent(taskHistoryUpdate.getTaskState(), taskHistoryUpdate); } }
private long getTaskRunningStartTime(SingularityTaskId task) { Optional<SingularityTaskHistory> taskHistory = taskManager.getTaskHistory(task); if (taskHistory.isPresent()) { java.util.Optional<SingularityTaskHistoryUpdate> taskRunningState = taskHistory.get().getTaskUpdates().stream().filter(h -> h.getTaskState().equals(ExtendedTaskState.TASK_RUNNING)).findFirst(); if (taskRunningState.isPresent()) { return taskRunningState.get().getTimestamp(); } LOG.error("Could not find time when task {} reached TASK_RUNNING state", task); } else { LOG.error("Could not find task history for {}", task); } return System.currentTimeMillis(); }
private Optional<SingularityDeployFailure> getNonHealthcheckedTaskFailure(Map<SingularityTaskId, List<SingularityTaskHistoryUpdate>> taskUpdates, SingularityTaskId taskId) { List<SingularityTaskHistoryUpdate> updates = taskUpdates.get(taskId); SingularityTaskHistoryUpdate lastUpdate = Iterables.getLast(updates); if (lastUpdate.getTaskState().isSuccess()) { return Optional.of(new SingularityDeployFailure(SingularityDeployFailureReason.TASK_EXPECTED_RUNNING_FINISHED, Optional.of(taskId), Optional.of(String.format("Task was expected to maintain TASK_RUNNING state but finished. (%s)", lastUpdate.getStatusMessage().or(""))))); } else if (lastUpdate.getTaskState().isDone()) { return Optional.of(new SingularityDeployFailure(SingularityDeployFailureReason.TASK_FAILED_ON_STARTUP, Optional.of(taskId), lastUpdate.getStatusMessage())); } else if (SingularityTaskHistoryUpdate.getCurrentState(updates) == SimplifiedTaskState.WAITING) { return Optional.of(new SingularityDeployFailure(SingularityDeployFailureReason.TASK_NEVER_ENTERED_RUNNING, Optional.of(taskId), Optional.of(String.format("Task never entered running state, last state was %s (%s)", lastUpdate.getTaskState().getDisplayName(), lastUpdate.getStatusMessage().or(""))))); } return Optional.absent(); } }
private String getTaskHistoryUpdateId(SingularityTaskHistoryUpdate taskUpdate) { return taskUpdate.getTaskId() + "-" + taskUpdate.getTaskState().name(); }
@Override public void sendTaskCompletedMail(SingularityTaskHistory taskHistory, SingularityRequest request) { final Optional<SingularityTaskHistoryUpdate> lastUpdate = taskHistory.getLastTaskUpdate(); if (!lastUpdate.isPresent()) { LOG.warn("Can't send task completed mail for task {} - no last update", taskHistory.getTask().getTaskId()); return; } final Optional<SingularityEmailType> emailType = getEmailType(lastUpdate.get().getTaskState(), request, taskHistory.getTaskUpdates()); if (!emailType.isPresent()) { LOG.debug("No configured emailType for {} and {}", request, lastUpdate.get().getTaskState()); return; } prepareTaskMail(Optional.of(taskHistory.getTask()), taskHistory.getTask().getTaskId(), request, emailType.get(), Collections.<String, Object> emptyMap(), taskHistory.getTaskUpdates(), lastUpdate.get().getTaskState(), taskHistory.getTaskMetadata()); }
@Override public int compareTo(SingularityTaskHistoryUpdate o) { return ComparisonChain.start() .compare(taskState.ordinal(), o.getTaskState().ordinal()) .compare(timestamp, o.getTimestamp()) .compare(o.getTaskId().getId(), getTaskId().getId()) .result(); }
public List<SingularityMailTaskHistoryUpdate> getJadeTaskHistory(Collection<SingularityTaskHistoryUpdate> taskHistory) { List<SingularityMailTaskHistoryUpdate> output = Lists.newArrayListWithCapacity(taskHistory.size()); for (SingularityTaskHistoryUpdate taskUpdate : taskHistory) { output.add( new SingularityMailTaskHistoryUpdate( humanizeTimestamp(taskUpdate.getTimestamp()), WordUtils.capitalize(taskUpdate.getTaskState().getDisplayName()), taskUpdate.getStatusMessage().or(""))); } return output; }
public Optional<SingularityTaskIdHistory> getMostRecentTask(SingularityRequest request) { List<SingularityTaskId> activeTaskIds = taskManager.getActiveTaskIdsForRequest(request.getId()); if (!activeTaskIds.isEmpty()) { SingularityTaskId lastTaskId = activeTaskIds.get(0); List<SingularityTaskHistoryUpdate> historyUpdates = taskManager.getTaskHistoryUpdates(lastTaskId); if (!historyUpdates.isEmpty()) { SingularityTaskHistoryUpdate lastUpdate = historyUpdates.get(historyUpdates.size() - 1); return Optional.of(new SingularityTaskIdHistory( lastTaskId, lastUpdate.getTimestamp(), Optional.of(lastUpdate.getTaskState()), Optional.absent() // runId not currently provided here, grabbing the full task data for this is a more expensive call )); } } List<SingularityTaskIdHistory> maybeRecentTasks = taskHistoryHelper.getBlendedHistory(new SingularityTaskHistoryQuery(request.getId()), 0 , 1); if (!maybeRecentTasks.isEmpty()) { return Optional.of(maybeRecentTasks.get(0)); } return Optional.absent(); }
public static SingularityTaskIdHistory fromTaskIdAndTaskAndUpdates(SingularityTaskId taskId, SingularityTask task, List<SingularityTaskHistoryUpdate> updates) { ExtendedTaskState lastTaskState = null; long updatedAt = taskId.getStartedAt(); if (updates != null && !updates.isEmpty()) { SingularityTaskHistoryUpdate lastUpdate = Collections.max(updates); lastTaskState = lastUpdate.getTaskState(); updatedAt = lastUpdate.getTimestamp(); } return new SingularityTaskIdHistory(taskId, updatedAt, Optional.fromNullable(lastTaskState), task.getTaskRequest().getPendingTask().getRunId()); }
@Test public void testTaskOrdering() { final SingularityTaskId taskId = new SingularityTaskId("r", "d", System.currentTimeMillis(), 1, "h", "r"); final Optional<String> msg = Optional.absent(); SingularityTaskHistoryUpdate update1 = new SingularityTaskHistoryUpdate(taskId, 1L, ExtendedTaskState.TASK_LAUNCHED, msg, Optional.<String>absent()); SingularityTaskHistoryUpdate update2 = new SingularityTaskHistoryUpdate(taskId, 2L, ExtendedTaskState.TASK_RUNNING, msg, Optional.<String>absent()); SingularityTaskHistoryUpdate update3 = new SingularityTaskHistoryUpdate(taskId, 2L, ExtendedTaskState.TASK_FAILED, msg, Optional.<String>absent()); List<SingularityTaskHistoryUpdate> list = Arrays.asList(update2, update1, update3); Collections.sort(list); Assert.assertTrue(list.get(0).getTaskState() == ExtendedTaskState.TASK_LAUNCHED); Assert.assertTrue(list.get(1).getTaskState() == ExtendedTaskState.TASK_RUNNING); Assert.assertTrue(list.get(2).getTaskState() == ExtendedTaskState.TASK_FAILED); }
@Override protected void handleExpiringObject(SingularityExpiringBounce expiringObject, SingularityRequestWithState requestWithState, String message) { for (SingularityTaskCleanup taskCleanup : taskManager.getCleanupTasks()) { if (taskCleanup.getTaskId().getRequestId().equals(expiringObject.getRequestId()) && taskCleanup.getActionId().isPresent() && expiringObject.getActionId().equals(taskCleanup.getActionId().get())) { LOG.info("Discarding cleanup for {} ({}) because of {}", taskCleanup.getTaskId(), taskCleanup, expiringObject); taskManager.deleteCleanupTask(taskCleanup.getTaskId().getId()); if (!taskManager.getTaskCleanup(taskCleanup.getTaskId().getId()).isPresent()) { LOG.info("No other task cleanups found, removing task cleanup update for {}", taskCleanup.getTaskId()); List<SingularityTaskHistoryUpdate> historyUpdates = taskManager.getTaskHistoryUpdates(taskCleanup.getTaskId()); Collections.sort(historyUpdates); if (Iterables.getLast(historyUpdates).getTaskState() == ExtendedTaskState.TASK_CLEANING) { Optional<SingularityTaskHistoryUpdate> maybePreviousHistoryUpdate = historyUpdates.size() > 1 ? Optional.of(historyUpdates.get(historyUpdates.size() - 2)) : Optional.<SingularityTaskHistoryUpdate>absent(); taskManager.deleteTaskHistoryUpdate(taskCleanup.getTaskId(), ExtendedTaskState.TASK_CLEANING, maybePreviousHistoryUpdate); } } } } Optional<SingularityPendingRequest> pendingRequest = requestManager.getPendingRequest(expiringObject.getRequestId(), expiringObject.getDeployId()); if (pendingRequest.isPresent() && pendingRequest.get().getActionId().isPresent() && pendingRequest.get().getActionId().get().equals(expiringObject.getActionId())) { LOG.info("Discarding pending request for {} ({}) because of {}", expiringObject.getRequestId(), pendingRequest.get(), expiringObject); requestManager.deletePendingRequest(pendingRequest.get()); } requestManager.addToPendingQueue(new SingularityPendingRequest(expiringObject.getRequestId(), expiringObject.getDeployId(), System.currentTimeMillis(), expiringObject.getUser(), PendingType.CANCEL_BOUNCE, Optional.absent(), Optional.absent(), Optional.absent(), Optional.of(message), Optional.of(expiringObject.getActionId()))); }
@Timed public SingularityCreateResult saveTaskHistoryUpdate(SingularityTaskHistoryUpdate taskHistoryUpdate, boolean overwriteExisting) { singularityEventListener.taskHistoryUpdateEvent(taskHistoryUpdate); if (overwriteExisting) { Optional<SingularityTaskHistoryUpdate> maybeExisting = getTaskHistoryUpdate(taskHistoryUpdate.getTaskId(), taskHistoryUpdate.getTaskState()); LOG.info("Found existing history {}", maybeExisting); SingularityTaskHistoryUpdate updateWithPrevious; if (maybeExisting.isPresent()) { updateWithPrevious = taskHistoryUpdate.withPrevious(maybeExisting.get()); LOG.info("Will save new update {}", updateWithPrevious); } else { updateWithPrevious = taskHistoryUpdate; } if (leaderCache.active()) { leaderCache.saveTaskHistoryUpdate(updateWithPrevious, overwriteExisting); } return save(getUpdatePath(taskHistoryUpdate.getTaskId(), taskHistoryUpdate.getTaskState()), updateWithPrevious, taskHistoryUpdateTranscoder); } else { if (leaderCache.active()) { leaderCache.saveTaskHistoryUpdate(taskHistoryUpdate, overwriteExisting); } return create(getUpdatePath(taskHistoryUpdate.getTaskId(), taskHistoryUpdate.getTaskState()), taskHistoryUpdate, taskHistoryUpdateTranscoder); } }
public static SingularityTaskState fromTaskHistory(SingularityTaskHistory taskHistory) { return new SingularityTaskState( Optional.of(taskHistory.getTask().getTaskId()), Optional.of(taskHistory.getTask().getTaskRequest().getPendingTask().getPendingTaskId()), taskHistory.getTask().getTaskRequest().getPendingTask().getRunId(), Optional.of(taskHistory.getLastTaskUpdate().get().getTaskState()), taskHistory.getTaskUpdates(), false ); }
@Override public void applyMigration() { final long start = System.currentTimeMillis(); final List<SingularityTaskId> taskIds = taskManager.getActiveTaskIds(); for (SingularityTaskId taskId : taskIds) { List<SingularityTaskHistoryUpdate> updates = Lists.reverse(taskManager.getTaskHistoryUpdates(taskId)); Optional<MesosTaskStatusObject> taskStatus = Optional.absent(); for (SingularityTaskHistoryUpdate update : updates) { if (update.getTaskState().toTaskState().isPresent()) { Optional<SingularityTask> task = taskManager.getTask(taskId); taskStatus = Optional.of(mesosProtosUtils.taskStatusFromProtos(TaskStatus.newBuilder() .setTaskId(TaskID.newBuilder().setValue(taskId.getId())) .setAgentId(MesosProtosUtils.toAgentId(task.get().getAgentId())) .setState(MesosProtosUtils.toTaskState(update.getTaskState())) .build())); break; } } SingularityTaskStatusHolder taskStatusHolder = new SingularityTaskStatusHolder(taskId, taskStatus, start, serverId, Optional.absent()); taskManager.saveLastActiveTaskStatus(taskStatusHolder); } }
private TaskCleanupResult cleanTask(SingularityExecutorTaskDefinition taskDefinition, Optional<SingularityTaskHistory> taskHistory) { SingularityExecutorTaskLogManager logManager = new SingularityExecutorTaskLogManager(taskDefinition, templateManager, baseConfiguration, executorConfiguration, LOG, jsonObjectFileHelper, false); SingularityExecutorTaskCleanup taskCleanup = new SingularityExecutorTaskCleanup(logManager, executorConfiguration, taskDefinition, LOG, dockerUtils); boolean cleanupTaskAppDirectory = !taskDefinition.getExecutorData().getPreserveTaskSandboxAfterFinish().or(Boolean.FALSE); if (taskDefinition.shouldLogrotateLogFile()) { checkForUncompressedLogrotatedFile(taskDefinition); } if (taskHistory.isPresent()) { final Optional<SingularityTaskHistoryUpdate> lastUpdate = JavaUtils.getLast(taskHistory.get().getTaskUpdates()); if (lastUpdate.isPresent()) { if (lastUpdate.get().getTaskState().isDone() && System.currentTimeMillis() - lastUpdate.get().getTimestamp() > TimeUnit.MINUTES.toMillis(15)) { LOG.info("Task {} is done for > 15 minutes, removing logrotate files"); taskCleanup.cleanUpLogs(); } if (lastUpdate.get().getTaskState().isFailed()) { final long delta = System.currentTimeMillis() - lastUpdate.get().getTimestamp(); if (delta < cleanupConfiguration.getCleanupAppDirectoryOfFailedTasksAfterMillis()) { LOG.info("Not cleaning up task app directory of {} because only {} has elapsed since it failed (will cleanup after {})", taskDefinition.getTaskId(), JavaUtils.durationFromMillis(delta), JavaUtils.durationFromMillis(cleanupConfiguration.getCleanupAppDirectoryOfFailedTasksAfterMillis())); cleanupTaskAppDirectory = false; } } } } boolean isDocker = (taskHistory.isPresent() && taskHistory.get().getTask().getTaskRequest().getDeploy().getContainerInfo().isPresent() && taskHistory.get().getTask().getTaskRequest().getDeploy().getContainerInfo().get().getType() == SingularityContainerType.DOCKER); return taskCleanup.cleanup(cleanupTaskAppDirectory, isDocker); }