@JsonIgnore public Optional<SingularityTaskHistoryUpdate> getLastTaskUpdate() { return JavaUtils.getLast(getTaskUpdates()); }
private long getTaskRunningStartTime(SingularityTaskId task) { Optional<SingularityTaskHistory> taskHistory = taskManager.getTaskHistory(task); if (taskHistory.isPresent()) { java.util.Optional<SingularityTaskHistoryUpdate> taskRunningState = taskHistory.get().getTaskUpdates().stream().filter(h -> h.getTaskState().equals(ExtendedTaskState.TASK_RUNNING)).findFirst(); if (taskRunningState.isPresent()) { return taskRunningState.get().getTimestamp(); } LOG.error("Could not find time when task {} reached TASK_RUNNING state", task); } else { LOG.error("Could not find task history for {}", task); } return System.currentTimeMillis(); }
public Optional<SingularityTaskIdHistory> getByRunId(String requestId, String runId) { for (SingularityTaskIdHistory history : getFromZk(Collections.singletonList(requestId))) { if (history.getRunId().isPresent() && history.getRunId().get().equals(runId)) { return Optional.of(history); } } Optional<SingularityTaskHistory> history = historyManager.getTaskHistoryByRunId(requestId, runId); if (history.isPresent()) { return Optional.of(SingularityTaskIdHistory.fromTaskIdAndTaskAndUpdates(history.get().getTask().getTaskId(), history.get().getTask(), history.get().getTaskUpdates())); } return Optional.absent(); }
@Override public void sendTaskCompletedMail(SingularityTaskHistory taskHistory, SingularityRequest request) { final Optional<SingularityTaskHistoryUpdate> lastUpdate = taskHistory.getLastTaskUpdate(); if (!lastUpdate.isPresent()) { LOG.warn("Can't send task completed mail for task {} - no last update", taskHistory.getTask().getTaskId()); return; } final Optional<SingularityEmailType> emailType = getEmailType(lastUpdate.get().getTaskState(), request, taskHistory.getTaskUpdates()); if (!emailType.isPresent()) { LOG.debug("No configured emailType for {} and {}", request, lastUpdate.get().getTaskState()); return; } prepareTaskMail(Optional.of(taskHistory.getTask()), taskHistory.getTask().getTaskId(), request, emailType.get(), Collections.<String, Object> emptyMap(), taskHistory.getTaskUpdates(), lastUpdate.get().getTaskState(), taskHistory.getTaskMetadata()); }
@Override protected boolean moveToHistory(SingularityTaskId object) { final Optional<SingularityTaskHistory> taskHistory = taskManager.getTaskHistory(object); if (taskHistory.isPresent()) { if (!taskHistory.get().getTaskUpdates().isEmpty()) { final long lastUpdateAt = taskHistory.get().getLastTaskUpdate().get().getTimestamp(); final long timeSinceLastUpdate = System.currentTimeMillis() - lastUpdateAt; if (timeSinceLastUpdate < taskMetadataConfiguration.getTaskPersistAfterFinishBufferMillis()) { LOG.debug("Not persisting {} yet - lastUpdate only happened {} ago, buffer {}", JavaUtils.durationFromMillis(timeSinceLastUpdate), JavaUtils.durationFromMillis(taskMetadataConfiguration.getTaskPersistAfterFinishBufferMillis())); return false; } } LOG.debug("Moving {} to history", object); try { historyManager.saveTaskHistory(taskHistory.get()); } catch (Throwable t) { LOG.warn("Failed to persist task into History for task {}", object, t); return false; } } else { LOG.warn("Inactive task {} did not have a task to persist", object); } return true; }
SimplifiedTaskState taskState = SingularityTaskHistoryUpdate.getCurrentState(history.get().getTaskUpdates()); end = Iterables.getLast(history.get().getTaskUpdates()).getTimestamp(); } else { end = System.currentTimeMillis();
public static SingularityTaskState fromTaskHistory(SingularityTaskHistory taskHistory) { return new SingularityTaskState( Optional.of(taskHistory.getTask().getTaskId()), Optional.of(taskHistory.getTask().getTaskRequest().getPendingTask().getPendingTaskId()), taskHistory.getTask().getTaskRequest().getPendingTask().getRunId(), Optional.of(taskHistory.getLastTaskUpdate().get().getTaskState()), taskHistory.getTaskUpdates(), false ); }
private TaskCleanupResult cleanTask(SingularityExecutorTaskDefinition taskDefinition, Optional<SingularityTaskHistory> taskHistory) { SingularityExecutorTaskLogManager logManager = new SingularityExecutorTaskLogManager(taskDefinition, templateManager, baseConfiguration, executorConfiguration, LOG, jsonObjectFileHelper, false); SingularityExecutorTaskCleanup taskCleanup = new SingularityExecutorTaskCleanup(logManager, executorConfiguration, taskDefinition, LOG, dockerUtils); boolean cleanupTaskAppDirectory = !taskDefinition.getExecutorData().getPreserveTaskSandboxAfterFinish().or(Boolean.FALSE); if (taskDefinition.shouldLogrotateLogFile()) { checkForUncompressedLogrotatedFile(taskDefinition); } if (taskHistory.isPresent()) { final Optional<SingularityTaskHistoryUpdate> lastUpdate = JavaUtils.getLast(taskHistory.get().getTaskUpdates()); if (lastUpdate.isPresent()) { if (lastUpdate.get().getTaskState().isDone() && System.currentTimeMillis() - lastUpdate.get().getTimestamp() > TimeUnit.MINUTES.toMillis(15)) { LOG.info("Task {} is done for > 15 minutes, removing logrotate files"); taskCleanup.cleanUpLogs(); } if (lastUpdate.get().getTaskState().isFailed()) { final long delta = System.currentTimeMillis() - lastUpdate.get().getTimestamp(); if (delta < cleanupConfiguration.getCleanupAppDirectoryOfFailedTasksAfterMillis()) { LOG.info("Not cleaning up task app directory of {} because only {} has elapsed since it failed (will cleanup after {})", taskDefinition.getTaskId(), JavaUtils.durationFromMillis(delta), JavaUtils.durationFromMillis(cleanupConfiguration.getCleanupAppDirectoryOfFailedTasksAfterMillis())); cleanupTaskAppDirectory = false; } } } } boolean isDocker = (taskHistory.isPresent() && taskHistory.get().getTask().getTaskRequest().getDeploy().getContainerInfo().isPresent() && taskHistory.get().getTask().getTaskRequest().getDeploy().getContainerInfo().get().getType() == SingularityContainerType.DOCKER); return taskCleanup.cleanup(cleanupTaskAppDirectory, isDocker); }
@Override public void saveTaskHistory(SingularityTaskHistory taskHistory) { if (history.getTaskHistoryForTask(taskHistory.getTask().getTaskId().getId()) != null) { if (LOG.isTraceEnabled()) { LOG.trace("saveTaskHistory -- existing taskHistory {}", taskHistory); } return; } SingularityTaskIdHistory taskIdHistory = SingularityTaskIdHistory.fromTaskIdAndTaskAndUpdates(taskHistory.getTask().getTaskId(), taskHistory.getTask(), taskHistory.getTaskUpdates()); String lastTaskStatus = null; if (taskIdHistory.getLastTaskState().isPresent()) { lastTaskStatus = taskIdHistory.getLastTaskState().get().name(); } if (LOG.isTraceEnabled()) { LOG.trace("saveTaskHistory -- will insert taskHistory {}", taskHistory); } history.insertTaskHistory(taskIdHistory.getTaskId().getRequestId(), taskIdHistory.getTaskId().getId(), taskHistoryTranscoder.toBytes(taskHistory), new Date(taskIdHistory.getUpdatedAt()), lastTaskStatus, taskHistory.getTask().getTaskRequest().getPendingTask().getRunId().orNull(), taskIdHistory.getTaskId().getDeployId(), taskIdHistory.getTaskId().getHost(), new Date(taskIdHistory.getTaskId().getStartedAt())); }
private long getTaskRunningStartTime(SingularityTaskId task) { Optional<SingularityTaskHistory> taskHistory = taskManager.getTaskHistory(task); if (taskHistory.isPresent()) { java.util.Optional<SingularityTaskHistoryUpdate> taskRunningState = taskHistory.get().getTaskUpdates().stream().filter(h -> h.getTaskState().equals(ExtendedTaskState.TASK_RUNNING)).findFirst(); if (taskRunningState.isPresent()) { return taskRunningState.get().getTimestamp(); } LOG.error("Could not find time when task {} reached TASK_RUNNING state", task); } else { LOG.error("Could not find task history for {}", task); } return System.currentTimeMillis(); }
public Optional<SingularityTaskIdHistory> getByRunId(String requestId, String runId) { for (SingularityTaskIdHistory history : getFromZk(Collections.singletonList(requestId))) { if (history.getRunId().isPresent() && history.getRunId().get().equals(runId)) { return Optional.of(history); } } Optional<SingularityTaskHistory> history = historyManager.getTaskHistoryByRunId(requestId, runId); if (history.isPresent()) { return Optional.of(SingularityTaskIdHistory.fromTaskIdAndTaskAndUpdates(history.get().getTask().getTaskId(), history.get().getTask(), history.get().getTaskUpdates())); } return Optional.absent(); }
@Override public void sendTaskCompletedMail(SingularityTaskHistory taskHistory, SingularityRequest request) { final Optional<SingularityTaskHistoryUpdate> lastUpdate = taskHistory.getLastTaskUpdate(); if (!lastUpdate.isPresent()) { LOG.warn("Can't send task completed mail for task {} - no last update", taskHistory.getTask().getTaskId()); return; } final Optional<SingularityEmailType> emailType = getEmailType(lastUpdate.get().getTaskState(), request, taskHistory.getTaskUpdates()); if (!emailType.isPresent()) { LOG.debug("No configured emailType for {} and {}", request, lastUpdate.get().getTaskState()); return; } prepareTaskMail(Optional.of(taskHistory.getTask()), taskHistory.getTask().getTaskId(), request, emailType.get(), Collections.<String, Object> emptyMap(), taskHistory.getTaskUpdates(), lastUpdate.get().getTaskState(), taskHistory.getTaskMetadata()); }
@Override protected boolean moveToHistory(SingularityTaskId object) { final Optional<SingularityTaskHistory> taskHistory = taskManager.getTaskHistory(object); if (taskHistory.isPresent()) { if (!taskHistory.get().getTaskUpdates().isEmpty()) { final long lastUpdateAt = taskHistory.get().getLastTaskUpdate().get().getTimestamp(); final long timeSinceLastUpdate = System.currentTimeMillis() - lastUpdateAt; if (timeSinceLastUpdate < taskMetadataConfiguration.getTaskPersistAfterFinishBufferMillis()) { LOG.debug("Not persisting {} yet - lastUpdate only happened {} ago, buffer {}", JavaUtils.durationFromMillis(timeSinceLastUpdate), JavaUtils.durationFromMillis(taskMetadataConfiguration.getTaskPersistAfterFinishBufferMillis())); return false; } } LOG.debug("Moving {} to history", object); try { historyManager.saveTaskHistory(taskHistory.get()); } catch (Throwable t) { LOG.warn("Failed to persist task into History for task {}", object, t); return false; } } else { LOG.warn("Inactive task {} did not have a task to persist", object); } return true; }
SimplifiedTaskState taskState = SingularityTaskHistoryUpdate.getCurrentState(history.get().getTaskUpdates()); end = Iterables.getLast(history.get().getTaskUpdates()).getTimestamp(); } else { end = System.currentTimeMillis();
@Override public void saveTaskHistory(SingularityTaskHistory taskHistory) { if (history.getTaskHistoryForTask(taskHistory.getTask().getTaskId().getId()) != null) { if (LOG.isTraceEnabled()) { LOG.trace("saveTaskHistory -- existing taskHistory {}", taskHistory); } return; } SingularityTaskIdHistory taskIdHistory = SingularityTaskIdHistory.fromTaskIdAndTaskAndUpdates(taskHistory.getTask().getTaskId(), taskHistory.getTask(), taskHistory.getTaskUpdates()); String lastTaskStatus = null; if (taskIdHistory.getLastTaskState().isPresent()) { lastTaskStatus = taskIdHistory.getLastTaskState().get().name(); } if (LOG.isTraceEnabled()) { LOG.trace("saveTaskHistory -- will insert taskHistory {}", taskHistory); } history.insertTaskHistory(taskIdHistory.getTaskId().getRequestId(), taskIdHistory.getTaskId().getId(), taskHistoryTranscoder.toBytes(taskHistory), new Date(taskIdHistory.getUpdatedAt()), lastTaskStatus, taskHistory.getTask().getTaskRequest().getPendingTask().getRunId().orNull(), taskIdHistory.getTaskId().getDeployId(), taskIdHistory.getTaskId().getHost(), new Date(taskIdHistory.getTaskId().getStartedAt())); }