@Override @JsonIgnore public long getCreateTimestampForCalculatingHistoryAge() { return getStartedAt(); }
private List<SingularityTaskUsage> getFullListOfTaskUsages(List<SingularityTaskUsage> pastTaskUsages, SingularityTaskUsage latestUsage, SingularityTaskId task) { List<SingularityTaskUsage> pastTaskUsagesCopy = new ArrayList<>(); pastTaskUsagesCopy.add(new SingularityTaskUsage(0, TimeUnit.MILLISECONDS.toSeconds(task.getStartedAt()), 0, 0, 0 , 0, 0)); // to calculate oldest cpu usage pastTaskUsagesCopy.addAll(pastTaskUsages); pastTaskUsagesCopy.add(latestUsage); return pastTaskUsagesCopy; }
@Override public int compare(SingularityTaskIdHistory o1, SingularityTaskIdHistory o2) { ComparisonChain chain = ComparisonChain.start(); if (localOrderDirection == OrderDirection.ASC) { chain = chain.compare(o1.getTaskId().getStartedAt(), o2.getTaskId().getStartedAt()); } else { chain = chain.compare(o2.getTaskId().getStartedAt(), o1.getTaskId().getStartedAt()); } return chain.compare(o1.getTaskId().getRequestId(), o2.getTaskId().getRequestId()).result(); }
private long getTimestampSeconds(SingularityTaskId taskId, long seconds) { return TimeUnit.MILLISECONDS.toSeconds(taskId.getStartedAt()) + seconds; }
private String fillInTaskIdValues(String string, SingularityOfferHolder offerHolder, SingularityTaskId taskId) { if (!Strings.isNullOrEmpty(string)) { string = string.replace("${TASK_REQUEST_ID}", taskId.getRequestId()) .replace("${TASK_DEPLOY_ID}", taskId.getDeployId()) .replace("${TASK_STARTED_AT}", Long.toString(taskId.getStartedAt())) .replace("${TASK_INSTANCE_NO}", Integer.toString(taskId.getInstanceNo())) .replace("${TASK_HOST}", offerHolder.getHostname()) .replace("${TASK_RACK_ID}", offerHolder.getRackId()) .replace("${TASK_ID}", taskId.getId()); } return string; }
public static SingularityTaskIdHistory fromTaskIdAndTaskAndUpdates(SingularityTaskId taskId, SingularityTask task, List<SingularityTaskHistoryUpdate> updates) { ExtendedTaskState lastTaskState = null; long updatedAt = taskId.getStartedAt(); if (updates != null && !updates.isEmpty()) { SingularityTaskHistoryUpdate lastUpdate = Collections.max(updates); lastTaskState = lastUpdate.getTaskState(); updatedAt = lastUpdate.getTimestamp(); } return new SingularityTaskIdHistory(taskId, updatedAt, Optional.fromNullable(lastTaskState), task.getTaskRequest().getPendingTask().getRunId()); }
if (startedAfter.isPresent() && startedAfter.get() >= taskId.getStartedAt()) { return false; if (startedBefore.isPresent() && startedBefore.get() <= taskId.getStartedAt()) { return false;
private void checkTaskExecutionTimeLimit(long now, SingularityTaskId taskId, SingularityRequest request) { final long runtime = now - taskId.getStartedAt(); if (request.getTaskExecutionTimeLimitMillis().or(configuration.getTaskExecutionTimeLimitMillis()).isPresent() && runtime >= request.getTaskExecutionTimeLimitMillis().or(configuration.getTaskExecutionTimeLimitMillis()).get()) { taskManager.createTaskCleanup(new SingularityTaskCleanup( Optional.<String>absent(), TaskCleanupType.TASK_EXCEEDED_TIME_LIMIT, now, taskId, Optional.of(String.format("Task has run for %s, which exceeds the maximum execution time of %s", DurationFormatUtils.formatDurationHMS(runtime), DurationFormatUtils.formatDurationHMS(request.getTaskExecutionTimeLimitMillis().or(configuration.getTaskExecutionTimeLimitMillis()).get())) ), Optional.of(UUID.randomUUID().toString()), Optional.<SingularityTaskShellCommandRequestId>absent()) ); } }
private boolean isTaskOverdue(SingularityTask task) { final long taskDuration = System.currentTimeMillis() - task.getTaskId().getStartedAt(); final boolean isOverdue = taskDuration > getKillAfterTaskNotRunningMillis(); if (isOverdue) { LOG.debug("Task {} is overdue (duration: {}), allowed limit {}", task.getTaskId(), JavaUtils.durationFromMillis(taskDuration), JavaUtils.durationFromMillis(getKillAfterTaskNotRunningMillis())); } return isOverdue; }
@Override public void runActionOnPoll() { final long now = System.currentTimeMillis(); final List<SingularityTaskId> activeTaskIds = taskManager.getActiveTaskIds(); final Set<String> requestIdsToLookup = Sets.newHashSetWithExpectedSize(activeTaskIds.size()); for (SingularityTaskId taskId : activeTaskIds) { requestIdsToLookup.add(taskId.getRequestId()); } final Map<String, SingularityRequestWithState> idToRequest = Maps.uniqueIndex(requestManager.getRequests(requestIdsToLookup), SingularityRequestWithState.REQUEST_STATE_TO_REQUEST_ID); for (SingularityTaskId taskId : activeTaskIds) { SingularityRequestWithState requestWithState = idToRequest.get(taskId.getRequestId()); if (requestWithState == null) { LOG.warn("Active request not found for task ID {}", taskId); continue; } SingularityRequest request = requestWithState.getRequest(); if (!request.isLongRunning()) { checkForOverdueScheduledJob(now - taskId.getStartedAt(), taskId, request); checkTaskExecutionTimeLimit(now, taskId, request); } } }
int i = 0; for (SingularityTaskId taskId : entry.getValue()) { final long age = start - taskId.getStartedAt();
} else { final CronExpression cronExpression = new CronExpression(scheduleExpression); final Date startDate = new Date(taskId.getStartedAt()); nextRunAtDate = cronExpression.getNextValidTimeAfter(startDate); return Optional.of(nextRunAtDate.getTime() - taskId.getStartedAt());
private Collection<String> getS3PrefixesForTask(S3Configuration s3Configuration, SingularityTaskId taskId, Optional<Long> startArg, Optional<Long> endArg, String group, SingularityUser user) { Optional<SingularityTaskHistory> history = getTaskHistory(taskId, user); long start = taskId.getStartedAt(); if (startArg.isPresent()) { start = Math.max(startArg.get(), start);
public Logger buildTaskLogger(String taskId, String executorId, String executorPid, String taskLogFile) { LOG.info("Building a task logger for {} pointing to {}", taskId, taskLogFile); LoggerContext context = new LoggerContext(); context.setName(executorPid); baseLogging.prepareRootLogger(context); String loggerId = taskId; try { SingularityTaskId singularityTaskId = SingularityTaskId.valueOf(taskId); loggerId = String.format("%s.%s.%s.%s.%s", singularityTaskId.getRequestId(), singularityTaskId.getDeployId(), singularityTaskId.getStartedAt(), singularityTaskId.getInstanceNo(), executorId); } catch (InvalidSingularityTaskIdException e) { LOG.info("Handling non-SingularityTaskId %s", taskId); } Logger taskLogger = context.getLogger(loggerId); taskLogger.detachAndStopAllAppenders(); if (baseLogging.getRootLogPath().isPresent()) { taskLogger.addAppender(baseLogging.buildFileAppender(context, baseLogging.getRootLogPath().get())); } taskLogger.addAppender(baseLogging.buildFileAppender(context, taskLogFile)); context.start(); return taskLogger; }
@Override public void saveTaskHistory(SingularityTaskHistory taskHistory) { if (history.getTaskHistoryForTask(taskHistory.getTask().getTaskId().getId()) != null) { if (LOG.isTraceEnabled()) { LOG.trace("saveTaskHistory -- existing taskHistory {}", taskHistory); } return; } SingularityTaskIdHistory taskIdHistory = SingularityTaskIdHistory.fromTaskIdAndTaskAndUpdates(taskHistory.getTask().getTaskId(), taskHistory.getTask(), taskHistory.getTaskUpdates()); String lastTaskStatus = null; if (taskIdHistory.getLastTaskState().isPresent()) { lastTaskStatus = taskIdHistory.getLastTaskState().get().name(); } if (LOG.isTraceEnabled()) { LOG.trace("saveTaskHistory -- will insert taskHistory {}", taskHistory); } history.insertTaskHistory(taskIdHistory.getTaskId().getRequestId(), taskIdHistory.getTaskId().getId(), taskHistoryTranscoder.toBytes(taskHistory), new Date(taskIdHistory.getUpdatedAt()), lastTaskStatus, taskHistory.getTask().getTaskRequest().getPendingTask().getRunId().orNull(), taskIdHistory.getTaskId().getDeployId(), taskIdHistory.getTaskId().getHost(), new Date(taskIdHistory.getTaskId().getStartedAt())); }
MesosTaskMonitorObject t1u1 = getTaskMonitor(t1, 10, TimeUnit.MILLISECONDS.toSeconds(taskId.getStartedAt()) + 5, 1000); mesosClient.setSlaveResourceUsage("host1", Collections.singletonList(t1u1)); usagePoller.runActionOnPoll(); MesosTaskMonitorObject t1u2 = getTaskMonitor(t1, 11, TimeUnit.MILLISECONDS.toSeconds(taskId.getStartedAt()) + 6, 1000); mesosClient.setSlaveResourceUsage("host1", Collections.singletonList(t1u2)); usagePoller.runActionOnPoll();
MesosTaskMonitorObject t1u1 = getTaskMonitor(t1, 10, TimeUnit.MILLISECONDS.toSeconds(taskId.getStartedAt()) + 5, 1000); mesosClient.setSlaveResourceUsage("host1", Collections.singletonList(t1u1)); usagePoller.runActionOnPoll();
SingularityTaskId taskId3 = taskManager.getActiveTaskIds().get(2); String t3 = taskId3.getId(); statusUpdate(taskManager.getTask(taskId1).get(), TaskState.TASK_STARTING, Optional.of(taskId1.getStartedAt())); statusUpdate(taskManager.getTask(taskId2).get(), TaskState.TASK_STARTING, Optional.of(taskId2.getStartedAt())); statusUpdate(taskManager.getTask(taskId3).get(), TaskState.TASK_STARTING, Optional.of(taskId3.getStartedAt())); MesosTaskMonitorObject t1u1 = getTaskMonitor(t1, 15, TimeUnit.MILLISECONDS.toSeconds(taskId1.getStartedAt()) + 5, 1024); MesosTaskMonitorObject t2u1 = getTaskMonitor(t2, 10, TimeUnit.MILLISECONDS.toSeconds(taskId2.getStartedAt()) + 5, 1024); MesosTaskMonitorObject t3u1 = getTaskMonitor(t3, 5, TimeUnit.MILLISECONDS.toSeconds(taskId3.getStartedAt()) + 5, 1024); mesosClient.setSlaveResourceUsage("host1", Arrays.asList(t1u1, t2u1, t3u1)); mesosClient.setSlaveMetricsSnapshot(
SingularityTaskId taskId2 = taskManager.getActiveTaskIds().get(1); String t2 = taskId2.getId(); statusUpdate(taskManager.getTask(taskId1).get(), TaskState.TASK_STARTING, Optional.of(taskId1.getStartedAt())); statusUpdate(taskManager.getTask(taskId2).get(), TaskState.TASK_STARTING, Optional.of(taskId2.getStartedAt())); MesosTaskMonitorObject t1u1 = getTaskMonitor(t1, 15, TimeUnit.MILLISECONDS.toSeconds(taskId1.getStartedAt()) + 5, 1024); MesosTaskMonitorObject t2u1 = getTaskMonitor(t2, 10, TimeUnit.MILLISECONDS.toSeconds(taskId2.getStartedAt()) + 5, 1024); mesosClient.setSlaveResourceUsage("host1", Arrays.asList(t1u1, t2u1)); mesosClient.setSlaveMetricsSnapshot(
private void assertEquals(SingularityTaskId one, SingularityTaskId two) { Assert.assertEquals(one, two); Assert.assertEquals(one.getDeployId(), two.getDeployId()); Assert.assertEquals(one.getRequestId(), two.getRequestId()); Assert.assertEquals(one.getSanitizedHost(), two.getSanitizedHost()); Assert.assertEquals(one.getSanitizedRackId(), two.getSanitizedRackId()); Assert.assertEquals(one.getStartedAt(), two.getStartedAt()); Assert.assertEquals(one.getInstanceNo(), two.getInstanceNo()); }