@Override public void run() { try { loadDirectoryAndContainer(task); } catch (Throwable t) { LOG.error("While fetching directory and container id for task: {}", task.getTaskId(), t); } } };
public void cacheActiveTasks(List<SingularityTask> activeTasks) { activeMissMeter.mark(); Map<SingularityTaskId, SingularityTask> newActiveTasks = new HashMap<>(activeTasks.size()); for (SingularityTask activeTask : activeTasks) { newActiveTasks.put(activeTask.getTaskId(), activeTask); } cachedActiveTasks = newActiveTasks; lastActiveTaskCache = System.currentTimeMillis(); }
public void checkHealthcheck(SingularityTask task) { if (!taskIdToHealthcheck.containsKey(task.getTaskId().getId())) { LOG.info("Enqueueing expected healthcheck for task {}", task.getTaskId()); Optional<SingularityTaskHealthcheckResult> lastHealthcheck = taskManager.getLastHealthcheck(task.getTaskId()); enqueueHealthcheck(task, false, true, !lastHealthcheck.isPresent()); } }
@Override public void onThrowable(Throwable t) { LOG.trace("Exception while making health check for task {}", task.getTaskId(), t); saveResult(Optional.<Integer> absent(), Optional.<String> absent(), Optional.of(String.format("Healthcheck failed due to exception: %s", t.getMessage())), Optional.of(t)); }
private void enqueueCheckWithDelay(final SingularityTask task, long delaySeconds, SingularityHealthchecker healthchecker) { LOG.trace("Enqueuing a new task check for task {} with delay {}", task.getTaskId(), DurationFormatUtils.formatDurationHMS(TimeUnit.SECONDS.toMillis(delaySeconds))); ScheduledFuture<?> future = executorService.schedule(getTaskCheck(task, healthchecker), delaySeconds, TimeUnit.SECONDS); taskIdToCheck.put(task.getTaskId().getId(), future); }
public void reEnqueueOrAbort(SingularityTask task, boolean inStartup) { try { enqueueHealthcheck(task, true, inStartup, false); } catch (Throwable t) { LOG.error("Caught throwable while re-enqueuing health check for {}, aborting", task.getTaskId(), t); exceptionNotifier.notify(String.format("Caught throwable while re-enqueuing health check (%s)", t.getMessage()), t, ImmutableMap.of("taskId", task.getTaskId().toString())); abort.abort(SingularityAbort.AbortReason.UNRECOVERABLE_ERROR, Optional.of(t)); } }
@Timed // should only be called on tasks that are new and not part of a pending deploy. public void enqueueNewTaskCheck(SingularityTask task, Optional<SingularityRequestWithState> requestWithState, SingularityHealthchecker healthchecker) { if (taskIdToCheck.containsKey(task.getTaskId().getId())) { LOG.trace("Already had a newTaskCheck for task {}", task.getTaskId()); return; } int delaySeconds = getDelaySeconds(task, requestWithState); enqueueCheckWithDelay(task, delaySeconds, healthchecker); }
@Override public void run() { try { asyncHealthcheck(task); } catch (Throwable t) { LOG.error("Uncaught throwable in async healthcheck", t); exceptionNotifier.notify(String.format("Uncaught throwable in async healthcheck (%s)", t.getMessage()), t, ImmutableMap.of("taskId", task.getTaskId().toString())); reEnqueueOrAbort(task, inStartup); } }
private void reEnqueueCheckOrAbort(SingularityTask task, SingularityHealthchecker healthchecker) { try { reEnqueueCheck(task, healthchecker); } catch (Throwable t) { LOG.error("Uncaught throwable re-enqueuing task check for task {}, aborting", task, t); exceptionNotifier.notify(String.format("Error in task check (%s)", t.getMessage()), t, ImmutableMap.of("taskId", task.getTaskId().toString())); abort.abort(AbortReason.UNRECOVERABLE_ERROR, Optional.of(t)); } }
private boolean isTaskOverdue(SingularityTask task) { final long taskDuration = System.currentTimeMillis() - task.getTaskId().getStartedAt(); final boolean isOverdue = taskDuration > getKillAfterTaskNotRunningMillis(); if (isOverdue) { LOG.debug("Task {} is overdue (duration: {}), allowed limit {}", task.getTaskId(), JavaUtils.durationFromMillis(taskDuration), JavaUtils.durationFromMillis(getKillAfterTaskNotRunningMillis())); } return isOverdue; }
public Optional<SingularityTaskIdHistory> getByRunId(String requestId, String runId) { for (SingularityTaskIdHistory history : getFromZk(Collections.singletonList(requestId))) { if (history.getRunId().isPresent() && history.getRunId().get().equals(runId)) { return Optional.of(history); } } Optional<SingularityTaskHistory> history = historyManager.getTaskHistoryByRunId(requestId, runId); if (history.isPresent()) { return Optional.of(SingularityTaskIdHistory.fromTaskIdAndTaskAndUpdates(history.get().getTask().getTaskId(), history.get().getTask(), history.get().getTaskUpdates())); } return Optional.absent(); }
private SingularityTask checkActiveTask(String taskId, SingularityAuthorizationScope scope, SingularityUser user) { SingularityTaskId taskIdObj = getTaskIdFromStr(taskId); Optional<SingularityTask> task = taskManager.getTask(taskIdObj); checkNotFound(task.isPresent() && taskManager.isActiveTask(taskId), "No active task with id %s", taskId); if (task.isPresent()) { authorizationHelper.checkForAuthorizationByRequestId(task.get().getTaskId().getRequestId(), user, scope); } return task.get(); }
@Timed public boolean enqueueHealthcheck(SingularityTask task, Optional<SingularityPendingDeploy> pendingDeploy, Optional<SingularityRequestWithState> request) { if (!shouldHealthcheck(task, request, pendingDeploy)) { return false; } Optional<SingularityTaskHealthcheckResult> lastHealthcheck = taskManager.getLastHealthcheck(task.getTaskId()); enqueueHealthcheck(task, true, true, !lastHealthcheck.isPresent()); return true; }
private Set<String> getRunningTaskIds() { final String slaveId = mesosClient.getSlaveState(mesosClient.getSlaveUri(hostname)).getId(); final Collection<SingularityTask> activeTasks = singularityClient.getActiveTasksOnSlave(slaveId); final Set<String> runningTaskIds = Sets.newHashSet(); for (SingularityTask task : activeTasks) { runningTaskIds.add(task.getTaskId().getId()); } return runningTaskIds; }
public void addMatchedTask(SingularityMesosTaskHolder taskHolder) { LOG.trace("Accepting task {} for offers {}", taskHolder.getTask().getTaskId(), offers.stream().map(Offer::getId).collect(Collectors.toList())); acceptedTasks.add(taskHolder); // subtract task resources from offer subtractResources(taskHolder.getMesosTask().getResourcesList()); // subtract executor resources from offer, if any are defined if (taskHolder.getMesosTask().hasExecutor() && taskHolder.getMesosTask().getExecutor().getResourcesCount() > 0) { subtractResources(taskHolder.getMesosTask().getExecutor().getResourcesList()); } }
private void match(List<SingularityTaskIdHistory> history, int num, SingularityTask... tasks) { Assert.assertEquals(num, history.size()); for (int i = 0; i < tasks.length; i++) { SingularityTaskIdHistory idHistory = history.get(i); SingularityTask task = tasks[i]; Assert.assertEquals(task.getTaskId(), idHistory.getTaskId()); } }
public static SingularityTaskState fromTaskHistory(SingularityTaskHistory taskHistory) { return new SingularityTaskState( Optional.of(taskHistory.getTask().getTaskId()), Optional.of(taskHistory.getTask().getTaskRequest().getPendingTask().getPendingTaskId()), taskHistory.getTask().getTaskRequest().getPendingTask().getRunId(), Optional.of(taskHistory.getLastTaskUpdate().get().getTaskState()), taskHistory.getTaskUpdates(), false ); }
private SingularityMesosTaskHolder acceptTask(SingularityOfferHolder offerHolder, Map<String, Integer> tasksPerOffer, SingularityTaskRequestHolder taskRequestHolder) { final SingularityTaskRequest taskRequest = taskRequestHolder.getTaskRequest(); final SingularityMesosTaskHolder taskHolder = mesosTaskBuilder.buildTask(offerHolder, offerHolder.getCurrentResources(), taskRequest, taskRequestHolder.getTaskResources(), taskRequestHolder.getExecutorResources()); final SingularityTask zkTask = taskSizeOptimizer.getSizeOptimizedTask(taskHolder); LOG.trace("Accepted and built task {}", zkTask); LOG.info("Launching task {} slot on slave {} ({})", taskHolder.getTask().getTaskId(), offerHolder.getSlaveId(), offerHolder.getHostname()); taskManager.createTaskAndDeletePendingTask(zkTask); addRequestToMapByOfferHost(tasksPerOffer, offerHolder.getHostname(), taskRequest.getRequest().getId()); return taskHolder; }
@Test public void testTaskKill() { initRequest(); initFirstDeploy(); SingularityTask firstTask = startTask(firstDeploy); taskResource.killTask(firstTask.getTaskId().getId(), Optional.absent(), singularityUser); cleaner.drainCleanupQueue(); killKilledTasks(); Assert.assertEquals(0, taskManager.getNumCleanupTasks()); Assert.assertEquals(0, taskManager.getNumActiveTasks()); }