private void cleanupTaskDueToDecomission(final Map<String, Optional<String>> requestIdsToUserToReschedule, final Set<SingularityTaskId> matchingTaskIds, SingularityTask task, SingularityMachineAbstraction<?> decommissioningObject) { requestIdsToUserToReschedule.put(task.getTaskRequest().getRequest().getId(), decommissioningObject.getCurrentState().getUser()); matchingTaskIds.add(task.getTaskId()); LOG.trace("Scheduling a cleanup task for {} due to decomissioning {}", task.getTaskId(), decommissioningObject); taskManager.createTaskCleanup(new SingularityTaskCleanup(decommissioningObject.getCurrentState().getUser(), TaskCleanupType.DECOMISSIONING, System.currentTimeMillis(), task.getTaskId(), Optional.of(String.format("%s %s is decomissioning", decommissioningObject.getTypeName(), decommissioningObject.getName())), Optional.<String>absent(), Optional.<SingularityTaskShellCommandRequestId>absent())); }
SingularityTask getSizeOptimizedTask(SingularityMesosTaskHolder taskHolder) { if (configuration.isStoreAllMesosTaskInfoForDebugging()) { return taskHolder.getTask(); } SingularityTask task = taskHolder.getTask(); TaskInfo.Builder mesosTask = taskHolder.getMesosTask().toBuilder(); mesosTask.clearData(); List<MesosOfferObject> offers = task.getOffers() .stream() .map(MesosOfferObject::sizeOptimized) .collect(Collectors.toList()); SingularityTaskRequest taskRequest = task.getTaskRequest(); if (task.getTaskRequest().getDeploy().getExecutorData().isPresent()) { SingularityDeployBuilder deploy = task.getTaskRequest().getDeploy().toBuilder(); deploy.setExecutorData(Optional.absent()); taskRequest = new SingularityTaskRequest(task.getTaskRequest().getRequest(), deploy.build(), task.getTaskRequest().getPendingTask()); } return new SingularityTask(taskRequest, task.getTaskId(), offers, mesosProtosUtils.taskFromProtos(mesosTask.build()), task.getRackId()); }
private void createTaskAndDeletePendingTaskPrivate(SingularityTask task) throws Exception { // TODO: Should more of the below be done within a transaction? deletePendingTask(task.getTaskRequest().getPendingTask().getPendingTaskId()); final long now = System.currentTimeMillis(); String msg = String.format("Task launched because of %s", task.getTaskRequest().getPendingTask().getPendingTaskId().getPendingType().name()); if (task.getTaskRequest().getPendingTask().getUser().isPresent()) { msg = String.format("%s by %s", msg, task.getTaskRequest().getPendingTask().getUser().get()); } if (task.getTaskRequest().getPendingTask().getMessage().isPresent()) { msg = String.format("%s (%s)", msg, task.getTaskRequest().getPendingTask().getMessage().get()); } saveTaskHistoryUpdate(new SingularityTaskHistoryUpdate(task.getTaskId(), now, ExtendedTaskState.TASK_LAUNCHED, Optional.of(msg), Optional.<String>absent())); saveLastActiveTaskStatus(new SingularityTaskStatusHolder(task.getTaskId(), Optional.absent(), now, serverId, Optional.of(task.getAgentId().getValue()))); try { final String path = getTaskPath(task.getTaskId()); CuratorTransactionFinal transaction = curator.inTransaction().create().forPath(path, taskTranscoder.toBytes(task)).and(); transaction.create().forPath(getActivePath(task.getTaskId().getId())).and().commit(); leaderCache.putActiveTask(task); taskCache.set(path, task); } catch (KeeperException.NodeExistsException nee) { LOG.error("Task or active path already existed for {}", task.getTaskId()); } }
private List<UpstreamInfo> tasksToUpstreams(List<SingularityTask> tasks, String requestId, Optional<String> loadBalancerUpstreamGroup) { final List<UpstreamInfo> upstreams = Lists.newArrayListWithCapacity(tasks.size()); for (SingularityTask task : tasks) { final Optional<Long> maybeLoadBalancerPort = MesosUtils.getPortByIndex(mesosProtosUtils.toResourceList(task.getMesosTask().getResources()), task.getTaskRequest().getDeploy().getLoadBalancerPortIndex().or(0)); if (maybeLoadBalancerPort.isPresent()) { String upstream = String.format("%s:%d", task.getHostname(), maybeLoadBalancerPort.get()); Optional<String> group = loadBalancerUpstreamGroup; if (taskLabelForLoadBalancerUpstreamGroup.isPresent()) { for (MesosParameter label : task.getMesosTask().getLabels().getLabels()) { if (label.hasKey() && label.getKey().equals(taskLabelForLoadBalancerUpstreamGroup.get()) && label.hasValue()) { group = Optional.of(label.getValue()); break; } } } upstreams.add(new UpstreamInfo(upstream, Optional.of(requestId), task.getRackId(), Optional.<String>absent(), group)); } else { LOG.warn("Task {} is missing port but is being passed to LB ({})", task.getTaskId(), task); } } return upstreams; }
@Override public String apply(@Nonnull SingularityTask input) { return input.getTaskRequest().getRequest().getId(); } };
private Optional<String> getHealthcheckUri(SingularityTask task) { if (!task.getTaskRequest().getDeploy().getHealthcheck().isPresent()) { return Optional.absent(); } HealthcheckOptions options = task.getTaskRequest().getDeploy().getHealthcheck().get(); final String hostname = task.getHostname(); Optional<Long> healthcheckPort = options.getPortNumber().or(MesosUtils.getPortByIndex(mesosProtosUtils.toResourceList(task.getMesosTask().getResources()), options.getPortIndex().or(0))); if (!healthcheckPort.isPresent() || healthcheckPort.get() < 1L) { return Optional.absent(); } if (!task.getTaskRequest().getDeploy().getHealthcheck().get().getUri().isPresent()) { return Optional.absent(); } String uri = task.getTaskRequest().getDeploy().getHealthcheck().get().getUri().get(); if (uri.startsWith("/")) { uri = uri.substring(1); } HealthcheckProtocol protocol = options.getProtocol().or(DEFAULT_HEALTH_CHECK_SCHEME); return Optional.of(String.format("%s://%s:%d/%s", protocol.getProtocol(), hostname, healthcheckPort.get(), uri)); }
@Test public void testUsagePollerSimple() { // works with no slaves usagePoller.runActionOnPoll(); cleaner.runActionOnPoll(); initRequest(); initFirstDeploy(); saveAndSchedule(request.toBuilder().setInstances(Optional.of(1))); resourceOffers(1); SingularityTask firstTask = taskManager.getActiveTasks().get(0); String hostname = firstTask.getHostname(); MesosTaskMonitorObject usage = getTaskMonitor(firstTask.getTaskId().getId(), 2, 5, 100); mesosClient.setSlaveResourceUsage(hostname, Collections.singletonList(usage)); usagePoller.runActionOnPoll(); String slaveId = firstTask.getAgentId().getValue(); List<String> slaves = usageManager.getSlavesWithUsage(); Assert.assertEquals(1, slaves.size()); Assert.assertEquals(slaves.get(0), slaveId); Assert.assertEquals(0, usageManager.getSlaveUsage(slaveId).get(0).getCpusUsed(), 0); Assert.assertEquals(100, usageManager.getSlaveUsage(slaveId).get(0).getMemoryBytesUsed(), 0); SingularityTaskUsage first = usageManager.getTaskUsage(firstTask.getTaskId().getId()).get(0); Assert.assertEquals(2, first.getCpuSeconds(), 0); Assert.assertEquals(100, first.getMemoryTotalBytes(), 0); Assert.assertEquals(5, first.getTimestamp(), 0); }
public void killAndRecord(SingularityTaskId taskId, Optional<RequestCleanupType> requestCleanupType, Optional<TaskCleanupType> taskCleanupType, Optional<Long> originalTimestamp, Optional<Integer> retries, Optional<String> user) { Preconditions.checkState(isRunning()); Optional<TaskCleanupType> maybeCleanupFromRequestAndTask = getTaskCleanupType(requestCleanupType, taskCleanupType); if (maybeCleanupFromRequestAndTask.isPresent() && (maybeCleanupFromRequestAndTask.get() == TaskCleanupType.USER_REQUESTED_DESTROY || maybeCleanupFromRequestAndTask.get() == TaskCleanupType.REQUEST_DELETING)) { Optional<SingularityTask> task = taskManager.getTask(taskId); if (task.isPresent()) { if (task.get().getTaskRequest().getDeploy().getCustomExecutorCmd().isPresent()) { byte[] messageBytes = transcoder.toBytes(new SingularityTaskDestroyFrameworkMessage(taskId, user)); mesosSchedulerClient.frameworkMessage( MesosProtosUtils.toExecutorId(task.get().getMesosTask().getExecutor().getExecutorId()), MesosProtosUtils.toAgentId(task.get().getMesosTask().getAgentId()), messageBytes ); } else { LOG.warn("Not using custom executor, will not send framework message to destroy task"); } } else { String message = String.format("No task data available to build kill task framework message for task %s", taskId); exceptionNotifier.notify(message); LOG.error(message); } } mesosSchedulerClient.kill(TaskID.newBuilder().setValue(taskId.toString()).build()); taskManager.saveKilledRecord(new SingularityKilledTaskIdRecord(taskId, System.currentTimeMillis(), originalTimestamp.or(System.currentTimeMillis()), requestCleanupType, taskCleanupType, retries.or(-1) + 1)); }
private void populateTaskEmailProperties(Map<String, Object> templateProperties, SingularityTaskId taskId, Collection<SingularityTaskHistoryUpdate> taskHistory, ExtendedTaskState taskState, List<SingularityTaskMetadata> taskMetadata, SingularityEmailType emailType) { Optional<SingularityTask> task = taskManager.getTask(taskId); Optional<String> directory = taskManager.getDirectory(taskId); templateProperties.put("singularityTaskLink", mailTemplateHelpers.getSingularityTaskLink(taskId.getId())); // Grab the tails of log files from remote mesos slaves. templateProperties.put("logTails", mailTemplateHelpers.getTaskLogs(taskId, task, directory)); templateProperties.put("taskId", taskId.getId()); templateProperties.put("deployId", taskId.getDeployId()); templateProperties.put("taskDirectory", directory.or("directory missing")); templateProperties.put("color", emailType.getColor()); if (task.isPresent()) { templateProperties.put("slaveHostname", task.get().getHostname()); if (task.get().getTaskRequest().getPendingTask().getCmdLineArgsList().isPresent()) { templateProperties.put("extraCmdLineArguments", task.get().getTaskRequest().getPendingTask().getCmdLineArgsList().get()); } } boolean needsBeenPrefix = taskState == ExtendedTaskState.TASK_LOST || taskState == ExtendedTaskState.TASK_KILLED; templateProperties.put("status", String.format("%s%s", needsBeenPrefix ? "has been " : "has ", taskState.getDisplayName())); templateProperties.put("taskStateLost", taskState == ExtendedTaskState.TASK_LOST); templateProperties.put("taskStateFailed", taskState == ExtendedTaskState.TASK_FAILED); templateProperties.put("taskStateFinished", taskState == ExtendedTaskState.TASK_FINISHED); templateProperties.put("taskStateKilled", taskState == ExtendedTaskState.TASK_KILLED); templateProperties.put("taskStateRunning", taskState == ExtendedTaskState.TASK_RUNNING); templateProperties.put("taskHasMetadata", !taskMetadata.isEmpty()); templateProperties.put("taskMetadata", mailTemplateHelpers.getJadeTaskMetadata(taskMetadata)); templateProperties.put("taskUpdates", mailTemplateHelpers.getJadeTaskHistory(taskHistory)); templateProperties.put("taskRan", mailTemplateHelpers.didTaskRun(taskHistory)); }
private void loadDirectoryAndContainer(SingularityTask task) { final long start = System.currentTimeMillis(); final String slaveUri = mesosClient.getSlaveUri(task.getHostname()); LOG.info("Fetching slave data to find log directory and container id for task {} from uri {}", task.getTaskId(), slaveUri); Optional<MesosExecutorObject> maybeExecutor = findExecutor(task.getTaskId(), slaveFramework.getExecutors()); if (maybeExecutor.isPresent()) { directory = Optional.of(maybeExecutor.get().getDirectory()); maybeExecutor = findExecutor(task.getTaskId(), slaveFramework.getCompletedExecutors()); if (maybeExecutor.isPresent()) { directory = Optional.of(maybeExecutor.get().getDirectory()); LOG.warn("Couldn't find matching executor for task {}", task.getTaskId()); return; LOG.debug("Found a directory {} and container id {} for task {}", directory.or(""), containerId.or(""), task.getTaskId()); taskManager.saveTaskDirectory(task.getTaskId(), directory.get()); taskManager.saveContainerId(task.getTaskId(), containerId.get()); LOG.trace("Updated task {} directory and container id in {}", task.getTaskId(), JavaUtils.duration(start));
taskResource.runShellCommand(singularityUser, task.getTaskId().getId(), new SingularityShellCommand("test-cmd", Optional.of(Arrays.asList("one", "two")), user, Optional.absent())); } catch (WebApplicationException exception) { assertEquals(400, exception.getResponse().getStatus()); taskResource.runShellCommand(singularityUser, task.getTaskId().getId(), new SingularityShellCommand("d1", Optional.of(Arrays.asList("one", "two")), user, Optional.absent())); } catch (WebApplicationException exception) { assertEquals(400, exception.getResponse().getStatus()); SingularityTaskShellCommandRequest firstShellRequest = taskResource.runShellCommand(singularityUser, task.getTaskId().getId(), new SingularityShellCommand("d1", Optional.of(Arrays.asList("o1", "o2")), user, Optional.absent())); SingularityTaskShellCommandRequest secondShellRequest = taskResource.runShellCommand(singularityUser, task.getTaskId().getId(), new SingularityShellCommand("d2", Optional.<List<String>> absent(), user, Optional.absent())); assertEquals(2, taskManager.getTaskShellCommandRequestsForTask(task.getTaskId()).size()); .setExecutorId(MesosProtosUtils.toExecutorId(task.getMesosTask().getExecutor().getExecutorId())) .setAgentId(MesosProtosUtils.toAgentId(task.getMesosTask().getAgentId())) .setData(ByteString.copyFrom(updateTranscoder.toBytes(new SingularityTaskShellCommandUpdate(firstShellRequest.getId(), System.currentTimeMillis(), Optional.<String> of("hi"), Optional.<String>absent(), UpdateType.STARTED)))) .build()); .setExecutorId(MesosProtosUtils.toExecutorId(task.getMesosTask().getExecutor().getExecutorId())) .setAgentId(MesosProtosUtils.toAgentId(task.getMesosTask().getAgentId())) .setData(ByteString.copyFrom(updateTranscoder.toBytes(new SingularityTaskShellCommandUpdate(new SingularityTaskShellCommandRequestId(task.getTaskId(), "wat", System.currentTimeMillis()), System.currentTimeMillis(), Optional.<String> of("hi"), Optional.<String>absent(), UpdateType.STARTED)))) .build()); .setExecutorId(MesosProtosUtils.toExecutorId(task.getMesosTask().getExecutor().getExecutorId())) .setAgentId(MesosProtosUtils.toAgentId(task.getMesosTask().getAgentId())) .setData(ByteString.copyFrom(updateTranscoder.toBytes(new SingularityTaskShellCommandUpdate(new SingularityTaskShellCommandRequestId(new SingularityTaskId("makingitup", "did", System.currentTimeMillis(), 1, "host", "rack"), "wat", System.currentTimeMillis()), System.currentTimeMillis(), Optional.<String> of("hi"), Optional.<String>absent(), UpdateType.STARTED)))) .build());
private long getNumInstancesWithAttribute(List<SingularityTaskId> taskIds, String attrKey, String attrValue) { return taskIds.stream() .map(id -> leaderCache.getSlave(taskManager.getTask(id).get().getMesosTask().getSlaveId().getValue()).get().getAttributes().get(attrKey)) .filter(Objects::nonNull) .filter(x -> x.equals(attrValue)) .count(); }
public List<SingularityTask> getTasksOnSlave(Collection<SingularityTaskId> activeTaskIds, SingularitySlave slave) { final List<SingularityTask> tasks = Lists.newArrayList(); final String sanitizedHost = JavaUtils.getReplaceHyphensWithUnderscores(slave.getHost()); for (SingularityTaskId activeTaskId : activeTaskIds) { if (activeTaskId.getSanitizedHost().equals(sanitizedHost)) { Optional<SingularityTask> maybeTask = getTask(activeTaskId); if (maybeTask.isPresent() && slave.getId().equals(maybeTask.get().getAgentId().getValue())) { tasks.add(maybeTask.get()); } } } return tasks; }
protected void statusUpdate(SingularityTask task, TaskState state, Optional<Long> timestamp) { TaskStatus.Builder bldr = TaskStatus.newBuilder() .setTaskId(MesosProtosUtils.toTaskId(task.getMesosTask().getTaskId())) .setAgentId(MesosProtosUtils.toAgentId(task.getAgentId())) .setState(state); if (timestamp.isPresent()) { bldr.setTimestamp(timestamp.get() / 1000); } sms.statusUpdate(bldr.build()).join(); }
@GET @Path("/task/{taskId}/statistics") @Operation( summary = "Retrieve resource usage statistics about a specific active task", responses = { @ApiResponse(responseCode = "404", description = "A task with this id, or slave and executor with matching statistics was not found") } ) public MesosTaskStatisticsObject getTaskStatistics( @Parameter(hidden = true) @Auth SingularityUser user, @Parameter(description = "Id of the task") @PathParam("taskId") String taskId) { SingularityTask task = checkActiveTask(taskId, SingularityAuthorizationScope.READ, user); String executorIdToMatch = null; if (task.getMesosTask().hasExecutor()) { executorIdToMatch = task.getMesosTask().getExecutor().getExecutorId().getValue(); } else { executorIdToMatch = taskId; } for (MesosTaskMonitorObject taskMonitor : mesosClient.getSlaveResourceUsage(task.getHostname())) { if (taskMonitor.getExecutorId().equals(executorIdToMatch)) { return taskMonitor.getStatistics(); } } throw notFound("Couldn't find executor %s for %s on slave %s", executorIdToMatch, taskId, task.getHostname()); }
final String slaveHostname = task.get().getHostname();
@Test public void testSchedulerRandomizesOffers() { initRequest(); initFirstDeploy(); requestResource.postRequest(request.toBuilder().setInstances(Optional.of(15)).build(), singularityUser); scheduler.drainPendingQueue(); sms.resourceOffers(Arrays.asList(createOffer(20, 1024, 20000), createOffer(20, 1024, 20000))); Assert.assertEquals(15, taskManager.getActiveTaskIds().size()); Set<String> offerIds = Sets.newHashSet(); for (SingularityTask activeTask : taskManager.getActiveTasks()) { offerIds.addAll(activeTask.getOffers().stream().map((o) -> o.getId().getValue()).collect(Collectors.toList())); } Assert.assertEquals(2, offerIds.size()); }
new SingularityTask(taskRequest, taskId, offerHolder.getOffers().stream().map((o) -> mesosProtosUtils.offerFromProtos(o)).collect(Collectors.toList()),
@Override public void run() { try { loadDirectoryAndContainer(task); } catch (Throwable t) { LOG.error("While fetching directory and container id for task: {}", task.getTaskId(), t); } } };