final String slaveHostname = task.get().getHostname();
private void populateTaskEmailProperties(Map<String, Object> templateProperties, SingularityTaskId taskId, Collection<SingularityTaskHistoryUpdate> taskHistory, ExtendedTaskState taskState, List<SingularityTaskMetadata> taskMetadata, SingularityEmailType emailType) { Optional<SingularityTask> task = taskManager.getTask(taskId); Optional<String> directory = taskManager.getDirectory(taskId); templateProperties.put("singularityTaskLink", mailTemplateHelpers.getSingularityTaskLink(taskId.getId())); // Grab the tails of log files from remote mesos slaves. templateProperties.put("logTails", mailTemplateHelpers.getTaskLogs(taskId, task, directory)); templateProperties.put("taskId", taskId.getId()); templateProperties.put("deployId", taskId.getDeployId()); templateProperties.put("taskDirectory", directory.or("directory missing")); templateProperties.put("color", emailType.getColor()); if (task.isPresent()) { templateProperties.put("slaveHostname", task.get().getHostname()); if (task.get().getTaskRequest().getPendingTask().getCmdLineArgsList().isPresent()) { templateProperties.put("extraCmdLineArguments", task.get().getTaskRequest().getPendingTask().getCmdLineArgsList().get()); } } boolean needsBeenPrefix = taskState == ExtendedTaskState.TASK_LOST || taskState == ExtendedTaskState.TASK_KILLED; templateProperties.put("status", String.format("%s%s", needsBeenPrefix ? "has been " : "has ", taskState.getDisplayName())); templateProperties.put("taskStateLost", taskState == ExtendedTaskState.TASK_LOST); templateProperties.put("taskStateFailed", taskState == ExtendedTaskState.TASK_FAILED); templateProperties.put("taskStateFinished", taskState == ExtendedTaskState.TASK_FINISHED); templateProperties.put("taskStateKilled", taskState == ExtendedTaskState.TASK_KILLED); templateProperties.put("taskStateRunning", taskState == ExtendedTaskState.TASK_RUNNING); templateProperties.put("taskHasMetadata", !taskMetadata.isEmpty()); templateProperties.put("taskMetadata", mailTemplateHelpers.getJadeTaskMetadata(taskMetadata)); templateProperties.put("taskUpdates", mailTemplateHelpers.getJadeTaskHistory(taskHistory)); templateProperties.put("taskRan", mailTemplateHelpers.didTaskRun(taskHistory)); }
private Optional<String> getHealthcheckUri(SingularityTask task) { if (!task.getTaskRequest().getDeploy().getHealthcheck().isPresent()) { return Optional.absent(); } HealthcheckOptions options = task.getTaskRequest().getDeploy().getHealthcheck().get(); final String hostname = task.getHostname(); Optional<Long> healthcheckPort = options.getPortNumber().or(MesosUtils.getPortByIndex(mesosProtosUtils.toResourceList(task.getMesosTask().getResources()), options.getPortIndex().or(0))); if (!healthcheckPort.isPresent() || healthcheckPort.get() < 1L) { return Optional.absent(); } if (!task.getTaskRequest().getDeploy().getHealthcheck().get().getUri().isPresent()) { return Optional.absent(); } String uri = task.getTaskRequest().getDeploy().getHealthcheck().get().getUri().get(); if (uri.startsWith("/")) { uri = uri.substring(1); } HealthcheckProtocol protocol = options.getProtocol().or(DEFAULT_HEALTH_CHECK_SCHEME); return Optional.of(String.format("%s://%s:%d/%s", protocol.getProtocol(), hostname, healthcheckPort.get(), uri)); }
private List<UpstreamInfo> tasksToUpstreams(List<SingularityTask> tasks, String requestId, Optional<String> loadBalancerUpstreamGroup) { final List<UpstreamInfo> upstreams = Lists.newArrayListWithCapacity(tasks.size()); for (SingularityTask task : tasks) { final Optional<Long> maybeLoadBalancerPort = MesosUtils.getPortByIndex(mesosProtosUtils.toResourceList(task.getMesosTask().getResources()), task.getTaskRequest().getDeploy().getLoadBalancerPortIndex().or(0)); if (maybeLoadBalancerPort.isPresent()) { String upstream = String.format("%s:%d", task.getHostname(), maybeLoadBalancerPort.get()); Optional<String> group = loadBalancerUpstreamGroup; if (taskLabelForLoadBalancerUpstreamGroup.isPresent()) { for (MesosParameter label : task.getMesosTask().getLabels().getLabels()) { if (label.hasKey() && label.getKey().equals(taskLabelForLoadBalancerUpstreamGroup.get()) && label.hasValue()) { group = Optional.of(label.getValue()); break; } } } upstreams.add(new UpstreamInfo(upstream, Optional.of(requestId), task.getRackId(), Optional.<String>absent(), group)); } else { LOG.warn("Task {} is missing port but is being passed to LB ({})", task.getTaskId(), task); } } return upstreams; }
final String slaveHostname = history.getTask().getHostname(); final String fullPath = new File(history.getDirectory().get(), path).toString();
private void loadDirectoryAndContainer(SingularityTask task) { final long start = System.currentTimeMillis(); final String slaveUri = mesosClient.getSlaveUri(task.getHostname());
final SingularityTaskHistory history = checkHistory(taskId, user); final String slaveHostname = history.getTask().getHostname(); final String pathToRoot = history.getDirectory().get(); final String fullPath = new File(pathToRoot, currentDirectory).toString();
@GET @Path("/task/{taskId}/statistics") @Operation( summary = "Retrieve resource usage statistics about a specific active task", responses = { @ApiResponse(responseCode = "404", description = "A task with this id, or slave and executor with matching statistics was not found") } ) public MesosTaskStatisticsObject getTaskStatistics( @Parameter(hidden = true) @Auth SingularityUser user, @Parameter(description = "Id of the task") @PathParam("taskId") String taskId) { SingularityTask task = checkActiveTask(taskId, SingularityAuthorizationScope.READ, user); String executorIdToMatch = null; if (task.getMesosTask().hasExecutor()) { executorIdToMatch = task.getMesosTask().getExecutor().getExecutorId().getValue(); } else { executorIdToMatch = taskId; } for (MesosTaskMonitorObject taskMonitor : mesosClient.getSlaveResourceUsage(task.getHostname())) { if (taskMonitor.getExecutorId().equals(executorIdToMatch)) { return taskMonitor.getStatistics(); } } throw notFound("Couldn't find executor %s for %s on slave %s", executorIdToMatch, taskId, task.getHostname()); }
@Test public void testUsagePollerSimple() { // works with no slaves usagePoller.runActionOnPoll(); cleaner.runActionOnPoll(); initRequest(); initFirstDeploy(); saveAndSchedule(request.toBuilder().setInstances(Optional.of(1))); resourceOffers(1); SingularityTask firstTask = taskManager.getActiveTasks().get(0); String hostname = firstTask.getHostname(); MesosTaskMonitorObject usage = getTaskMonitor(firstTask.getTaskId().getId(), 2, 5, 100); mesosClient.setSlaveResourceUsage(hostname, Collections.singletonList(usage)); usagePoller.runActionOnPoll(); String slaveId = firstTask.getAgentId().getValue(); List<String> slaves = usageManager.getSlavesWithUsage(); Assert.assertEquals(1, slaves.size()); Assert.assertEquals(slaves.get(0), slaveId); Assert.assertEquals(0, usageManager.getSlaveUsage(slaveId).get(0).getCpusUsed(), 0); Assert.assertEquals(100, usageManager.getSlaveUsage(slaveId).get(0).getMemoryBytesUsed(), 0); SingularityTaskUsage first = usageManager.getTaskUsage(firstTask.getTaskId().getId()).get(0); Assert.assertEquals(2, first.getCpuSeconds(), 0); Assert.assertEquals(100, first.getMemoryTotalBytes(), 0); Assert.assertEquals(5, first.getTimestamp(), 0); }
final String slaveHostname = task.get().getHostname();
private void populateTaskEmailProperties(Map<String, Object> templateProperties, SingularityTaskId taskId, Collection<SingularityTaskHistoryUpdate> taskHistory, ExtendedTaskState taskState, List<SingularityTaskMetadata> taskMetadata, SingularityEmailType emailType) { Optional<SingularityTask> task = taskManager.getTask(taskId); Optional<String> directory = taskManager.getDirectory(taskId); templateProperties.put("singularityTaskLink", mailTemplateHelpers.getSingularityTaskLink(taskId.getId())); // Grab the tails of log files from remote mesos slaves. templateProperties.put("logTails", mailTemplateHelpers.getTaskLogs(taskId, task, directory)); templateProperties.put("taskId", taskId.getId()); templateProperties.put("deployId", taskId.getDeployId()); templateProperties.put("taskDirectory", directory.or("directory missing")); templateProperties.put("color", emailType.getColor()); if (task.isPresent()) { templateProperties.put("slaveHostname", task.get().getHostname()); if (task.get().getTaskRequest().getPendingTask().getCmdLineArgsList().isPresent()) { templateProperties.put("extraCmdLineArguments", task.get().getTaskRequest().getPendingTask().getCmdLineArgsList().get()); } } boolean needsBeenPrefix = taskState == ExtendedTaskState.TASK_LOST || taskState == ExtendedTaskState.TASK_KILLED; templateProperties.put("status", String.format("%s%s", needsBeenPrefix ? "has been " : "has ", taskState.getDisplayName())); templateProperties.put("taskStateLost", taskState == ExtendedTaskState.TASK_LOST); templateProperties.put("taskStateFailed", taskState == ExtendedTaskState.TASK_FAILED); templateProperties.put("taskStateFinished", taskState == ExtendedTaskState.TASK_FINISHED); templateProperties.put("taskStateKilled", taskState == ExtendedTaskState.TASK_KILLED); templateProperties.put("taskStateRunning", taskState == ExtendedTaskState.TASK_RUNNING); templateProperties.put("taskHasMetadata", !taskMetadata.isEmpty()); templateProperties.put("taskMetadata", mailTemplateHelpers.getJadeTaskMetadata(taskMetadata)); templateProperties.put("taskUpdates", mailTemplateHelpers.getJadeTaskHistory(taskHistory)); templateProperties.put("taskRan", mailTemplateHelpers.didTaskRun(taskHistory)); }
final String slaveHostname = history.getTask().getHostname(); final String fullPath = new File(history.getDirectory().get(), path).toString();
private Optional<String> getHealthcheckUri(SingularityTask task) { if (!task.getTaskRequest().getDeploy().getHealthcheck().isPresent()) { return Optional.absent(); } HealthcheckOptions options = task.getTaskRequest().getDeploy().getHealthcheck().get(); final String hostname = task.getHostname(); Optional<Long> healthcheckPort = options.getPortNumber().or(MesosUtils.getPortByIndex(mesosProtosUtils.toResourceList(task.getMesosTask().getResources()), options.getPortIndex().or(0))); if (!healthcheckPort.isPresent() || healthcheckPort.get() < 1L) { return Optional.absent(); } if (!task.getTaskRequest().getDeploy().getHealthcheck().get().getUri().isPresent()) { return Optional.absent(); } String uri = task.getTaskRequest().getDeploy().getHealthcheck().get().getUri().get(); if (uri.startsWith("/")) { uri = uri.substring(1); } HealthcheckProtocol protocol = options.getProtocol().or(DEFAULT_HEALTH_CHECK_SCHEME); return Optional.of(String.format("%s://%s:%d/%s", protocol.getProtocol(), hostname, healthcheckPort.get(), uri)); }
private List<UpstreamInfo> tasksToUpstreams(List<SingularityTask> tasks, String requestId, Optional<String> loadBalancerUpstreamGroup) { final List<UpstreamInfo> upstreams = Lists.newArrayListWithCapacity(tasks.size()); for (SingularityTask task : tasks) { final Optional<Long> maybeLoadBalancerPort = MesosUtils.getPortByIndex(mesosProtosUtils.toResourceList(task.getMesosTask().getResources()), task.getTaskRequest().getDeploy().getLoadBalancerPortIndex().or(0)); if (maybeLoadBalancerPort.isPresent()) { String upstream = String.format("%s:%d", task.getHostname(), maybeLoadBalancerPort.get()); Optional<String> group = loadBalancerUpstreamGroup; if (taskLabelForLoadBalancerUpstreamGroup.isPresent()) { for (MesosParameter label : task.getMesosTask().getLabels().getLabels()) { if (label.hasKey() && label.getKey().equals(taskLabelForLoadBalancerUpstreamGroup.get()) && label.hasValue()) { group = Optional.of(label.getValue()); break; } } } upstreams.add(new UpstreamInfo(upstream, Optional.of(requestId), task.getRackId(), Optional.<String>absent(), group)); } else { LOG.warn("Task {} is missing port but is being passed to LB ({})", task.getTaskId(), task); } } return upstreams; }
private void loadDirectoryAndContainer(SingularityTask task) { final long start = System.currentTimeMillis(); final String slaveUri = mesosClient.getSlaveUri(task.getHostname());
final SingularityTaskHistory history = checkHistory(taskId, user); final String slaveHostname = history.getTask().getHostname(); final String pathToRoot = history.getDirectory().get(); final String fullPath = new File(pathToRoot, currentDirectory).toString();
@GET @Path("/task/{taskId}/statistics") @Operation( summary = "Retrieve resource usage statistics about a specific active task", responses = { @ApiResponse(responseCode = "404", description = "A task with this id, or slave and executor with matching statistics was not found") } ) public MesosTaskStatisticsObject getTaskStatistics( @Parameter(hidden = true) @Auth SingularityUser user, @Parameter(description = "Id of the task") @PathParam("taskId") String taskId) { SingularityTask task = checkActiveTask(taskId, SingularityAuthorizationScope.READ, user); String executorIdToMatch = null; if (task.getMesosTask().hasExecutor()) { executorIdToMatch = task.getMesosTask().getExecutor().getExecutorId().getValue(); } else { executorIdToMatch = taskId; } for (MesosTaskMonitorObject taskMonitor : mesosClient.getSlaveResourceUsage(task.getHostname())) { if (taskMonitor.getExecutorId().equals(executorIdToMatch)) { return taskMonitor.getStatistics(); } } throw notFound("Couldn't find executor %s for %s on slave %s", executorIdToMatch, taskId, task.getHostname()); }
@Test public void testUsagePollerSimple() { // works with no slaves usagePoller.runActionOnPoll(); cleaner.runActionOnPoll(); initRequest(); initFirstDeploy(); saveAndSchedule(request.toBuilder().setInstances(Optional.of(1))); resourceOffers(1); SingularityTask firstTask = taskManager.getActiveTasks().get(0); String hostname = firstTask.getHostname(); MesosTaskMonitorObject usage = getTaskMonitor(firstTask.getTaskId().getId(), 2, 5, 100); mesosClient.setSlaveResourceUsage(hostname, Collections.singletonList(usage)); usagePoller.runActionOnPoll(); String slaveId = firstTask.getAgentId().getValue(); List<String> slaves = usageManager.getSlavesWithUsage(); Assert.assertEquals(1, slaves.size()); Assert.assertEquals(slaves.get(0), slaveId); Assert.assertEquals(0, usageManager.getSlaveUsage(slaveId).get(0).getCpusUsed(), 0); Assert.assertEquals(100, usageManager.getSlaveUsage(slaveId).get(0).getMemoryBytesUsed(), 0); SingularityTaskUsage first = usageManager.getTaskUsage(firstTask.getTaskId().getId()).get(0); Assert.assertEquals(2, first.getCpuSeconds(), 0); Assert.assertEquals(100, first.getMemoryTotalBytes(), 0); Assert.assertEquals(5, first.getTimestamp(), 0); }