private long getNumInstancesWithAttribute(List<SingularityTaskId> taskIds, String attrKey, String attrValue) { return taskIds.stream() .map(id -> leaderCache.getSlave(taskManager.getTask(id).get().getMesosTask().getSlaveId().getValue()).get().getAttributes().get(attrKey)) .filter(Objects::nonNull) .filter(x -> x.equals(attrValue)) .count(); }
SingularitySlave slave = slaveManager.getSlave(taskManager.getTask(taskId).get().getMesosTask().getSlaveId().getValue()).get(); for (Entry<String, String> entry : slave.getAttributes().entrySet()) { attributeTaskMap
private List<UpstreamInfo> tasksToUpstreams(List<SingularityTask> tasks, String requestId, Optional<String> loadBalancerUpstreamGroup) { final List<UpstreamInfo> upstreams = Lists.newArrayListWithCapacity(tasks.size()); for (SingularityTask task : tasks) { final Optional<Long> maybeLoadBalancerPort = MesosUtils.getPortByIndex(mesosProtosUtils.toResourceList(task.getMesosTask().getResources()), task.getTaskRequest().getDeploy().getLoadBalancerPortIndex().or(0)); if (maybeLoadBalancerPort.isPresent()) { String upstream = String.format("%s:%d", task.getHostname(), maybeLoadBalancerPort.get()); Optional<String> group = loadBalancerUpstreamGroup; if (taskLabelForLoadBalancerUpstreamGroup.isPresent()) { for (MesosParameter label : task.getMesosTask().getLabels().getLabels()) { if (label.hasKey() && label.getKey().equals(taskLabelForLoadBalancerUpstreamGroup.get()) && label.hasValue()) { group = Optional.of(label.getValue()); break; } } } upstreams.add(new UpstreamInfo(upstream, Optional.of(requestId), task.getRackId(), Optional.<String>absent(), group)); } else { LOG.warn("Task {} is missing port but is being passed to LB ({})", task.getTaskId(), task); } } return upstreams; }
private Optional<String> getHealthcheckUri(SingularityTask task) { if (!task.getTaskRequest().getDeploy().getHealthcheck().isPresent()) { return Optional.absent(); } HealthcheckOptions options = task.getTaskRequest().getDeploy().getHealthcheck().get(); final String hostname = task.getHostname(); Optional<Long> healthcheckPort = options.getPortNumber().or(MesosUtils.getPortByIndex(mesosProtosUtils.toResourceList(task.getMesosTask().getResources()), options.getPortIndex().or(0))); if (!healthcheckPort.isPresent() || healthcheckPort.get() < 1L) { return Optional.absent(); } if (!task.getTaskRequest().getDeploy().getHealthcheck().get().getUri().isPresent()) { return Optional.absent(); } String uri = task.getTaskRequest().getDeploy().getHealthcheck().get().getUri().get(); if (uri.startsWith("/")) { uri = uri.substring(1); } HealthcheckProtocol protocol = options.getProtocol().or(DEFAULT_HEALTH_CHECK_SCHEME); return Optional.of(String.format("%s://%s:%d/%s", protocol.getProtocol(), hostname, healthcheckPort.get(), uri)); }
public void killAndRecord(SingularityTaskId taskId, Optional<RequestCleanupType> requestCleanupType, Optional<TaskCleanupType> taskCleanupType, Optional<Long> originalTimestamp, Optional<Integer> retries, Optional<String> user) { Preconditions.checkState(isRunning()); Optional<TaskCleanupType> maybeCleanupFromRequestAndTask = getTaskCleanupType(requestCleanupType, taskCleanupType); if (maybeCleanupFromRequestAndTask.isPresent() && (maybeCleanupFromRequestAndTask.get() == TaskCleanupType.USER_REQUESTED_DESTROY || maybeCleanupFromRequestAndTask.get() == TaskCleanupType.REQUEST_DELETING)) { Optional<SingularityTask> task = taskManager.getTask(taskId); if (task.isPresent()) { if (task.get().getTaskRequest().getDeploy().getCustomExecutorCmd().isPresent()) { byte[] messageBytes = transcoder.toBytes(new SingularityTaskDestroyFrameworkMessage(taskId, user)); mesosSchedulerClient.frameworkMessage( MesosProtosUtils.toExecutorId(task.get().getMesosTask().getExecutor().getExecutorId()), MesosProtosUtils.toAgentId(task.get().getMesosTask().getAgentId()), messageBytes ); } else { LOG.warn("Not using custom executor, will not send framework message to destroy task"); } } else { String message = String.format("No task data available to build kill task framework message for task %s", taskId); exceptionNotifier.notify(message); LOG.error(message); } } mesosSchedulerClient.kill(TaskID.newBuilder().setValue(taskId.toString()).build()); taskManager.saveKilledRecord(new SingularityKilledTaskIdRecord(taskId, System.currentTimeMillis(), originalTimestamp.or(System.currentTimeMillis()), requestCleanupType, taskCleanupType, retries.or(-1) + 1)); }
@Override public void runActionOnPoll() { final long start = System.currentTimeMillis(); final List<SingularityTaskShellCommandRequest> shellRequests = taskManager.getAllQueuedTaskShellCommandRequests(); if (!schedulerClient.isRunning()) { LOG.warn("Unable to process shell requests because scheduler driver isn't present ({} tasks waiting)", shellRequests.size()); return; } if (shellRequests.isEmpty()) { LOG.trace("No shell requests to send."); return; } for (SingularityTaskShellCommandRequest shellRequest : shellRequests) { Optional<SingularityTask> task = taskManager.getTask(shellRequest.getTaskId()); if (!task.isPresent() || !taskManager.isActiveTask(shellRequest.getTaskId().getId())) { LOG.info("Skipping shell request {} because {} didn't exist or isn't active", shellRequest, shellRequest.getTaskId()); continue; } final ExecutorID executorId = MesosProtosUtils.toExecutorId(task.get().getMesosTask().getExecutor().getExecutorId()); final AgentID slaveId = MesosProtosUtils.toAgentId(task.get().getMesosTask().getAgentId()); final byte[] bytes = transcoder.toBytes(shellRequest); schedulerClient.frameworkMessage(executorId, slaveId, bytes); LOG.info("Sent {} ({} bytes) to {} on {}", shellRequest, bytes.length, executorId, slaveId); taskManager.saveTaskShellCommandRequestToTask(shellRequest); taskManager.deleteTaskShellCommandRequestFromQueue(shellRequest); } LOG.info("Sent {} shell requests to executors in {}", shellRequests.size(), JavaUtils.duration(start)); }
@GET @Path("/task/{taskId}/statistics") @Operation( summary = "Retrieve resource usage statistics about a specific active task", responses = { @ApiResponse(responseCode = "404", description = "A task with this id, or slave and executor with matching statistics was not found") } ) public MesosTaskStatisticsObject getTaskStatistics( @Parameter(hidden = true) @Auth SingularityUser user, @Parameter(description = "Id of the task") @PathParam("taskId") String taskId) { SingularityTask task = checkActiveTask(taskId, SingularityAuthorizationScope.READ, user); String executorIdToMatch = null; if (task.getMesosTask().hasExecutor()) { executorIdToMatch = task.getMesosTask().getExecutor().getExecutorId().getValue(); } else { executorIdToMatch = taskId; } for (MesosTaskMonitorObject taskMonitor : mesosClient.getSlaveResourceUsage(task.getHostname())) { if (taskMonitor.getExecutorId().equals(executorIdToMatch)) { return taskMonitor.getStatistics(); } } throw notFound("Couldn't find executor %s for %s on slave %s", executorIdToMatch, taskId, task.getHostname()); }
protected void statusUpdate(SingularityTask task, TaskState state, Optional<Long> timestamp) { TaskStatus.Builder bldr = TaskStatus.newBuilder() .setTaskId(MesosProtosUtils.toTaskId(task.getMesosTask().getTaskId())) .setAgentId(MesosProtosUtils.toAgentId(task.getAgentId())) .setState(state); if (timestamp.isPresent()) { bldr.setTimestamp(timestamp.get() / 1000); } sms.statusUpdate(bldr.build()).join(); }
@Test public void testDeployWithImmediateRunIsLaunchedImmediately() { initRequestWithType(RequestType.SCHEDULED, false); String deployId = "d1"; SingularityRunNowRequest runNowRequest = new SingularityRunNowRequestBuilder().setMessage("Message").build(); SingularityDeploy deploy = new SingularityDeployBuilder(requestId, deployId) .setRunImmediately(Optional.of(runNowRequest)) .setCommand(Optional.of("printenv > tmp.txt")) .build(); SingularityDeployRequest deployRequest = new SingularityDeployRequest(deploy, Optional.absent(), Optional.absent()); deployResource.deploy(deployRequest, singularityUser); deployChecker.checkDeploys(); scheduler.drainPendingQueue(); resourceOffers(); Assert.assertEquals(1, taskManager.getNumActiveTasks()); Assert.assertEquals(0, taskManager.getNumScheduledTasks()); SingularityTaskId taskId = taskManager.getActiveTaskIdsForDeploy(requestId, deployId).get(0); SingularityTask task = taskManager.getTask(taskId).get(); Map<String, Object> command = (Map<String, Object>) task.getMesosTask().getAllOtherFields().get("command"); Assert.assertEquals("printenv > tmp.txt", (String) command.get("value")); }
@Test public void testCustomResourcesWithRunNowRequest() { SingularityRequestBuilder bldr = new SingularityRequestBuilder(requestId, RequestType.ON_DEMAND); requestResource.postRequest(bldr.build(), singularityUser); deploy("d2"); SingularityRunNowRequest runNowRequest = new SingularityRunNowRequestBuilder() .setResources(new Resources(2, 2, 0)) .build(); requestResource.scheduleImmediately(singularityUser, requestId, runNowRequest); scheduler.drainPendingQueue(); SingularityPendingTask pendingTaskWithResourcs = taskManager.getPendingTasks().get(0); Assert.assertTrue(pendingTaskWithResourcs.getResources().isPresent()); Assert.assertEquals(pendingTaskWithResourcs.getResources().get().getCpus(), 2, 0.0); sms.resourceOffers(Arrays.asList(createOffer(5, 5, 5, "slave1", "host1", Optional.of("rack1")))); SingularityTask task = taskManager.getActiveTasks().get(0); Assert.assertEquals(MesosUtils.getNumCpus(mesosProtosUtils.toResourceList(task.getMesosTask().getResources()), Optional.<String>absent()), 2.0, 0.0); }
@Test public void testAcceptOffersWithRoleForRequestWithRole() { SingularityRequestBuilder bldr = new SingularityRequestBuilder(requestId, RequestType.ON_DEMAND); bldr.setRequiredRole(Optional.of("test-role")); requestResource.postRequest(bldr.build(), singularityUser); deploy("d2"); SingularityRunNowRequest runNowRequest = new SingularityRunNowRequestBuilder() .setResources(new Resources(2, 2, 0)) .build(); requestResource.scheduleImmediately(singularityUser, requestId, runNowRequest); scheduler.drainPendingQueue(); SingularityPendingTask pendingTaskWithResources = taskManager.getPendingTasks().get(0); Assert.assertTrue(pendingTaskWithResources.getResources().isPresent()); Assert.assertEquals(pendingTaskWithResources.getResources().get().getCpus(), 2, 0.0); sms.resourceOffers(Arrays.asList(createOffer(5, 5, 5))); pendingTaskWithResources = taskManager.getPendingTasks().get(0); Assert.assertTrue(pendingTaskWithResources.getResources().isPresent()); Assert.assertEquals(pendingTaskWithResources.getResources().get().getCpus(), 2, 0.0); sms.resourceOffers(Arrays.asList(createOffer(5, 5, 5, Optional.of("test-role")))); SingularityTask task = taskManager.getActiveTasks().get(0); Assert.assertEquals(MesosUtils.getNumCpus(mesosProtosUtils.toResourceList(task.getMesosTask().getResources()), Optional.of("test-role")), 2.0, 0.0); }
.setExecutorId(MesosProtosUtils.toExecutorId(task.getMesosTask().getExecutor().getExecutorId())) .setAgentId(MesosProtosUtils.toAgentId(task.getMesosTask().getAgentId())) .setData(ByteString.copyFrom(updateTranscoder.toBytes(new SingularityTaskShellCommandUpdate(firstShellRequest.getId(), System.currentTimeMillis(), Optional.<String> of("hi"), Optional.<String>absent(), UpdateType.STARTED)))) .build()); .setExecutorId(MesosProtosUtils.toExecutorId(task.getMesosTask().getExecutor().getExecutorId())) .setAgentId(MesosProtosUtils.toAgentId(task.getMesosTask().getAgentId())) .setData(ByteString.copyFrom(updateTranscoder.toBytes(new SingularityTaskShellCommandUpdate(new SingularityTaskShellCommandRequestId(task.getTaskId(), "wat", System.currentTimeMillis()), System.currentTimeMillis(), Optional.<String> of("hi"), Optional.<String>absent(), UpdateType.STARTED)))) .build()); .setExecutorId(MesosProtosUtils.toExecutorId(task.getMesosTask().getExecutor().getExecutorId())) .setAgentId(MesosProtosUtils.toAgentId(task.getMesosTask().getAgentId())) .setData(ByteString.copyFrom(updateTranscoder.toBytes(new SingularityTaskShellCommandUpdate(new SingularityTaskShellCommandRequestId(new SingularityTaskId("makingitup", "did", System.currentTimeMillis(), 1, "host", "rack"), "wat", System.currentTimeMillis()), System.currentTimeMillis(), Optional.<String> of("hi"), Optional.<String>absent(), UpdateType.STARTED)))) .build());
private long getNumInstancesWithAttribute(List<SingularityTaskId> taskIds, String attrKey, String attrValue) { return taskIds.stream() .map(id -> leaderCache.getSlave(taskManager.getTask(id).get().getMesosTask().getSlaveId().getValue()).get().getAttributes().get(attrKey)) .filter(Objects::nonNull) .filter(x -> x.equals(attrValue)) .count(); }
private List<UpstreamInfo> tasksToUpstreams(List<SingularityTask> tasks, String requestId, Optional<String> loadBalancerUpstreamGroup) { final List<UpstreamInfo> upstreams = Lists.newArrayListWithCapacity(tasks.size()); for (SingularityTask task : tasks) { final Optional<Long> maybeLoadBalancerPort = MesosUtils.getPortByIndex(mesosProtosUtils.toResourceList(task.getMesosTask().getResources()), task.getTaskRequest().getDeploy().getLoadBalancerPortIndex().or(0)); if (maybeLoadBalancerPort.isPresent()) { String upstream = String.format("%s:%d", task.getHostname(), maybeLoadBalancerPort.get()); Optional<String> group = loadBalancerUpstreamGroup; if (taskLabelForLoadBalancerUpstreamGroup.isPresent()) { for (MesosParameter label : task.getMesosTask().getLabels().getLabels()) { if (label.hasKey() && label.getKey().equals(taskLabelForLoadBalancerUpstreamGroup.get()) && label.hasValue()) { group = Optional.of(label.getValue()); break; } } } upstreams.add(new UpstreamInfo(upstream, Optional.of(requestId), task.getRackId(), Optional.<String>absent(), group)); } else { LOG.warn("Task {} is missing port but is being passed to LB ({})", task.getTaskId(), task); } } return upstreams; }
private Optional<String> getHealthcheckUri(SingularityTask task) { if (!task.getTaskRequest().getDeploy().getHealthcheck().isPresent()) { return Optional.absent(); } HealthcheckOptions options = task.getTaskRequest().getDeploy().getHealthcheck().get(); final String hostname = task.getHostname(); Optional<Long> healthcheckPort = options.getPortNumber().or(MesosUtils.getPortByIndex(mesosProtosUtils.toResourceList(task.getMesosTask().getResources()), options.getPortIndex().or(0))); if (!healthcheckPort.isPresent() || healthcheckPort.get() < 1L) { return Optional.absent(); } if (!task.getTaskRequest().getDeploy().getHealthcheck().get().getUri().isPresent()) { return Optional.absent(); } String uri = task.getTaskRequest().getDeploy().getHealthcheck().get().getUri().get(); if (uri.startsWith("/")) { uri = uri.substring(1); } HealthcheckProtocol protocol = options.getProtocol().or(DEFAULT_HEALTH_CHECK_SCHEME); return Optional.of(String.format("%s://%s:%d/%s", protocol.getProtocol(), hostname, healthcheckPort.get(), uri)); }
public void killAndRecord(SingularityTaskId taskId, Optional<RequestCleanupType> requestCleanupType, Optional<TaskCleanupType> taskCleanupType, Optional<Long> originalTimestamp, Optional<Integer> retries, Optional<String> user) { Preconditions.checkState(isRunning()); Optional<TaskCleanupType> maybeCleanupFromRequestAndTask = getTaskCleanupType(requestCleanupType, taskCleanupType); if (maybeCleanupFromRequestAndTask.isPresent() && (maybeCleanupFromRequestAndTask.get() == TaskCleanupType.USER_REQUESTED_DESTROY || maybeCleanupFromRequestAndTask.get() == TaskCleanupType.REQUEST_DELETING)) { Optional<SingularityTask> task = taskManager.getTask(taskId); if (task.isPresent()) { if (task.get().getTaskRequest().getDeploy().getCustomExecutorCmd().isPresent()) { byte[] messageBytes = transcoder.toBytes(new SingularityTaskDestroyFrameworkMessage(taskId, user)); mesosSchedulerClient.frameworkMessage( MesosProtosUtils.toExecutorId(task.get().getMesosTask().getExecutor().getExecutorId()), MesosProtosUtils.toAgentId(task.get().getMesosTask().getAgentId()), messageBytes ); } else { LOG.warn("Not using custom executor, will not send framework message to destroy task"); } } else { String message = String.format("No task data available to build kill task framework message for task %s", taskId); exceptionNotifier.notify(message); LOG.error(message); } } mesosSchedulerClient.kill(TaskID.newBuilder().setValue(taskId.toString()).build()); taskManager.saveKilledRecord(new SingularityKilledTaskIdRecord(taskId, System.currentTimeMillis(), originalTimestamp.or(System.currentTimeMillis()), requestCleanupType, taskCleanupType, retries.or(-1) + 1)); }
protected void statusUpdate(SingularityTask task, TaskState state, Optional<Long> timestamp) { TaskStatus.Builder bldr = TaskStatus.newBuilder() .setTaskId(MesosProtosUtils.toTaskId(task.getMesosTask().getTaskId())) .setAgentId(MesosProtosUtils.toAgentId(task.getAgentId())) .setState(state); if (timestamp.isPresent()) { bldr.setTimestamp(timestamp.get() / 1000); } sms.statusUpdate(bldr.build()).join(); }
@Test public void testDeployWithImmediateRunIsLaunchedImmediately() { initRequestWithType(RequestType.SCHEDULED, false); String deployId = "d1"; SingularityRunNowRequest runNowRequest = new SingularityRunNowRequestBuilder().setMessage("Message").build(); SingularityDeploy deploy = new SingularityDeployBuilder(requestId, deployId) .setRunImmediately(Optional.of(runNowRequest)) .setCommand(Optional.of("printenv > tmp.txt")) .build(); SingularityDeployRequest deployRequest = new SingularityDeployRequest(deploy, Optional.absent(), Optional.absent()); deployResource.deploy(deployRequest, singularityUser); deployChecker.checkDeploys(); scheduler.drainPendingQueue(); resourceOffers(); Assert.assertEquals(1, taskManager.getNumActiveTasks()); Assert.assertEquals(0, taskManager.getNumScheduledTasks()); SingularityTaskId taskId = taskManager.getActiveTaskIdsForDeploy(requestId, deployId).get(0); SingularityTask task = taskManager.getTask(taskId).get(); Map<String, Object> command = (Map<String, Object>) task.getMesosTask().getAllOtherFields().get("command"); Assert.assertEquals("printenv > tmp.txt", (String) command.get("value")); }
@Test public void testCustomResourcesWithRunNowRequest() { SingularityRequestBuilder bldr = new SingularityRequestBuilder(requestId, RequestType.ON_DEMAND); requestResource.postRequest(bldr.build(), singularityUser); deploy("d2"); SingularityRunNowRequest runNowRequest = new SingularityRunNowRequestBuilder() .setResources(new Resources(2, 2, 0)) .build(); requestResource.scheduleImmediately(singularityUser, requestId, runNowRequest); scheduler.drainPendingQueue(); SingularityPendingTask pendingTaskWithResourcs = taskManager.getPendingTasks().get(0); Assert.assertTrue(pendingTaskWithResourcs.getResources().isPresent()); Assert.assertEquals(pendingTaskWithResourcs.getResources().get().getCpus(), 2, 0.0); sms.resourceOffers(Arrays.asList(createOffer(5, 5, 5, "slave1", "host1", Optional.of("rack1")))); SingularityTask task = taskManager.getActiveTasks().get(0); Assert.assertEquals(MesosUtils.getNumCpus(mesosProtosUtils.toResourceList(task.getMesosTask().getResources()), Optional.<String>absent()), 2.0, 0.0); }
@Test public void testAcceptOffersWithRoleForRequestWithRole() { SingularityRequestBuilder bldr = new SingularityRequestBuilder(requestId, RequestType.ON_DEMAND); bldr.setRequiredRole(Optional.of("test-role")); requestResource.postRequest(bldr.build(), singularityUser); deploy("d2"); SingularityRunNowRequest runNowRequest = new SingularityRunNowRequestBuilder() .setResources(new Resources(2, 2, 0)) .build(); requestResource.scheduleImmediately(singularityUser, requestId, runNowRequest); scheduler.drainPendingQueue(); SingularityPendingTask pendingTaskWithResources = taskManager.getPendingTasks().get(0); Assert.assertTrue(pendingTaskWithResources.getResources().isPresent()); Assert.assertEquals(pendingTaskWithResources.getResources().get().getCpus(), 2, 0.0); sms.resourceOffers(Arrays.asList(createOffer(5, 5, 5))); pendingTaskWithResources = taskManager.getPendingTasks().get(0); Assert.assertTrue(pendingTaskWithResources.getResources().isPresent()); Assert.assertEquals(pendingTaskWithResources.getResources().get().getCpus(), 2, 0.0); sms.resourceOffers(Arrays.asList(createOffer(5, 5, 5, Optional.of("test-role")))); SingularityTask task = taskManager.getActiveTasks().get(0); Assert.assertEquals(MesosUtils.getNumCpus(mesosProtosUtils.toResourceList(task.getMesosTask().getResources()), Optional.of("test-role")), 2.0, 0.0); }