public List<SingularityTask> getTasksOnSlave(Collection<SingularityTaskId> activeTaskIds, SingularitySlave slave) { final List<SingularityTask> tasks = Lists.newArrayList(); final String sanitizedHost = JavaUtils.getReplaceHyphensWithUnderscores(slave.getHost()); for (SingularityTaskId activeTaskId : activeTaskIds) { if (activeTaskId.getSanitizedHost().equals(sanitizedHost)) { Optional<SingularityTask> maybeTask = getTask(activeTaskId); if (maybeTask.isPresent() && slave.getId().equals(maybeTask.get().getAgentId().getValue())) { tasks.add(maybeTask.get()); } } } return tasks; }
private boolean hasTaskLeftOnSlave(SingularityTaskId taskId, String slaveId, SingularityLeaderCache stateCache) { for (SingularityTaskId activeTaskId : stateCache.getActiveTaskIds()) { if (!activeTaskId.equals(taskId) && activeTaskId.getSanitizedHost().equals(taskId.getSanitizedHost())) { Optional<SingularityTask> maybeTask = taskManager.getTask(activeTaskId); if (maybeTask.isPresent() && slaveId.equals(maybeTask.get().getAgentId().getValue())) { return true; } } } return false; }
@Override public void applyMigration() { final long start = System.currentTimeMillis(); final List<SingularityTaskId> taskIds = taskManager.getActiveTaskIds(); for (SingularityTaskId taskId : taskIds) { List<SingularityTaskHistoryUpdate> updates = Lists.reverse(taskManager.getTaskHistoryUpdates(taskId)); Optional<MesosTaskStatusObject> taskStatus = Optional.absent(); for (SingularityTaskHistoryUpdate update : updates) { if (update.getTaskState().toTaskState().isPresent()) { Optional<SingularityTask> task = taskManager.getTask(taskId); taskStatus = Optional.of(mesosProtosUtils.taskStatusFromProtos(TaskStatus.newBuilder() .setTaskId(TaskID.newBuilder().setValue(taskId.getId())) .setAgentId(MesosProtosUtils.toAgentId(task.get().getAgentId())) .setState(MesosProtosUtils.toTaskState(update.getTaskState())) .build())); break; } } SingularityTaskStatusHolder taskStatusHolder = new SingularityTaskStatusHolder(taskId, taskStatus, start, serverId, Optional.absent()); taskManager.saveLastActiveTaskStatus(taskStatusHolder); } }
private void createTaskAndDeletePendingTaskPrivate(SingularityTask task) throws Exception { // TODO: Should more of the below be done within a transaction? deletePendingTask(task.getTaskRequest().getPendingTask().getPendingTaskId()); final long now = System.currentTimeMillis(); String msg = String.format("Task launched because of %s", task.getTaskRequest().getPendingTask().getPendingTaskId().getPendingType().name()); if (task.getTaskRequest().getPendingTask().getUser().isPresent()) { msg = String.format("%s by %s", msg, task.getTaskRequest().getPendingTask().getUser().get()); } if (task.getTaskRequest().getPendingTask().getMessage().isPresent()) { msg = String.format("%s (%s)", msg, task.getTaskRequest().getPendingTask().getMessage().get()); } saveTaskHistoryUpdate(new SingularityTaskHistoryUpdate(task.getTaskId(), now, ExtendedTaskState.TASK_LAUNCHED, Optional.of(msg), Optional.<String>absent())); saveLastActiveTaskStatus(new SingularityTaskStatusHolder(task.getTaskId(), Optional.absent(), now, serverId, Optional.of(task.getAgentId().getValue()))); try { final String path = getTaskPath(task.getTaskId()); CuratorTransactionFinal transaction = curator.inTransaction().create().forPath(path, taskTranscoder.toBytes(task)).and(); transaction.create().forPath(getActivePath(task.getTaskId().getId())).and().commit(); leaderCache.putActiveTask(task); taskCache.set(path, task); } catch (KeeperException.NodeExistsException nee) { LOG.error("Task or active path already existed for {}", task.getTaskId()); } }
protected void statusUpdate(SingularityTask task, TaskState state, Optional<Long> timestamp) { TaskStatus.Builder bldr = TaskStatus.newBuilder() .setTaskId(MesosProtosUtils.toTaskId(task.getMesosTask().getTaskId())) .setAgentId(MesosProtosUtils.toAgentId(task.getAgentId())) .setState(state); if (timestamp.isPresent()) { bldr.setTimestamp(timestamp.get() / 1000); } sms.statusUpdate(bldr.build()).join(); }
@Test public void testDecommissionDoesntKillPendingDeploy() { initRequest(); deployResource.deploy(new SingularityDeployRequest(new SingularityDeployBuilder(requestId, "d1").setCommand(Optional.of("cmd")).build(), Optional.absent(), Optional.absent()), singularityUser); scheduler.drainPendingQueue(); deployChecker.checkDeploys(); resourceOffers(); Assert.assertEquals(1, taskManager.getNumActiveTasks()); slaveResource.decommissionSlave(singularityUser, taskManager.getActiveTasks().get(0).getAgentId().getValue(), null); scheduler.checkForDecomissions(); cleaner.drainCleanupQueue(); killKilledTasks(); Assert.assertEquals(1, taskManager.getNumActiveTasks()); Assert.assertEquals(1, taskManager.getNumCleanupTasks()); Assert.assertEquals(0, taskManager.getKilledTaskIdRecords().size()); configuration.setPendingDeployHoldTaskDuringDecommissionMillis(1); try { Thread.sleep(2); } catch (InterruptedException e) {} cleaner.drainCleanupQueue(); killKilledTasks(); Assert.assertEquals(0, taskManager.getNumActiveTasks()); Assert.assertEquals(0, taskManager.getNumCleanupTasks()); }
@Test public void testUsagePollerSimple() { // works with no slaves usagePoller.runActionOnPoll(); cleaner.runActionOnPoll(); initRequest(); initFirstDeploy(); saveAndSchedule(request.toBuilder().setInstances(Optional.of(1))); resourceOffers(1); SingularityTask firstTask = taskManager.getActiveTasks().get(0); String hostname = firstTask.getHostname(); MesosTaskMonitorObject usage = getTaskMonitor(firstTask.getTaskId().getId(), 2, 5, 100); mesosClient.setSlaveResourceUsage(hostname, Collections.singletonList(usage)); usagePoller.runActionOnPoll(); String slaveId = firstTask.getAgentId().getValue(); List<String> slaves = usageManager.getSlavesWithUsage(); Assert.assertEquals(1, slaves.size()); Assert.assertEquals(slaves.get(0), slaveId); Assert.assertEquals(0, usageManager.getSlaveUsage(slaveId).get(0).getCpusUsed(), 0); Assert.assertEquals(100, usageManager.getSlaveUsage(slaveId).get(0).getMemoryBytesUsed(), 0); SingularityTaskUsage first = usageManager.getTaskUsage(firstTask.getTaskId().getId()).get(0); Assert.assertEquals(2, first.getCpuSeconds(), 0); Assert.assertEquals(100, first.getMemoryTotalBytes(), 0); Assert.assertEquals(5, first.getTimestamp(), 0); }
public List<SingularityTask> getTasksOnSlave(Collection<SingularityTaskId> activeTaskIds, SingularitySlave slave) { final List<SingularityTask> tasks = Lists.newArrayList(); final String sanitizedHost = JavaUtils.getReplaceHyphensWithUnderscores(slave.getHost()); for (SingularityTaskId activeTaskId : activeTaskIds) { if (activeTaskId.getSanitizedHost().equals(sanitizedHost)) { Optional<SingularityTask> maybeTask = getTask(activeTaskId); if (maybeTask.isPresent() && slave.getId().equals(maybeTask.get().getAgentId().getValue())) { tasks.add(maybeTask.get()); } } } return tasks; }
private boolean hasTaskLeftOnSlave(SingularityTaskId taskId, String slaveId, SingularityLeaderCache stateCache) { for (SingularityTaskId activeTaskId : stateCache.getActiveTaskIds()) { if (!activeTaskId.equals(taskId) && activeTaskId.getSanitizedHost().equals(taskId.getSanitizedHost())) { Optional<SingularityTask> maybeTask = taskManager.getTask(activeTaskId); if (maybeTask.isPresent() && slaveId.equals(maybeTask.get().getAgentId().getValue())) { return true; } } } return false; }
@Override public void applyMigration() { final long start = System.currentTimeMillis(); final List<SingularityTaskId> taskIds = taskManager.getActiveTaskIds(); for (SingularityTaskId taskId : taskIds) { List<SingularityTaskHistoryUpdate> updates = Lists.reverse(taskManager.getTaskHistoryUpdates(taskId)); Optional<MesosTaskStatusObject> taskStatus = Optional.absent(); for (SingularityTaskHistoryUpdate update : updates) { if (update.getTaskState().toTaskState().isPresent()) { Optional<SingularityTask> task = taskManager.getTask(taskId); taskStatus = Optional.of(mesosProtosUtils.taskStatusFromProtos(TaskStatus.newBuilder() .setTaskId(TaskID.newBuilder().setValue(taskId.getId())) .setAgentId(MesosProtosUtils.toAgentId(task.get().getAgentId())) .setState(MesosProtosUtils.toTaskState(update.getTaskState())) .build())); break; } } SingularityTaskStatusHolder taskStatusHolder = new SingularityTaskStatusHolder(taskId, taskStatus, start, serverId, Optional.absent()); taskManager.saveLastActiveTaskStatus(taskStatusHolder); } }
private void createTaskAndDeletePendingTaskPrivate(SingularityTask task) throws Exception { // TODO: Should more of the below be done within a transaction? deletePendingTask(task.getTaskRequest().getPendingTask().getPendingTaskId()); final long now = System.currentTimeMillis(); String msg = String.format("Task launched because of %s", task.getTaskRequest().getPendingTask().getPendingTaskId().getPendingType().name()); if (task.getTaskRequest().getPendingTask().getUser().isPresent()) { msg = String.format("%s by %s", msg, task.getTaskRequest().getPendingTask().getUser().get()); } if (task.getTaskRequest().getPendingTask().getMessage().isPresent()) { msg = String.format("%s (%s)", msg, task.getTaskRequest().getPendingTask().getMessage().get()); } saveTaskHistoryUpdate(new SingularityTaskHistoryUpdate(task.getTaskId(), now, ExtendedTaskState.TASK_LAUNCHED, Optional.of(msg), Optional.<String>absent())); saveLastActiveTaskStatus(new SingularityTaskStatusHolder(task.getTaskId(), Optional.absent(), now, serverId, Optional.of(task.getAgentId().getValue()))); try { final String path = getTaskPath(task.getTaskId()); CuratorTransactionFinal transaction = curator.inTransaction().create().forPath(path, taskTranscoder.toBytes(task)).and(); transaction.create().forPath(getActivePath(task.getTaskId().getId())).and().commit(); leaderCache.putActiveTask(task); taskCache.set(path, task); } catch (KeeperException.NodeExistsException nee) { LOG.error("Task or active path already existed for {}", task.getTaskId()); } }
protected void statusUpdate(SingularityTask task, TaskState state, Optional<Long> timestamp) { TaskStatus.Builder bldr = TaskStatus.newBuilder() .setTaskId(MesosProtosUtils.toTaskId(task.getMesosTask().getTaskId())) .setAgentId(MesosProtosUtils.toAgentId(task.getAgentId())) .setState(state); if (timestamp.isPresent()) { bldr.setTimestamp(timestamp.get() / 1000); } sms.statusUpdate(bldr.build()).join(); }
@Test public void testUsagePollerSimple() { // works with no slaves usagePoller.runActionOnPoll(); cleaner.runActionOnPoll(); initRequest(); initFirstDeploy(); saveAndSchedule(request.toBuilder().setInstances(Optional.of(1))); resourceOffers(1); SingularityTask firstTask = taskManager.getActiveTasks().get(0); String hostname = firstTask.getHostname(); MesosTaskMonitorObject usage = getTaskMonitor(firstTask.getTaskId().getId(), 2, 5, 100); mesosClient.setSlaveResourceUsage(hostname, Collections.singletonList(usage)); usagePoller.runActionOnPoll(); String slaveId = firstTask.getAgentId().getValue(); List<String> slaves = usageManager.getSlavesWithUsage(); Assert.assertEquals(1, slaves.size()); Assert.assertEquals(slaves.get(0), slaveId); Assert.assertEquals(0, usageManager.getSlaveUsage(slaveId).get(0).getCpusUsed(), 0); Assert.assertEquals(100, usageManager.getSlaveUsage(slaveId).get(0).getMemoryBytesUsed(), 0); SingularityTaskUsage first = usageManager.getTaskUsage(firstTask.getTaskId().getId()).get(0); Assert.assertEquals(2, first.getCpuSeconds(), 0); Assert.assertEquals(100, first.getMemoryTotalBytes(), 0); Assert.assertEquals(5, first.getTimestamp(), 0); }
@Test public void testDecommissionDoesntKillPendingDeploy() { initRequest(); deployResource.deploy(new SingularityDeployRequest(new SingularityDeployBuilder(requestId, "d1").setCommand(Optional.of("cmd")).build(), Optional.absent(), Optional.absent()), singularityUser); scheduler.drainPendingQueue(); deployChecker.checkDeploys(); resourceOffers(); Assert.assertEquals(1, taskManager.getNumActiveTasks()); slaveResource.decommissionSlave(singularityUser, taskManager.getActiveTasks().get(0).getAgentId().getValue(), null); scheduler.checkForDecomissions(); cleaner.drainCleanupQueue(); killKilledTasks(); Assert.assertEquals(1, taskManager.getNumActiveTasks()); Assert.assertEquals(1, taskManager.getNumCleanupTasks()); Assert.assertEquals(0, taskManager.getKilledTaskIdRecords().size()); configuration.setPendingDeployHoldTaskDuringDecommissionMillis(1); try { Thread.sleep(2); } catch (InterruptedException e) {} cleaner.drainCleanupQueue(); killKilledTasks(); Assert.assertEquals(0, taskManager.getNumActiveTasks()); Assert.assertEquals(0, taskManager.getNumCleanupTasks()); }