@Override public boolean apply(@Nonnull SingularityPendingTaskId input) { return input.getDeployId().equals(deployId); }
@Override public boolean apply(@Nonnull SingularityPendingTask input) { return input.getPendingTaskId().getDeployId().equals(deployId); }
private Map<SingularityDeployKey, SingularityPendingTaskId> getDeployKeyToPendingTaskId() { final List<SingularityPendingTaskId> pendingTaskIds = taskManager.getPendingTaskIds(); final Map<SingularityDeployKey, SingularityPendingTaskId> deployKeyToPendingTaskId = Maps.newHashMapWithExpectedSize(pendingTaskIds.size()); for (SingularityPendingTaskId taskId : pendingTaskIds) { SingularityDeployKey deployKey = new SingularityDeployKey(taskId.getRequestId(), taskId.getDeployId()); deployKeyToPendingTaskId.put(deployKey, taskId); } return deployKeyToPendingTaskId; }
@Override public int compareTo(SingularityPendingTaskId o) { return ComparisonChain.start() .compare(this.getNextRunAt(), o.getNextRunAt()) .compare(this.getRequestId(), o.getRequestId()) .compare(this.getDeployId(), o.getDeployId()) .compare(this.getInstanceNo(), o.getInstanceNo()) .compare(this.getCreatedAt(), o.getCreatedAt()) .compare(this.getPendingType(), o.getPendingType()) .result(); }
private void deleteObsoletePendingTasks(SingularityPendingDeploy pendingDeploy) { List<SingularityPendingTaskId> obsoletePendingTasks = taskManager.getPendingTaskIds() .stream() .filter(taskId -> taskId.getRequestId().equals(pendingDeploy.getDeployMarker().getRequestId())) .filter(taskId -> !taskId.getDeployId().equals(pendingDeploy.getDeployMarker().getDeployId())) .collect(Collectors.toList()); for (SingularityPendingTaskId pendingTaskId : obsoletePendingTasks) { LOG.debug("Deleting obsolete pending task {}", pendingTaskId.getId()); taskManager.deletePendingTask(pendingTaskId); } }
private void deleteScheduledTasks(final Collection<SingularityPendingTask> scheduledTasks, SingularityPendingRequest pendingRequest) { List<SingularityPendingTask> tasksForDeploy = scheduledTasks .stream() .filter(task -> pendingRequest.getRequestId().equals(task.getPendingTaskId().getRequestId())) .filter(task -> pendingRequest.getDeployId().equals(task.getPendingTaskId().getDeployId())) .collect(Collectors.toList()); for (SingularityPendingTask task : tasksForDeploy) { LOG.debug("Deleting pending task {} in order to reschedule {}", task.getPendingTaskId().getId(), pendingRequest); taskManager.deletePendingTask(task.getPendingTaskId()); } }
public List<SingularityTaskRequest> getTaskRequests(List<SingularityPendingTask> tasks) { final Multimap<String, SingularityPendingTask> requestIdToPendingTaskId = ArrayListMultimap.create(tasks.size(), 1); for (SingularityPendingTask task : tasks) { requestIdToPendingTaskId.put(task.getPendingTaskId().getRequestId(), task); } final List<SingularityRequestWithState> matchingRequests = requestManager.getRequests(requestIdToPendingTaskId.keySet()); final Map<SingularityPendingTask, SingularityDeployKey> deployKeys = SingularityDeployKey.fromPendingTasks(requestIdToPendingTaskId.values()); final Map<SingularityDeployKey, SingularityDeploy> matchingDeploys = deployManager.getDeploysForKeys(Sets.newHashSet(deployKeys.values())); final List<SingularityTaskRequest> taskRequests = Lists.newArrayListWithCapacity(matchingRequests.size()); for (SingularityRequestWithState request : matchingRequests) { Optional<SingularityPendingDeploy> maybePendingDeploy = deployManager.getPendingDeploy(request.getRequest().getId()); for (SingularityPendingTask task : requestIdToPendingTaskId.get(request.getRequest().getId())) { SingularityDeploy foundDeploy = matchingDeploys.get(deployKeys.get(task)); if (foundDeploy == null) { LOG.warn("Couldn't find a matching deploy for pending task {}", task); continue; } if (!request.getState().isRunnable()) { LOG.warn("Request was in state {} for pending task {}", request.getState(), task); continue; } Optional<SingularityRequest> updatedRequest = maybePendingDeploy.isPresent() && maybePendingDeploy.get().getDeployMarker().getDeployId().equals(task.getPendingTaskId().getDeployId()) ? maybePendingDeploy.get().getUpdatedRequest() : Optional.<SingularityRequest>absent(); taskRequests.add(new SingularityTaskRequest(updatedRequest.or(request.getRequest()), foundDeploy, task)); } } return taskRequests; }
public static SingularityDeployKey fromPendingTask(SingularityPendingTask pendingTask) { return new SingularityDeployKey(pendingTask.getPendingTaskId().getRequestId(), pendingTask.getPendingTaskId().getDeployId()); }
@Test public void testNewlyDeployedScheduledTasksAreScheduledAfterStartup() { initScheduledRequest(); initFirstDeploy(); SingularityTask runningTask = launchTask(request, firstDeploy, 1, TaskState.TASK_RUNNING); long now = System.currentTimeMillis(); initSecondDeploy(); requestManager.addToPendingQueue(new SingularityPendingRequest(requestId, secondDeployId, now, Optional.absent(), PendingType.STARTUP, Optional.absent(), Optional.absent())); deployChecker.checkDeploys(); resourceOffers(); // There's an instance running, so we shouldn't schedule a pending task yet Assert.assertTrue(taskManager.getPendingTaskIds().isEmpty()); statusUpdate(runningTask, TaskState.TASK_FINISHED); scheduler.drainPendingQueue(); // Now a pending task should be scheduled with the new deploy Assert.assertEquals(1, taskManager.getPendingTaskIds().size()); Assert.assertEquals(PendingType.NEW_DEPLOY, taskManager.getPendingTaskIds().get(0).getPendingType()); Assert.assertEquals(secondDeployId, taskManager.getPendingTaskIds().get(0).getDeployId()); }
@Test public void testSchedulerIsolatesPendingTasksBasedOnDeploy() { initRequest(); initFirstDeploy(); initSecondDeploy(); SingularityPendingTask p1 = pendingTask(requestId, firstDeployId, PendingType.ONEOFF); SingularityPendingTask p2 = pendingTask(requestId, firstDeployId, PendingType.TASK_DONE); SingularityPendingTask p3 = pendingTask(requestId, secondDeployId, PendingType.TASK_DONE); taskManager.savePendingTask(p1); taskManager.savePendingTask(p2); taskManager.savePendingTask(p3); requestManager.addToPendingQueue(new SingularityPendingRequest(requestId, secondDeployId, System.currentTimeMillis(), Optional.<String> absent(), PendingType.NEW_DEPLOY, Optional.<Boolean> absent(), Optional.<String> absent())); scheduler.drainPendingQueue(); // we expect there to be 3 pending tasks : List<SingularityPendingTask> returnedScheduledTasks = taskManager.getPendingTasks(); Assert.assertEquals(3, returnedScheduledTasks.size()); Assert.assertTrue(returnedScheduledTasks.contains(p1)); Assert.assertTrue(returnedScheduledTasks.contains(p2)); Assert.assertTrue(!returnedScheduledTasks.contains(p3)); boolean found = false; for (SingularityPendingTask pendingTask : returnedScheduledTasks) { if (pendingTask.getPendingTaskId().getDeployId().equals(secondDeployId)) { found = true; Assert.assertEquals(PendingType.NEW_DEPLOY, pendingTask.getPendingTaskId().getPendingType()); } } Assert.assertTrue(found); }
@Test public void testUsesNewRequestDataFromPendingDeploy() { initRequest(); initFirstDeploy(); saveAndSchedule(request.toBuilder().setInstances(Optional.of(2))); scheduler.drainPendingQueue(); Assert.assertEquals(2, taskManager.getPendingTaskIds().size()); Assert.assertEquals(2, requestManager.getRequest(requestId).get().getRequest().getInstancesSafe()); SingularityRequest request = requestResource.getRequest(requestId, singularityUser).getRequest(); SingularityRequest newRequest = request.toBuilder().setInstances(Optional.of(1)).build(); String deployId = "test_new_request_data"; SingularityDeploy deploy = new SingularityDeployBuilder(request.getId(), deployId).setCommand(Optional.of("sleep 100")).build(); deployResource.deploy(new SingularityDeployRequest(deploy, Optional.absent(), Optional.absent(), Optional.of(newRequest)), singularityUser); deployChecker.checkDeploys(); scheduler.drainPendingQueue(); List<SingularityPendingTaskId> pendingTaskIdsForNewDeploy = new ArrayList<>(); for (SingularityPendingTaskId pendingTaskId : taskManager.getPendingTaskIds()) { if (pendingTaskId.getDeployId().equals(deployId)) { pendingTaskIdsForNewDeploy.add(pendingTaskId); } } Assert.assertEquals(1, pendingTaskIdsForNewDeploy.size()); Assert.assertEquals(2, requestManager.getRequest(requestId).get().getRequest().getInstancesSafe()); resourceOffers(); for (SingularityTaskId taskId : taskManager.getActiveTaskIdsForDeploy(requestId, deployId)) { statusUpdate(taskManager.getTask(taskId).get(), TaskState.TASK_RUNNING); } deployChecker.checkDeploys(); Assert.assertEquals(1, requestManager.getRequest(requestId).get().getRequest().getInstancesSafe()); }
@Test public void testAfterDeployWaitsForScheduledTaskToFinish() { initScheduledRequest(); initFirstDeploy(); SingularityTask firstTask = launchTask(request, firstDeploy, 1, TaskState.TASK_RUNNING); Assert.assertTrue(taskManager.getPendingTasks().isEmpty()); Assert.assertTrue(taskManager.getActiveTaskIds().contains(firstTask.getTaskId())); Assert.assertEquals(1, taskManager.getActiveTaskIds().size()); Assert.assertTrue(taskManager.getCleanupTaskIds().isEmpty()); deploy("nextDeployId"); deployChecker.checkDeploys(); scheduler.drainPendingQueue(); // no second task should be scheduled Assert.assertTrue(taskManager.getPendingTasks().isEmpty()); Assert.assertTrue(taskManager.getActiveTaskIds().contains(firstTask.getTaskId())); Assert.assertEquals(1, taskManager.getActiveTaskIds().size()); Assert.assertTrue(!taskManager.getCleanupTaskIds().isEmpty()); statusUpdate(firstTask, TaskState.TASK_FINISHED); scheduler.drainPendingQueue(); cleaner.drainCleanupQueue(); Assert.assertTrue(!taskManager.getPendingTasks().isEmpty()); Assert.assertTrue(taskManager.getActiveTaskIds().isEmpty()); Assert.assertTrue(taskManager.getCleanupTaskIds().isEmpty()); SingularityPendingTaskId pendingTaskId = taskManager.getPendingTaskIds().get(0); Assert.assertEquals("nextDeployId", pendingTaskId.getDeployId()); Assert.assertEquals(requestId, pendingTaskId.getRequestId()); }
@Test public void testDeployManagerHandlesFailedLBTask() { initLoadBalancedRequest(); initFirstDeploy(); SingularityTask firstTask = startTask(firstDeploy); initSecondDeploy(); SingularityTask secondTask = startTask(secondDeploy); // this should cause an LB call to happen: deployChecker.checkDeploys(); Assert.assertTrue(taskManager.getLoadBalancerState(secondTask.getTaskId(), LoadBalancerRequestType.ADD).isPresent()); Assert.assertTrue(!taskManager.getLoadBalancerState(secondTask.getTaskId(), LoadBalancerRequestType.DEPLOY).isPresent()); Assert.assertTrue(!taskManager.getLoadBalancerState(secondTask.getTaskId(), LoadBalancerRequestType.REMOVE).isPresent()); statusUpdate(secondTask, TaskState.TASK_FAILED); statusUpdate(firstTask, TaskState.TASK_FAILED); scheduler.drainPendingQueue(); deployChecker.checkDeploys(); Assert.assertTrue(deployManager.getDeployResult(requestId, secondDeployId).get().getDeployState() == DeployState.FAILED); List<SingularityPendingTask> pendingTasks = taskManager.getPendingTasks(); Assert.assertTrue(pendingTasks.size() == 1); Assert.assertTrue(pendingTasks.get(0).getPendingTaskId().getDeployId().equals(firstDeployId)); }
List<SingularityPendingTaskId> pendingTaskIds = taskManager.getPendingTaskIds(); Assert.assertEquals(1, pendingTaskIds.size()); Assert.assertEquals(firstDeployId, pendingTaskIds.get(0).getDeployId());
private Map<SingularityDeployKey, SingularityPendingTaskId> getDeployKeyToPendingTaskId() { final List<SingularityPendingTaskId> pendingTaskIds = taskManager.getPendingTaskIds(); final Map<SingularityDeployKey, SingularityPendingTaskId> deployKeyToPendingTaskId = Maps.newHashMapWithExpectedSize(pendingTaskIds.size()); for (SingularityPendingTaskId taskId : pendingTaskIds) { SingularityDeployKey deployKey = new SingularityDeployKey(taskId.getRequestId(), taskId.getDeployId()); deployKeyToPendingTaskId.put(deployKey, taskId); } return deployKeyToPendingTaskId; }
private void deleteObsoletePendingTasks(SingularityPendingDeploy pendingDeploy) { List<SingularityPendingTaskId> obsoletePendingTasks = taskManager.getPendingTaskIds() .stream() .filter(taskId -> taskId.getRequestId().equals(pendingDeploy.getDeployMarker().getRequestId())) .filter(taskId -> !taskId.getDeployId().equals(pendingDeploy.getDeployMarker().getDeployId())) .collect(Collectors.toList()); for (SingularityPendingTaskId pendingTaskId : obsoletePendingTasks) { LOG.debug("Deleting obsolete pending task {}", pendingTaskId.getId()); taskManager.deletePendingTask(pendingTaskId); } }
private void deleteScheduledTasks(final Collection<SingularityPendingTask> scheduledTasks, SingularityPendingRequest pendingRequest) { List<SingularityPendingTask> tasksForDeploy = scheduledTasks .stream() .filter(task -> pendingRequest.getRequestId().equals(task.getPendingTaskId().getRequestId())) .filter(task -> pendingRequest.getDeployId().equals(task.getPendingTaskId().getDeployId())) .collect(Collectors.toList()); for (SingularityPendingTask task : tasksForDeploy) { LOG.debug("Deleting pending task {} in order to reschedule {}", task.getPendingTaskId().getId(), pendingRequest); taskManager.deletePendingTask(task.getPendingTaskId()); } }
public List<SingularityTaskRequest> getTaskRequests(List<SingularityPendingTask> tasks) { final Multimap<String, SingularityPendingTask> requestIdToPendingTaskId = ArrayListMultimap.create(tasks.size(), 1); for (SingularityPendingTask task : tasks) { requestIdToPendingTaskId.put(task.getPendingTaskId().getRequestId(), task); } final List<SingularityRequestWithState> matchingRequests = requestManager.getRequests(requestIdToPendingTaskId.keySet()); final Map<SingularityPendingTask, SingularityDeployKey> deployKeys = SingularityDeployKey.fromPendingTasks(requestIdToPendingTaskId.values()); final Map<SingularityDeployKey, SingularityDeploy> matchingDeploys = deployManager.getDeploysForKeys(Sets.newHashSet(deployKeys.values())); final List<SingularityTaskRequest> taskRequests = Lists.newArrayListWithCapacity(matchingRequests.size()); for (SingularityRequestWithState request : matchingRequests) { Optional<SingularityPendingDeploy> maybePendingDeploy = deployManager.getPendingDeploy(request.getRequest().getId()); for (SingularityPendingTask task : requestIdToPendingTaskId.get(request.getRequest().getId())) { SingularityDeploy foundDeploy = matchingDeploys.get(deployKeys.get(task)); if (foundDeploy == null) { LOG.warn("Couldn't find a matching deploy for pending task {}", task); continue; } if (!request.getState().isRunnable()) { LOG.warn("Request was in state {} for pending task {}", request.getState(), task); continue; } Optional<SingularityRequest> updatedRequest = maybePendingDeploy.isPresent() && maybePendingDeploy.get().getDeployMarker().getDeployId().equals(task.getPendingTaskId().getDeployId()) ? maybePendingDeploy.get().getUpdatedRequest() : Optional.<SingularityRequest>absent(); taskRequests.add(new SingularityTaskRequest(updatedRequest.or(request.getRequest()), foundDeploy, task)); } } return taskRequests; }
@Test public void testNewlyDeployedScheduledTasksAreScheduledAfterStartup() { initScheduledRequest(); initFirstDeploy(); SingularityTask runningTask = launchTask(request, firstDeploy, 1, TaskState.TASK_RUNNING); long now = System.currentTimeMillis(); initSecondDeploy(); requestManager.addToPendingQueue(new SingularityPendingRequest(requestId, secondDeployId, now, Optional.absent(), PendingType.STARTUP, Optional.absent(), Optional.absent())); deployChecker.checkDeploys(); resourceOffers(); // There's an instance running, so we shouldn't schedule a pending task yet Assert.assertTrue(taskManager.getPendingTaskIds().isEmpty()); statusUpdate(runningTask, TaskState.TASK_FINISHED); scheduler.drainPendingQueue(); // Now a pending task should be scheduled with the new deploy Assert.assertEquals(1, taskManager.getPendingTaskIds().size()); Assert.assertEquals(PendingType.NEW_DEPLOY, taskManager.getPendingTaskIds().get(0).getPendingType()); Assert.assertEquals(secondDeployId, taskManager.getPendingTaskIds().get(0).getDeployId()); }
@Test public void testDeployManagerHandlesFailedLBTask() { initLoadBalancedRequest(); initFirstDeploy(); SingularityTask firstTask = startTask(firstDeploy); initSecondDeploy(); SingularityTask secondTask = startTask(secondDeploy); // this should cause an LB call to happen: deployChecker.checkDeploys(); Assert.assertTrue(taskManager.getLoadBalancerState(secondTask.getTaskId(), LoadBalancerRequestType.ADD).isPresent()); Assert.assertTrue(!taskManager.getLoadBalancerState(secondTask.getTaskId(), LoadBalancerRequestType.DEPLOY).isPresent()); Assert.assertTrue(!taskManager.getLoadBalancerState(secondTask.getTaskId(), LoadBalancerRequestType.REMOVE).isPresent()); statusUpdate(secondTask, TaskState.TASK_FAILED); statusUpdate(firstTask, TaskState.TASK_FAILED); scheduler.drainPendingQueue(); deployChecker.checkDeploys(); Assert.assertTrue(deployManager.getDeployResult(requestId, secondDeployId).get().getDeployState() == DeployState.FAILED); List<SingularityPendingTask> pendingTasks = taskManager.getPendingTasks(); Assert.assertTrue(pendingTasks.size() == 1); Assert.assertTrue(pendingTasks.get(0).getPendingTaskId().getDeployId().equals(firstDeployId)); }