private void drainRequestCleanupQueue() { final long start = System.currentTimeMillis(); final List<SingularityRequestCleanup> cleanupRequests = requestManager.getCleanupRequests(); if (cleanupRequests.isEmpty()) { LOG.trace("Request cleanup queue is empty"); return; } LOG.info("Cleaning up {} requests", cleanupRequests.size()); AtomicInteger numTasksKilled = new AtomicInteger(0); AtomicInteger numScheduledTasksRemoved = new AtomicInteger(0); cleanupRequests.parallelStream().forEach((requestCleanup) -> { lock.runWithRequestLock(() -> { processRequestCleanup(start, numTasksKilled, numScheduledTasksRemoved, requestCleanup); }, requestCleanup.getRequestId(), String.format("%s#%s", getClass().getSimpleName(), "drainRequestCleanupQueue")); }); LOG.info("Killed {} tasks (removed {} scheduled) in {}", numTasksKilled.get(), numScheduledTasksRemoved.get(), JavaUtils.duration(start)); }
@GET @PropertyFiltering @Path("/queued/cleanup") @Operation(summary = "Retrieve the list of requests being cleaned up") public List<SingularityRequestCleanup> getCleanupRequests(@Parameter(hidden = true) @Auth SingularityUser user) { return authorizationHelper.filterByAuthorizedRequests(user, requestManager.getCleanupRequests(), SingularityTransformHelpers.REQUEST_CLEANUP_TO_REQUEST_ID, SingularityAuthorizationScope.READ); }
@Test public void badPauseExpires() { initRequest(); requestManager.createCleanupRequest(new SingularityRequestCleanup(Optional.<String>absent(), RequestCleanupType.PAUSING, System.currentTimeMillis(), Optional.<Boolean>absent(), Optional.absent(), requestId, Optional.<String>absent(), Optional.<Boolean> absent(), Optional.<String>absent(), Optional.<String>absent(), Optional.<SingularityShellCommand>absent())); cleaner.drainCleanupQueue(); Assert.assertTrue(!requestManager.getCleanupRequests().isEmpty()); configuration.setCleanupEverySeconds(0); sleep(1); cleaner.drainCleanupQueue(); Assert.assertTrue(requestManager.getCleanupRequests().isEmpty()); }
@Test public void testScaleWithBounceDoesNotLaunchExtraInstances() { initRequest(); initFirstDeploy(); launchTask(request, firstDeploy, 1, TaskState.TASK_RUNNING); requestResource.scale(requestId, new SingularityScaleRequest(Optional.of(5), Optional.of(1L), Optional.absent(), Optional.absent(), Optional.absent(), Optional.of(true), Optional.absent(), Optional.absent()), singularityUser); Assert.assertEquals(1, requestManager.getCleanupRequests().size()); cleaner.drainCleanupQueue(); Assert.assertEquals(1, taskManager.getNumCleanupTasks()); scheduler.drainPendingQueue(); Assert.assertEquals(5, taskManager.getPendingTaskIds().size()); }
@Test public void testCleanerLeavesPausedRequestTasksByDemand() { initScheduledRequest(); initFirstDeploy(); SingularityTask firstTask = launchTask(request, firstDeploy, 1, TaskState.TASK_RUNNING); createAndSchedulePendingTask(firstDeployId); requestResource.pause(requestId, Optional.of(new SingularityPauseRequest(Optional.of(false), Optional. absent(), Optional. absent(), Optional.absent(), Optional.absent())), singularityUser); cleaner.drainCleanupQueue(); Assert.assertTrue(taskManager.getKilledTaskIdRecords().isEmpty()); Assert.assertTrue(taskManager.getPendingTaskIds().isEmpty()); Assert.assertTrue(requestManager.getCleanupRequests().isEmpty()); statusUpdate(firstTask, TaskState.TASK_FINISHED); // make sure something new isn't scheduled! Assert.assertTrue(taskManager.getPendingTaskIds().isEmpty()); }
@Test public void testTaskDestroy() { initRequest(); initFirstDeploy(); SingularityTask firstTask = startTask(firstDeploy, 1); SingularityTask secondTask = startTask(firstDeploy, 2); SingularityTask thirdTask = startTask(firstDeploy, 3); taskResource.killTask(secondTask.getTaskId().getId(), Optional.of( new SingularityKillTaskRequest(Optional.of(true), Optional.of("kill -9 bb"), Optional.absent(), Optional.absent(), Optional.absent())), singularityUser); cleaner.drainCleanupQueue(); killKilledTasks(); Assert.assertEquals(2, taskManager.getNumActiveTasks()); System.out.println(requestManager.getCleanupRequests()); Assert.assertEquals(0, requestManager.getCleanupRequests().size()); Assert.assertEquals(RequestState.ACTIVE, requestManager.getRequest(requestId).get().getState()); }
@Test public void testLbCleanupSkippedOnSkipRemoveFlag() { configuration.setDeleteRemovedRequestsFromLoadBalancer(true); initLoadBalancedRequest(); initLoadBalancedDeploy(); startTask(firstDeploy); boolean removeFromLoadBalancer = false; SingularityDeleteRequestRequest deleteRequest = new SingularityDeleteRequestRequest(Optional.absent(), Optional.absent(), Optional.of(removeFromLoadBalancer)); requestResource.deleteRequest(requestId, Optional.of(deleteRequest), singularityUser); testingLbClient.setNextBaragonRequestState(BaragonRequestState.WAITING); Assert.assertFalse("Tasks should get cleaned up", requestManager.getCleanupRequests().isEmpty()); cleaner.drainCleanupQueue(); killKilledTasks(); Assert.assertFalse("The request should get cleaned up", requestManager.getCleanupRequests().isEmpty()); cleaner.drainCleanupQueue(); Assert.assertTrue("The request should not be removed from the load balancer", requestManager.getLbCleanupRequestIds().isEmpty()); }
@Test public void testKilledTaskIdRecords() { initScheduledRequest(); initFirstDeploy(); launchTask(request, firstDeploy, 1, TaskState.TASK_RUNNING); requestResource.deleteRequest(requestId, Optional.absent(), singularityUser); Assert.assertTrue(requestManager.getCleanupRequests().size() == 1); cleaner.drainCleanupQueue(); Assert.assertTrue(!taskManager.getKilledTaskIdRecords().isEmpty()); killKilledTasks(); cleaner.drainCleanupQueue(); Assert.assertTrue(requestManager.getCleanupRequests().isEmpty()); Assert.assertTrue(taskManager.getKilledTaskIdRecords().isEmpty()); }
@Test public void testLbCleanupOccursOnRequestDelete() { configuration.setDeleteRemovedRequestsFromLoadBalancer(true); initLoadBalancedRequest(); initLoadBalancedDeploy(); startTask(firstDeploy); requestResource.deleteRequest(requestId, Optional.absent(), singularityUser); testingLbClient.setNextBaragonRequestState(BaragonRequestState.WAITING); Assert.assertFalse("Tasks should get cleaned up", requestManager.getCleanupRequests().isEmpty()); cleaner.drainCleanupQueue(); killKilledTasks(); Assert.assertFalse("The request should get cleaned up", requestManager.getCleanupRequests().isEmpty()); cleaner.drainCleanupQueue(); Assert.assertFalse("The request should get removed from the load balancer", requestManager.getLbCleanupRequestIds().isEmpty()); }
Optional.of(new SingularityBounceRequest(Optional.absent(), Optional.absent(), Optional.of(1L), Optional.of("aid"), Optional.absent(), Optional.absent())), singularityUser); Assert.assertTrue(!requestManager.getCleanupRequests().get(0).getMessage().isPresent()); Assert.assertEquals("aid", requestManager.getCleanupRequests().get(0).getActionId().get()); Assert.assertEquals(0, requestManager.getCleanupRequests().size()); Assert.assertEquals(3, taskManager.getCleanupTaskIds().size()); Assert.assertEquals(3, taskManager.getCleanupTaskIds().size()); Assert.assertEquals(0, requestManager.getPendingRequests().size()); Assert.assertEquals(0, requestManager.getCleanupRequests().size()); Assert.assertEquals(0, taskManager.getCleanupTaskIds().size()); Assert.assertEquals(0, requestManager.getPendingRequests().size()); Assert.assertEquals(0, requestManager.getCleanupRequests().size());
Assert.assertEquals("msg", requestManager.getCleanupRequests().get(0).getMessage().get()); Assert.assertTrue(requestManager.getCleanupRequests().get(0).getActionId().isPresent()); String actionId = requestManager.getCleanupRequests().get(0).getActionId().get();
Assert.assertEquals(1, requestManager.getCleanupRequests().size()); cleaner.drainCleanupQueue(); resourceOffers(); Assert.assertEquals(1, requestManager.getCleanupRequests().size()); cleaner.drainCleanupQueue(); Assert.assertEquals(4, taskManager.getKilledTaskIdRecords().size());
@Test public void itSetsRequestStateToDeletedAfterAllTasksAreCleanedUp() { initRequest(); SingularityRequest request = requestResource.getRequest(requestId, singularityUser).getRequest(); requestResource.postRequest(request.toBuilder().setInstances(Optional.of(2)).build(), singularityUser); initFirstDeploy(); launchTask(request, firstDeploy, 1, TaskState.TASK_RUNNING); launchTask(request, firstDeploy, 2, TaskState.TASK_RUNNING); Assert.assertEquals(requestId, requestManager.getActiveRequests().iterator().next().getRequest().getId()); Assert.assertEquals(2, taskManager.getActiveTaskIds().size()); requestManager.startDeletingRequest(request, Optional.absent(), Optional.absent(), Optional.absent(), Optional.absent()); Assert.assertEquals(requestId, requestManager.getCleanupRequests().get(0).getRequestId()); Assert.assertEquals(RequestState.DELETING, requestManager.getRequest(requestId).get().getState()); cleaner.drainCleanupQueue(); Assert.assertEquals(0, taskManager.getCleanupTaskIds().size()); killKilledTasks(); cleaner.drainCleanupQueue(); Assert.assertFalse(requestManager.getRequest(requestId).isPresent()); }
@Test public void itSetsRequestStateToDeletedIfTaskCleanupFails() { initRequest(); SingularityRequest request = requestResource.getRequest(requestId, singularityUser).getRequest(); requestResource.postRequest(request.toBuilder().setInstances(Optional.of(2)).build(), singularityUser); initFirstDeploy(); SingularityTask firstTask = launchTask(request, firstDeploy, 1, TaskState.TASK_RUNNING); launchTask(request, firstDeploy, 2, TaskState.TASK_RUNNING); Assert.assertEquals(requestId, requestManager.getActiveRequests().iterator().next().getRequest().getId()); Assert.assertEquals(2, taskManager.getActiveTaskIds().size()); requestManager.startDeletingRequest(request, Optional.absent(), Optional.absent(), Optional.absent(), Optional.absent()); Assert.assertEquals(requestId, requestManager.getCleanupRequests().get(0).getRequestId()); Assert.assertEquals(RequestState.DELETING, requestManager.getRequest(requestId).get().getState()); statusUpdate(firstTask, TaskState.TASK_FAILED); Assert.assertEquals(1, taskManager.getActiveTaskIds().size()); cleaner.drainCleanupQueue(); Assert.assertEquals(0, taskManager.getCleanupTaskIds().size()); killKilledTasks(); cleaner.drainCleanupQueue(); Assert.assertFalse(requestManager.getRequest(requestId).isPresent()); }
@GET @PropertyFiltering @Path("/queued/cleanup") @Operation(summary = "Retrieve the list of requests being cleaned up") public List<SingularityRequestCleanup> getCleanupRequests(@Parameter(hidden = true) @Auth SingularityUser user) { return authorizationHelper.filterByAuthorizedRequests(user, requestManager.getCleanupRequests(), SingularityTransformHelpers.REQUEST_CLEANUP_TO_REQUEST_ID, SingularityAuthorizationScope.READ); }
@Test public void badPauseExpires() { initRequest(); requestManager.createCleanupRequest(new SingularityRequestCleanup(Optional.<String>absent(), RequestCleanupType.PAUSING, System.currentTimeMillis(), Optional.<Boolean>absent(), Optional.absent(), requestId, Optional.<String>absent(), Optional.<Boolean> absent(), Optional.<String>absent(), Optional.<String>absent(), Optional.<SingularityShellCommand>absent())); cleaner.drainCleanupQueue(); Assert.assertTrue(!requestManager.getCleanupRequests().isEmpty()); configuration.setCleanupEverySeconds(0); sleep(1); cleaner.drainCleanupQueue(); Assert.assertTrue(requestManager.getCleanupRequests().isEmpty()); }
@Test public void testScaleWithBounceDoesNotLaunchExtraInstances() { initRequest(); initFirstDeploy(); launchTask(request, firstDeploy, 1, TaskState.TASK_RUNNING); requestResource.scale(requestId, new SingularityScaleRequest(Optional.of(5), Optional.of(1L), Optional.absent(), Optional.absent(), Optional.absent(), Optional.of(true), Optional.absent(), Optional.absent()), singularityUser); Assert.assertEquals(1, requestManager.getCleanupRequests().size()); cleaner.drainCleanupQueue(); Assert.assertEquals(1, taskManager.getNumCleanupTasks()); scheduler.drainPendingQueue(); Assert.assertEquals(5, taskManager.getPendingTaskIds().size()); }
@Test public void testCleanerLeavesPausedRequestTasksByDemand() { initScheduledRequest(); initFirstDeploy(); SingularityTask firstTask = launchTask(request, firstDeploy, 1, TaskState.TASK_RUNNING); createAndSchedulePendingTask(firstDeployId); requestResource.pause(requestId, Optional.of(new SingularityPauseRequest(Optional.of(false), Optional. absent(), Optional. absent(), Optional.absent(), Optional.absent())), singularityUser); cleaner.drainCleanupQueue(); Assert.assertTrue(taskManager.getKilledTaskIdRecords().isEmpty()); Assert.assertTrue(taskManager.getPendingTaskIds().isEmpty()); Assert.assertTrue(requestManager.getCleanupRequests().isEmpty()); statusUpdate(firstTask, TaskState.TASK_FINISHED); // make sure something new isn't scheduled! Assert.assertTrue(taskManager.getPendingTaskIds().isEmpty()); }
@Test public void testKilledTaskIdRecords() { initScheduledRequest(); initFirstDeploy(); launchTask(request, firstDeploy, 1, TaskState.TASK_RUNNING); requestResource.deleteRequest(requestId, Optional.absent(), singularityUser); Assert.assertTrue(requestManager.getCleanupRequests().size() == 1); cleaner.drainCleanupQueue(); Assert.assertTrue(!taskManager.getKilledTaskIdRecords().isEmpty()); killKilledTasks(); cleaner.drainCleanupQueue(); Assert.assertTrue(requestManager.getCleanupRequests().isEmpty()); Assert.assertTrue(taskManager.getKilledTaskIdRecords().isEmpty()); }
@Test public void testLbCleanupOccursOnRequestDelete() { configuration.setDeleteRemovedRequestsFromLoadBalancer(true); initLoadBalancedRequest(); initLoadBalancedDeploy(); startTask(firstDeploy); requestResource.deleteRequest(requestId, Optional.absent(), singularityUser); testingLbClient.setNextBaragonRequestState(BaragonRequestState.WAITING); Assert.assertFalse("Tasks should get cleaned up", requestManager.getCleanupRequests().isEmpty()); cleaner.drainCleanupQueue(); killKilledTasks(); Assert.assertFalse("The request should get cleaned up", requestManager.getCleanupRequests().isEmpty()); cleaner.drainCleanupQueue(); Assert.assertFalse("The request should get removed from the load balancer", requestManager.getLbCleanupRequestIds().isEmpty()); }