private void bounce(SingularityRequestCleanup requestCleanup, final List<SingularityTaskId> activeTaskIds) { final long start = System.currentTimeMillis(); final List<SingularityTaskId> matchingTaskIds = new ArrayList<>(); for (SingularityTaskId activeTaskId : activeTaskIds) { if (activeTaskId.getRequestId().equals(requestCleanup.getRequestId()) && activeTaskId.getDeployId().equals(requestCleanup.getDeployId().get())) { matchingTaskIds.add(activeTaskId); } } for (SingularityTaskId matchingTaskId : matchingTaskIds) { LOG.debug("Adding task {} to cleanup (bounce)", matchingTaskId.getId()); Optional<SingularityTaskShellCommandRequestId> runBeforeKillId = Optional.absent(); if (requestCleanup.getRunShellCommandBeforeKill().isPresent()) { SingularityTaskShellCommandRequest shellRequest = new SingularityTaskShellCommandRequest(matchingTaskId, requestCleanup.getUser(), System.currentTimeMillis(), requestCleanup.getRunShellCommandBeforeKill().get()); taskManager.saveTaskShellCommandRequestToQueue(shellRequest); runBeforeKillId = Optional.of(shellRequest.getId()); } taskManager.createTaskCleanup(new SingularityTaskCleanup(requestCleanup.getUser(), requestCleanup.getCleanupType().getTaskCleanupType().get(), start, matchingTaskId, requestCleanup.getMessage(), requestCleanup.getActionId(), runBeforeKillId)); } if (matchingTaskIds.isEmpty() && requestCleanup.getDeployId().isPresent()) { Optional<SingularityExpiringBounce> expiringBounce = requestManager.getExpiringBounce(requestCleanup.getRequestId()); if (expiringBounce.isPresent() && expiringBounce.get().getDeployId().equalsIgnoreCase(requestCleanup.getDeployId().get())) { LOG.info("No running tasks for request {}. Marking bounce {} complete and starting new tasks", expiringBounce.get().getRequestId(), expiringBounce.get()); requestManager.removeExpiringBounce(requestCleanup.getRequestId()); } requestManager.markBounceComplete(requestCleanup.getRequestId()); } requestManager.addToPendingQueue(new SingularityPendingRequest(requestCleanup.getRequestId(), requestCleanup.getDeployId().get(), requestCleanup.getTimestamp(), requestCleanup.getUser(), PendingType.BOUNCE, Optional.absent(), Optional.absent(), requestCleanup.getSkipHealthchecks(), requestCleanup.getMessage(), requestCleanup.getActionId())); LOG.info("Added {} tasks for request {} to cleanup bounce queue in {}", matchingTaskIds.size(), requestCleanup.getRequestId(), JavaUtils.duration(start)); }
if (includeFullRequestData) { CompletableFuture<Optional<SingularityTaskIdsByStatus>> maybeTaskIdsByStatus = CompletableFuture.supplyAsync(() -> getTaskIdsByStatusForRequest(requestWithState)).exceptionally((throwable) -> Optional.absent()); CompletableFuture<Optional<SingularityExpiringBounce>> maybeExpiringBounce = CompletableFuture.supplyAsync(() -> requestManager.getExpiringBounce(requestWithState.getRequest().getId())).exceptionally((throwable) -> Optional.absent()); CompletableFuture<Optional<SingularityExpiringPause>> maybeExpiringPause = CompletableFuture.supplyAsync(() -> requestManager.getExpiringPause(requestWithState.getRequest().getId())).exceptionally((throwable) -> Optional.absent()); CompletableFuture<Optional<SingularityExpiringScale>> maybeExpiringScale = CompletableFuture.supplyAsync(() -> requestManager.getExpiringScale(requestWithState.getRequest().getId())).exceptionally((throwable) -> Optional.absent());
protected SingularityRequestParent fillEntireRequest(SingularityRequestWithState requestWithState, Optional<SingularityRequest> newRequestData) { final String requestId = requestWithState.getRequest().getId(); final Optional<SingularityRequestDeployState> requestDeployState = deployManager.getRequestDeployState(requestId); Optional<SingularityDeploy> activeDeploy = Optional.absent(); Optional<SingularityDeploy> pendingDeploy = Optional.absent(); if (requestDeployState.isPresent()) { activeDeploy = fillDeploy(requestDeployState.get().getActiveDeploy()); pendingDeploy = fillDeploy(requestDeployState.get().getPendingDeploy()); } Optional<SingularityPendingDeploy> pendingDeployState = deployManager.getPendingDeploy(requestId); return new SingularityRequestParent(newRequestData.or(requestWithState.getRequest()), requestWithState.getState(), requestDeployState, activeDeploy, pendingDeploy, pendingDeployState, requestManager.getExpiringBounce(requestId), requestManager.getExpiringPause(requestId), requestManager.getExpiringScale(requestId), requestManager.getExpiringSkipHealthchecks(requestId), requestHelper.getTaskIdsByStatusForRequest(requestId), requestHistoryHelper.getLastHistory(requestId), requestHelper.getMostRecentTask(requestWithState.getRequest())); }
Optional<SingularityExpiringBounce> expiringBounce = requestManager.getExpiringBounce(taskId.getRequestId());
@Test public void testExpiringBounceGoesAway() { initRequest(); initFirstDeploy(); startTask(firstDeploy, 1); requestResource.bounce(requestId, Optional.of(new SingularityBounceRequest(Optional.of(false), Optional.absent(), Optional.of(1L), Optional.absent(), Optional.of("msg"), Optional.absent())), singularityUser); cleaner.drainCleanupQueue(); resourceOffers(); runLaunchedTasks(); cleaner.drainCleanupQueue(); killKilledTasks(); Assert.assertTrue(taskManager.getCleanupTaskIds().isEmpty()); Assert.assertEquals(1, taskManager.getActiveTaskIds().size()); Assert.assertTrue(!requestManager.getExpiringBounce(requestId).isPresent()); }
@Test public void testBounceOnPendingInstancesReleasesLock() { initRequest(); initFirstDeploy(); SingularityTask task = startTask(firstDeploy, 1); statusUpdate(task, TaskState.TASK_FAILED); killKilledTasks(); Assert.assertEquals("Bounce starts when tasks have not yet been launched", 0, taskManager.getActiveTaskIds().size()); requestResource.bounce(requestId, Optional.of(new SingularityBounceRequest(Optional.absent(), Optional.of(true), Optional.absent(), Optional.absent(), Optional.absent(), Optional.absent())), singularityUser); // It acquires a lock on the bounce Assert.assertTrue("Lock on bounce should be acquired during bounce", requestManager.getExpiringBounce(requestId).isPresent()); cleaner.drainCleanupQueue(); scheduler.drainPendingQueue(); resourceOffers(); for (SingularityTaskId singularityTaskId : taskManager.getActiveTaskIds()) { taskManager.saveTaskHistoryUpdate(new SingularityTaskHistoryUpdate(singularityTaskId, System.currentTimeMillis(), ExtendedTaskState.TASK_RUNNING, Optional.absent(), Optional.absent(), Collections.emptySet())); } cleaner.drainCleanupQueue(); killKilledTasks(); // It finishes with one task running and the bounce released Assert.assertEquals("Should end bounce with target number of tasks", 1, taskManager.getActiveTaskIds().size()); for (SingularityTaskId singularityTaskId : taskManager.getActiveTaskIds()) { String statusMessage = taskManager.getTaskHistoryUpdates(singularityTaskId) .get(0) .getStatusMessage() .get(); Assert.assertTrue("Task was started by bounce", statusMessage.contains("BOUNCE")); } Assert.assertFalse("Lock on bounce should be released after bounce", requestManager.getExpiringBounce(requestId).isPresent()); }
Assert.assertTrue("Lock on bounce should be acquired during bounce", requestManager.getExpiringBounce(requestId).isPresent()); Assert.assertTrue("Task was started by bounce", statusMessage.contains("BOUNCE")); Assert.assertFalse("Lock on bounce should be released after bounce", requestManager.getExpiringBounce(requestId).isPresent());
Assert.assertTrue(!requestManager.getExpiringBounce(requestId).isPresent()); Assert.assertTrue(requestManager.getPendingRequests().isEmpty()); Assert.assertTrue(taskManager.getPendingTaskIds().isEmpty());
private void bounce(SingularityRequestCleanup requestCleanup, final List<SingularityTaskId> activeTaskIds) { final long start = System.currentTimeMillis(); final List<SingularityTaskId> matchingTaskIds = new ArrayList<>(); for (SingularityTaskId activeTaskId : activeTaskIds) { if (activeTaskId.getRequestId().equals(requestCleanup.getRequestId()) && activeTaskId.getDeployId().equals(requestCleanup.getDeployId().get())) { matchingTaskIds.add(activeTaskId); } } for (SingularityTaskId matchingTaskId : matchingTaskIds) { LOG.debug("Adding task {} to cleanup (bounce)", matchingTaskId.getId()); Optional<SingularityTaskShellCommandRequestId> runBeforeKillId = Optional.absent(); if (requestCleanup.getRunShellCommandBeforeKill().isPresent()) { SingularityTaskShellCommandRequest shellRequest = new SingularityTaskShellCommandRequest(matchingTaskId, requestCleanup.getUser(), System.currentTimeMillis(), requestCleanup.getRunShellCommandBeforeKill().get()); taskManager.saveTaskShellCommandRequestToQueue(shellRequest); runBeforeKillId = Optional.of(shellRequest.getId()); } taskManager.createTaskCleanup(new SingularityTaskCleanup(requestCleanup.getUser(), requestCleanup.getCleanupType().getTaskCleanupType().get(), start, matchingTaskId, requestCleanup.getMessage(), requestCleanup.getActionId(), runBeforeKillId)); } if (matchingTaskIds.isEmpty() && requestCleanup.getDeployId().isPresent()) { Optional<SingularityExpiringBounce> expiringBounce = requestManager.getExpiringBounce(requestCleanup.getRequestId()); if (expiringBounce.isPresent() && expiringBounce.get().getDeployId().equalsIgnoreCase(requestCleanup.getDeployId().get())) { LOG.info("No running tasks for request {}. Marking bounce {} complete and starting new tasks", expiringBounce.get().getRequestId(), expiringBounce.get()); requestManager.removeExpiringBounce(requestCleanup.getRequestId()); } requestManager.markBounceComplete(requestCleanup.getRequestId()); } requestManager.addToPendingQueue(new SingularityPendingRequest(requestCleanup.getRequestId(), requestCleanup.getDeployId().get(), requestCleanup.getTimestamp(), requestCleanup.getUser(), PendingType.BOUNCE, Optional.absent(), Optional.absent(), requestCleanup.getSkipHealthchecks(), requestCleanup.getMessage(), requestCleanup.getActionId())); LOG.info("Added {} tasks for request {} to cleanup bounce queue in {}", matchingTaskIds.size(), requestCleanup.getRequestId(), JavaUtils.duration(start)); }
protected SingularityRequestParent fillEntireRequest(SingularityRequestWithState requestWithState, Optional<SingularityRequest> newRequestData) { final String requestId = requestWithState.getRequest().getId(); final Optional<SingularityRequestDeployState> requestDeployState = deployManager.getRequestDeployState(requestId); Optional<SingularityDeploy> activeDeploy = Optional.absent(); Optional<SingularityDeploy> pendingDeploy = Optional.absent(); if (requestDeployState.isPresent()) { activeDeploy = fillDeploy(requestDeployState.get().getActiveDeploy()); pendingDeploy = fillDeploy(requestDeployState.get().getPendingDeploy()); } Optional<SingularityPendingDeploy> pendingDeployState = deployManager.getPendingDeploy(requestId); return new SingularityRequestParent(newRequestData.or(requestWithState.getRequest()), requestWithState.getState(), requestDeployState, activeDeploy, pendingDeploy, pendingDeployState, requestManager.getExpiringBounce(requestId), requestManager.getExpiringPause(requestId), requestManager.getExpiringScale(requestId), requestManager.getExpiringSkipHealthchecks(requestId), requestHelper.getTaskIdsByStatusForRequest(requestId), requestHistoryHelper.getLastHistory(requestId), requestHelper.getMostRecentTask(requestWithState.getRequest())); }
if (includeFullRequestData) { CompletableFuture<Optional<SingularityTaskIdsByStatus>> maybeTaskIdsByStatus = CompletableFuture.supplyAsync(() -> getTaskIdsByStatusForRequest(requestWithState)).exceptionally((throwable) -> Optional.absent()); CompletableFuture<Optional<SingularityExpiringBounce>> maybeExpiringBounce = CompletableFuture.supplyAsync(() -> requestManager.getExpiringBounce(requestWithState.getRequest().getId())).exceptionally((throwable) -> Optional.absent()); CompletableFuture<Optional<SingularityExpiringPause>> maybeExpiringPause = CompletableFuture.supplyAsync(() -> requestManager.getExpiringPause(requestWithState.getRequest().getId())).exceptionally((throwable) -> Optional.absent()); CompletableFuture<Optional<SingularityExpiringScale>> maybeExpiringScale = CompletableFuture.supplyAsync(() -> requestManager.getExpiringScale(requestWithState.getRequest().getId())).exceptionally((throwable) -> Optional.absent());
Optional<SingularityExpiringBounce> expiringBounce = requestManager.getExpiringBounce(taskId.getRequestId());
@Test public void testExpiringBounceGoesAway() { initRequest(); initFirstDeploy(); startTask(firstDeploy, 1); requestResource.bounce(requestId, Optional.of(new SingularityBounceRequest(Optional.of(false), Optional.absent(), Optional.of(1L), Optional.absent(), Optional.of("msg"), Optional.absent())), singularityUser); cleaner.drainCleanupQueue(); resourceOffers(); runLaunchedTasks(); cleaner.drainCleanupQueue(); killKilledTasks(); Assert.assertTrue(taskManager.getCleanupTaskIds().isEmpty()); Assert.assertEquals(1, taskManager.getActiveTaskIds().size()); Assert.assertTrue(!requestManager.getExpiringBounce(requestId).isPresent()); }
@Test public void testBounceOnPendingInstancesReleasesLock() { initRequest(); initFirstDeploy(); SingularityTask task = startTask(firstDeploy, 1); statusUpdate(task, TaskState.TASK_FAILED); killKilledTasks(); Assert.assertEquals("Bounce starts when tasks have not yet been launched", 0, taskManager.getActiveTaskIds().size()); requestResource.bounce(requestId, Optional.of(new SingularityBounceRequest(Optional.absent(), Optional.of(true), Optional.absent(), Optional.absent(), Optional.absent(), Optional.absent())), singularityUser); // It acquires a lock on the bounce Assert.assertTrue("Lock on bounce should be acquired during bounce", requestManager.getExpiringBounce(requestId).isPresent()); cleaner.drainCleanupQueue(); scheduler.drainPendingQueue(); resourceOffers(); for (SingularityTaskId singularityTaskId : taskManager.getActiveTaskIds()) { taskManager.saveTaskHistoryUpdate(new SingularityTaskHistoryUpdate(singularityTaskId, System.currentTimeMillis(), ExtendedTaskState.TASK_RUNNING, Optional.absent(), Optional.absent(), Collections.emptySet())); } cleaner.drainCleanupQueue(); killKilledTasks(); // It finishes with one task running and the bounce released Assert.assertEquals("Should end bounce with target number of tasks", 1, taskManager.getActiveTaskIds().size()); for (SingularityTaskId singularityTaskId : taskManager.getActiveTaskIds()) { String statusMessage = taskManager.getTaskHistoryUpdates(singularityTaskId) .get(0) .getStatusMessage() .get(); Assert.assertTrue("Task was started by bounce", statusMessage.contains("BOUNCE")); } Assert.assertFalse("Lock on bounce should be released after bounce", requestManager.getExpiringBounce(requestId).isPresent()); }
Assert.assertTrue("Lock on bounce should be acquired during bounce", requestManager.getExpiringBounce(requestId).isPresent()); Assert.assertTrue("Task was started by bounce", statusMessage.contains("BOUNCE")); Assert.assertFalse("Lock on bounce should be released after bounce", requestManager.getExpiringBounce(requestId).isPresent());
Assert.assertTrue(!requestManager.getExpiringBounce(requestId).isPresent()); Assert.assertTrue(requestManager.getPendingRequests().isEmpty()); Assert.assertTrue(taskManager.getPendingTaskIds().isEmpty());