private TaskCleanupType getCleanupType(SingularityPendingDeploy pendingDeploy, SingularityRequest request, SingularityDeployResult deployResult) { if (pendingDeploy.getDeployProgress().isPresent() && pendingDeploy.getDeployProgress().get().getDeployInstanceCountPerStep() != request.getInstancesSafe()) { // For incremental deploys, return a special cleanup type if (deployResult.getDeployState() == DeployState.FAILED) { return TaskCleanupType.INCREMENTAL_DEPLOY_FAILED; } else if (deployResult.getDeployState() == DeployState.CANCELED) { return TaskCleanupType.INCREMENTAL_DEPLOY_CANCELLED; } } return deployResult.getDeployState().getCleanupType(); }
private void cleanupTasks(SingularityPendingDeploy pendingDeploy, SingularityRequest request, SingularityDeployResult deployResult, Iterable<SingularityTaskId> tasksToKill) { for (SingularityTaskId matchingTask : tasksToKill) { taskManager.saveTaskCleanup(new SingularityTaskCleanup(pendingDeploy.getDeployMarker().getUser(), getCleanupType(pendingDeploy, request, deployResult), deployResult.getTimestamp(), matchingTask, Optional.of(String.format("Deploy %s - %s", pendingDeploy.getDeployMarker().getDeployId(), deployResult.getDeployState().name())), Optional.<String> absent(), Optional.<SingularityTaskShellCommandRequestId>absent())); } }
@Override public void saveDeployHistory(SingularityDeployHistory deployHistory) { if (LOG.isTraceEnabled()) { LOG.trace("saveDeployHistory {}", deployHistory); } history.insertDeployHistory(deployHistory.getDeployMarker().getRequestId(), deployHistory.getDeployMarker().getDeployId(), new Date(deployHistory.getDeployMarker().getTimestamp()), getUserField(deployHistory.getDeployMarker().getUser()), getMessageField(deployHistory.getDeployMarker().getMessage()), deployHistory.getDeployResult().isPresent() ? new Date(deployHistory.getDeployResult().get().getTimestamp()) : new Date(deployHistory.getDeployMarker().getTimestamp()), deployHistory.getDeployResult().isPresent() ? deployHistory.getDeployResult().get().getDeployState().name() : DeployState.CANCELED.name(), deployHistoryTranscoder.toBytes(deployHistory)); }
@Test public void testDeployTimesOut() { initRequest(); final long hourAgo = System.currentTimeMillis() - TimeUnit.HOURS.toMillis(1); final String deployId = "timeout_test"; SingularityDeployBuilder db = new SingularityDeployBuilder(requestId, deployId); db.setDeployHealthTimeoutSeconds(Optional.of(TimeUnit.MINUTES.toSeconds(1))); initDeploy(db, hourAgo); deployChecker.checkDeploys(); Assert.assertEquals(DeployState.OVERDUE, deployManager.getDeployResult(requestId, deployId).get().getDeployState()); }
@Test public void testDeployFailsForInvalidRequestState() { initRequest(); SingularityRequest request = requestResource.getRequest(requestId, singularityUser).getRequest(); initFirstDeploy(); deploy(secondDeployId, Optional.<Boolean>absent(), Optional.of(1), Optional.of(false), false); requestManager.pause(request, System.currentTimeMillis(), Optional.<String>absent(), Optional.<String>absent()); deployChecker.checkDeploys(); Assert.assertEquals(DeployState.FAILED, deployManager.getDeployResult(requestId, secondDeployId).get().getDeployState()); }
@Test public void testRemovedRequestData() { long now = System.currentTimeMillis(); initRequest(); SingularityDeployBuilder db = new SingularityDeployBuilder(requestId, firstDeployId); db.setMaxTaskRetries(Optional.of(1)); initDeploy(db, now); deployChecker.checkDeploys(); Assert.assertEquals(DeployState.WAITING, deployManager.getPendingDeploys().get(0).getCurrentDeployState()); requestManager.startDeletingRequest(request, Optional.absent(), Optional.<String>absent(), Optional.<String>absent(), Optional.<String>absent()); requestManager.markDeleted(request, now, Optional.<String>absent(), Optional.<String>absent()); deployChecker.checkDeploys(); SingularityDeployResult deployResult = deployManager.getDeployResult(requestId, firstDeployId).get(); Assert.assertEquals(DeployState.FAILED, deployResult.getDeployState()); Assert.assertTrue(deployResult.getMessage().get().contains("MISSING")); }
@Test public void testMaxHealthcheckRetries() { initRequest(); final String deployId = "retry_test"; HealthcheckOptions options = new HealthcheckOptionsBuilder("http://uri").setMaxRetries(Optional.of(2)).build(); SingularityDeployBuilder db = new SingularityDeployBuilder(requestId, deployId).setHealthcheck(Optional.of(options)); SingularityDeploy deploy = initDeploy(db, System.currentTimeMillis()); deployChecker.checkDeploys(); Assert.assertTrue(!deployManager.getDeployResult(requestId, deployId).isPresent()); SingularityTask task = launchTask(request, deploy, System.currentTimeMillis(), 1, TaskState.TASK_RUNNING); deployChecker.checkDeploys(); Assert.assertTrue(!deployManager.getDeployResult(requestId, deployId).isPresent()); taskManager.saveHealthcheckResult(new SingularityTaskHealthcheckResult(Optional.of(503), Optional.of(1000L), System.currentTimeMillis(), Optional.<String> absent(), Optional.<String> absent(), task.getTaskId(), Optional.<Boolean>absent())); taskManager.saveHealthcheckResult(new SingularityTaskHealthcheckResult(Optional.of(503), Optional.of(1000L), System.currentTimeMillis() + 1, Optional.<String> absent(), Optional.<String> absent(), task.getTaskId(), Optional.<Boolean>absent())); deployChecker.checkDeploys(); Assert.assertTrue(!deployManager.getDeployResult(requestId, deployId).isPresent()); taskManager.saveHealthcheckResult(new SingularityTaskHealthcheckResult(Optional.of(503), Optional.of(1000L), System.currentTimeMillis() + 1, Optional.<String> absent(), Optional.<String> absent(), task.getTaskId(), Optional.<Boolean>absent())); deployChecker.checkDeploys(); Assert.assertEquals(DeployState.FAILED, deployManager.getDeployResult(requestId, deployId).get().getDeployState()); }
@Test public void testStartupDoesNotCountTowardsRetries() { initRequest(); final String deployId = "retry_test"; HealthcheckOptions options = new HealthcheckOptionsBuilder("http://uri").setMaxRetries(Optional.of(1)).build(); SingularityDeployBuilder db = new SingularityDeployBuilder(requestId, deployId).setHealthcheck(Optional.of(options)); SingularityDeploy deploy = initDeploy(db, System.currentTimeMillis()); deployChecker.checkDeploys(); Assert.assertTrue(!deployManager.getDeployResult(requestId, deployId).isPresent()); SingularityTask task = launchTask(request, deploy, System.currentTimeMillis(), 1, TaskState.TASK_RUNNING); deployChecker.checkDeploys(); Assert.assertTrue(!deployManager.getDeployResult(requestId, deployId).isPresent()); taskManager.saveHealthcheckResult(new SingularityTaskHealthcheckResult(Optional.<Integer>absent(), Optional.of(1000L), System.currentTimeMillis(), Optional.<String> absent(), Optional.of("ConnectionRefused"), task.getTaskId(), Optional.of(true))); taskManager.saveHealthcheckResult(new SingularityTaskHealthcheckResult(Optional.<Integer>absent(), Optional.of(1000L), System.currentTimeMillis() + 1, Optional.<String> absent(), Optional.of("ConnectionRefused"), task.getTaskId(), Optional.of(true))); deployChecker.checkDeploys(); Assert.assertTrue(!deployManager.getDeployResult(requestId, deployId).isPresent()); taskManager.saveHealthcheckResult(new SingularityTaskHealthcheckResult(Optional.of(503), Optional.of(1000L), System.currentTimeMillis() + 1, Optional.<String> absent(), Optional.<String> absent(), task.getTaskId(), Optional.<Boolean>absent())); deployChecker.checkDeploys(); Assert.assertTrue(!deployManager.getDeployResult(requestId, deployId).isPresent()); taskManager.saveHealthcheckResult(new SingularityTaskHealthcheckResult(Optional.of(200), Optional.of(1000L), System.currentTimeMillis() + 1, Optional.<String> absent(), Optional.<String> absent(), task.getTaskId(), Optional.<Boolean>absent())); deployChecker.checkDeploys(); Assert.assertEquals(DeployState.SUCCEEDED, deployManager.getDeployResult(requestId, deployId).get().getDeployState()); }
@Test public void testStartupTimeout() { initRequest(); final long hourAgo = System.currentTimeMillis() - TimeUnit.HOURS.toMillis(1); final String deployId = "startup_timeout_test"; HealthcheckOptions options = new HealthcheckOptionsBuilder("http://uri").setMaxRetries(Optional.of(2)).setStartupTimeoutSeconds(Optional.of((int) TimeUnit.MINUTES.toSeconds(30))).build(); SingularityDeployBuilder db = new SingularityDeployBuilder(requestId, deployId).setHealthcheck(Optional.of(options)); db.setDeployHealthTimeoutSeconds(Optional.of(TimeUnit.DAYS.toMillis(1))); SingularityDeploy deploy = initDeploy(db, hourAgo); deployChecker.checkDeploys(); Assert.assertTrue(!deployManager.getDeployResult(requestId, deployId).isPresent()); SingularityTask task = launchTask(request, deploy, hourAgo, hourAgo + 1, 1, TaskState.TASK_RUNNING); deployChecker.checkDeploys(); Assert.assertTrue(!deployManager.getDeployResult(requestId, deployId).isPresent()); taskManager.saveHealthcheckResult(new SingularityTaskHealthcheckResult(Optional.<Integer>absent(), Optional.of(1000L), hourAgo + 1, Optional.<String> absent(), Optional.of("ERROR"), task.getTaskId(), Optional.of(true))); deployChecker.checkDeploys(); Assert.assertEquals(DeployState.FAILED, deployManager.getDeployResult(requestId, deployId).get().getDeployState()); }
@Test public void testHealthchecksSuccess() { initRequest(); final String deployId = "hc_test"; HealthcheckOptions options = new HealthcheckOptionsBuilder("http://uri").setMaxRetries(Optional.of(2)).build(); SingularityDeployBuilder db = new SingularityDeployBuilder(requestId, deployId).setHealthcheck(Optional.of(options)); SingularityDeploy deploy = initDeploy(db, System.currentTimeMillis()); deployChecker.checkDeploys(); Assert.assertTrue(!deployManager.getDeployResult(requestId, deployId).isPresent()); SingularityTask task = launchTask(request, deploy, System.currentTimeMillis(), 1, TaskState.TASK_RUNNING); deployChecker.checkDeploys(); Assert.assertTrue(!deployManager.getDeployResult(requestId, deployId).isPresent()); taskManager.saveHealthcheckResult(new SingularityTaskHealthcheckResult(Optional.of(503), Optional.of(1000L), System.currentTimeMillis(), Optional.<String>absent(), Optional.<String>absent(), task.getTaskId(), Optional.<Boolean>absent())); deployChecker.checkDeploys(); Assert.assertTrue(!deployManager.getDeployResult(requestId, deployId).isPresent()); taskManager.saveHealthcheckResult(new SingularityTaskHealthcheckResult(Optional.of(200), Optional.of(1000L), System.currentTimeMillis() + 1, Optional.<String>absent(), Optional.<String>absent(), task.getTaskId(), Optional.<Boolean>absent())); deployChecker.checkDeploys(); Assert.assertEquals(DeployState.SUCCEEDED, deployManager.getDeployResult(requestId, deployId).get().getDeployState()); }
@Test public void testFailingStatusCodes() { initRequest(); final String deployId = "retry_test"; List<Integer> failureCodes = ImmutableList.of(404); HealthcheckOptions options = new HealthcheckOptionsBuilder("http://uri").setMaxRetries(Optional.of(3)).setFailureStatusCodes(Optional.of(failureCodes)).build(); SingularityDeployBuilder db = new SingularityDeployBuilder(requestId, deployId).setHealthcheck(Optional.of(options)); SingularityDeploy deploy = initDeploy(db, System.currentTimeMillis()); deployChecker.checkDeploys(); Assert.assertTrue(!deployManager.getDeployResult(requestId, deployId).isPresent()); SingularityTask task = launchTask(request, deploy, System.currentTimeMillis(), 1, TaskState.TASK_RUNNING); deployChecker.checkDeploys(); Assert.assertTrue(!deployManager.getDeployResult(requestId, deployId).isPresent()); taskManager.saveHealthcheckResult(new SingularityTaskHealthcheckResult(Optional.of(503), Optional.of(1000L), System.currentTimeMillis(), Optional.<String> absent(), Optional.<String> absent(), task.getTaskId(), Optional.<Boolean>absent())); deployChecker.checkDeploys(); Assert.assertTrue(!deployManager.getDeployResult(requestId, deployId).isPresent()); taskManager.saveHealthcheckResult(new SingularityTaskHealthcheckResult(Optional.of(404), Optional.of(1000L), System.currentTimeMillis() + 1, Optional.<String> absent(), Optional.<String> absent(), task.getTaskId(), Optional.<Boolean>absent())); deployChecker.checkDeploys(); // Bad status code should cause instant failure even though retries remain Assert.assertEquals(DeployState.FAILED, deployManager.getDeployResult(requestId, deployId).get().getDeployState()); }
@Test public void testSkipDeployHealthchecks() { initRequest(); final String deployId = "deploy_test"; SingularityDeployBuilder db = new SingularityDeployBuilder(requestId, deployId); db.setHealthcheckUri(Optional.of("http://uri")); db.setSkipHealthchecksOnDeploy(Optional.of(true)); SingularityDeploy deploy = initDeploy(db, System.currentTimeMillis()); deployChecker.checkDeploys(); Assert.assertTrue(!deployManager.getDeployResult(requestId, deployId).isPresent()); launchTask(request, deploy, System.currentTimeMillis(), 1, TaskState.TASK_RUNNING); deployChecker.checkDeploys(); Assert.assertEquals(DeployState.SUCCEEDED, deployManager.getDeployResult(requestId, deployId).get().getDeployState()); }
@Test public void testHealthchecksTimeout() { initRequest(); final long hourAgo = System.currentTimeMillis() - TimeUnit.HOURS.toMillis(1); final String deployId = "timeout_test"; HealthcheckOptions options = new HealthcheckOptionsBuilder("http://uri").setMaxRetries(Optional.of(2)).build(); SingularityDeployBuilder db = new SingularityDeployBuilder(requestId, deployId).setHealthcheck(Optional.of(options)); db.setDeployHealthTimeoutSeconds(Optional.of(TimeUnit.DAYS.toMillis(1))); SingularityDeploy deploy = initDeploy(db, hourAgo); deployChecker.checkDeploys(); Assert.assertTrue(!deployManager.getDeployResult(requestId, deployId).isPresent()); SingularityTask task = launchTask(request, deploy, hourAgo, hourAgo + 1, 1, TaskState.TASK_RUNNING); deployChecker.checkDeploys(); Assert.assertTrue(!deployManager.getDeployResult(requestId, deployId).isPresent()); taskManager.saveHealthcheckResult(new SingularityTaskHealthcheckResult(Optional.of(503), Optional.of(1000L), hourAgo + 1, Optional.<String> absent(), Optional.<String> absent(), task.getTaskId(), Optional.<Boolean>absent())); deployChecker.checkDeploys(); Assert.assertEquals(DeployState.FAILED, deployManager.getDeployResult(requestId, deployId).get().getDeployState()); }
@Test public void testDeployFailsAfterMaxTaskRetries() { initRequest(); SingularityDeployBuilder db = new SingularityDeployBuilder(requestId, firstDeployId); db.setMaxTaskRetries(Optional.of(1)); SingularityDeploy deploy = initDeploy(db, System.currentTimeMillis()); deployChecker.checkDeploys(); Assert.assertTrue(!deployManager.getDeployResult(requestId, firstDeployId).isPresent()); SingularityTask task = launchTask(request, deploy, System.currentTimeMillis(), 1, TaskState.TASK_FAILED); deployChecker.checkDeploys(); Assert.assertEquals(deployManager.getPendingDeploys().get(0).getCurrentDeployState(), DeployState.WAITING); SingularityTask taskTryTwo = launchTask(request, deploy, System.currentTimeMillis(), 1, TaskState.TASK_FAILED); deployChecker.checkDeploys(); Assert.assertEquals(deployManager.getDeployResult(requestId, firstDeployId).get().getDeployState(), DeployState.FAILED); }
@Test public void testDeploySucceedsWithTaskRetries() { initRequest(); SingularityDeployBuilder db = new SingularityDeployBuilder(requestId, firstDeployId); db.setMaxTaskRetries(Optional.of(1)); SingularityDeploy deploy = initDeploy(db, System.currentTimeMillis()); deployChecker.checkDeploys(); Assert.assertTrue(!deployManager.getDeployResult(requestId, firstDeployId).isPresent()); SingularityTask task = launchTask(request, deploy, System.currentTimeMillis(), 1, TaskState.TASK_FAILED); deployChecker.checkDeploys(); Assert.assertEquals(deployManager.getPendingDeploys().get(0).getCurrentDeployState(), DeployState.WAITING); SingularityTask taskTryTwo = launchTask(request, deploy, System.currentTimeMillis(), 1, TaskState.TASK_RUNNING); deployChecker.checkDeploys(); Assert.assertEquals(deployManager.getDeployResult(requestId, firstDeployId).get().getDeployState(), DeployState.SUCCEEDED); }
@Test public void testDeployFails() { initRequest(); SingularityRequest request = requestResource.getRequest(requestId, singularityUser).getRequest(); initFirstDeploy(); SingularityTask firstTask = launchTask(request, firstDeploy, 1, TaskState.TASK_RUNNING); deploy(secondDeployId, Optional.<Boolean>absent(), Optional.of(1), Optional.of(false), false); deployChecker.checkDeploys(); scheduler.drainPendingQueue(); Assert.assertEquals(1, taskManager.getPendingTaskIds().size()); resourceOffers(); Assert.assertEquals(1, taskManager.getActiveTaskIdsForDeploy(requestId, secondDeployId).size()); SingularityTaskId firstNewTaskId = taskManager.getActiveTaskIdsForDeploy(requestId, secondDeployId).get(0); statusUpdate(taskManager.getTask(firstNewTaskId).get(), TaskState.TASK_FAILED); deployChecker.checkDeploys(); Assert.assertFalse(taskManager.getCleanupTaskIds().contains(firstTask.getTaskId())); Assert.assertEquals(DeployState.FAILED, deployManager.getDeployResult(requestId, secondDeployId).get().getDeployState()); }
@Test public void testDeployCleanupOverwritesTaskBounceCleanup() { initRequest(); initFirstDeploy(); final SingularityTask oldTask = startTask(firstDeploy); taskResource .killTask(oldTask.getTaskId().getId(), Optional.of(new SingularityKillTaskRequest(Optional.absent(), Optional.absent(), Optional.absent(), Optional.of(true), Optional.absent())), singularityUser); final Optional<SingularityTaskCleanup> taskCleanup = taskManager.getTaskCleanup(oldTask.getTaskId().getId()); Assert.assertTrue(taskCleanup.isPresent()); Assert.assertEquals(TaskCleanupType.USER_REQUESTED_TASK_BOUNCE, taskCleanup.get().getCleanupType()); initSecondDeploy(); startTask(secondDeploy); deployChecker.checkDeploys(); Assert.assertEquals(DeployState.SUCCEEDED, deployManager.getDeployResult(requestId, secondDeployId).get().getDeployState()); Assert.assertEquals(TaskCleanupType.DEPLOY_STEP_FINISHED, taskManager.getTaskCleanup(oldTask.getTaskId().getId()).get().getCleanupType()); cleaner.drainCleanupQueue(); Assert.assertFalse(taskManager.getTaskCleanup(oldTask.getTaskId().getId()).isPresent()); }
@Test public void testCancelDeploy() { initRequest(); SingularityRequest request = requestResource.getRequest(requestId, singularityUser).getRequest(); initFirstDeploy(); SingularityTask firstTask = launchTask(request, firstDeploy, 1, TaskState.TASK_RUNNING); deploy(secondDeployId, Optional.<Boolean>absent(), Optional.of(1), Optional.of(false), false); deployChecker.checkDeploys(); scheduler.drainPendingQueue(); Assert.assertEquals(1, taskManager.getPendingTaskIds().size()); resourceOffers(); Assert.assertEquals(1, taskManager.getActiveTaskIdsForDeploy(requestId, secondDeployId).size()); SingularityTaskId firstNewTaskId = taskManager.getActiveTaskIdsForDeploy(requestId, secondDeployId).get(0); statusUpdate(taskManager.getTask(firstNewTaskId).get(), TaskState.TASK_RUNNING); deployResource.cancelDeploy(singularityUser, requestId, secondDeployId); deployChecker.checkDeploys(); Assert.assertTrue(taskManager.getCleanupTaskIds().contains(firstNewTaskId)); Assert.assertFalse(taskManager.getCleanupTaskIds().contains(firstTask.getTaskId())); Assert.assertEquals(DeployState.CANCELED, deployManager.getDeployResult(requestId, secondDeployId).get().getDeployState()); }
@Test public void testDeployManagerHandlesFailedLBTask() { initLoadBalancedRequest(); initFirstDeploy(); SingularityTask firstTask = startTask(firstDeploy); initSecondDeploy(); SingularityTask secondTask = startTask(secondDeploy); // this should cause an LB call to happen: deployChecker.checkDeploys(); Assert.assertTrue(taskManager.getLoadBalancerState(secondTask.getTaskId(), LoadBalancerRequestType.ADD).isPresent()); Assert.assertTrue(!taskManager.getLoadBalancerState(secondTask.getTaskId(), LoadBalancerRequestType.DEPLOY).isPresent()); Assert.assertTrue(!taskManager.getLoadBalancerState(secondTask.getTaskId(), LoadBalancerRequestType.REMOVE).isPresent()); statusUpdate(secondTask, TaskState.TASK_FAILED); statusUpdate(firstTask, TaskState.TASK_FAILED); scheduler.drainPendingQueue(); deployChecker.checkDeploys(); Assert.assertTrue(deployManager.getDeployResult(requestId, secondDeployId).get().getDeployState() == DeployState.FAILED); List<SingularityPendingTask> pendingTasks = taskManager.getPendingTasks(); Assert.assertTrue(pendingTasks.size() == 1); Assert.assertTrue(pendingTasks.get(0).getPendingTaskId().getDeployId().equals(firstDeployId)); }
@Test public void testDeployAllInstancesAtOnce() { initRequest(); SingularityRequest request = requestResource.getRequest(requestId, singularityUser).getRequest(); requestResource.postRequest(request.toBuilder().setInstances(Optional.of(2)).build(), singularityUser); initFirstDeploy(); launchTask(request, firstDeploy, 1, TaskState.TASK_RUNNING); launchTask(request, firstDeploy, 2, TaskState.TASK_RUNNING); deploy(secondDeployId); deployChecker.checkDeploys(); scheduler.drainPendingQueue(); Assert.assertEquals(2, taskManager.getActiveTaskIds().size()); Assert.assertEquals(2, taskManager.getPendingTaskIds().size()); resourceOffers(); Assert.assertEquals(2, taskManager.getActiveTaskIdsForDeploy(requestId, secondDeployId).size()); for (SingularityTaskId taskId : taskManager.getActiveTaskIdsForDeploy(requestId, secondDeployId)) { statusUpdate(taskManager.getTask(taskId).get(), TaskState.TASK_RUNNING); } deployChecker.checkDeploys(); Assert.assertEquals(2, taskManager.getCleanupTaskIds().size()); Assert.assertEquals(DeployState.SUCCEEDED, deployManager.getDeployResult(requestId, secondDeployId).get().getDeployState()); Assert.assertEquals(2, taskManager.getActiveTaskIdsForDeploy(requestId, secondDeployId).size()); }