@Override public void runActionOnPoll() { if (!disasterManager.isDisabled(SingularityAction.RUN_DEPLOY_POLLER)) { final long start = System.currentTimeMillis(); final int numDeploys = deployChecker.checkDeploys(); if (numDeploys == 0) { LOG.trace("No pending deploys"); } else { LOG.info("Checked {} deploys in {}", numDeploys, JavaUtils.duration(start)); } } else { LOG.warn("Deploy poller is currently disabled"); } } }
@Test public void testQueueMultipleOneOffs() { SingularityRequestBuilder bldr = new SingularityRequestBuilder(requestId, RequestType.ON_DEMAND); requestResource.postRequest(bldr.build(), singularityUser); deploy("on_demand_deploy"); deployChecker.checkDeploys(); requestManager.addToPendingQueue(new SingularityPendingRequest(requestId, "on_demand_deploy", System.currentTimeMillis(), Optional.<String>absent(), PendingType.ONEOFF, Optional.<List<String>>absent(), Optional.<String>absent(), Optional.<Boolean>absent(), Optional.<String>absent(), Optional.<String>absent())); requestManager.addToPendingQueue(new SingularityPendingRequest(requestId, "on_demand_deploy", System.currentTimeMillis(), Optional.<String>absent(), PendingType.ONEOFF, Optional.<List<String>>absent(), Optional.<String>absent(), Optional.<Boolean>absent(), Optional.<String>absent(), Optional.<String>absent())); scheduler.drainPendingQueue(); Assert.assertEquals(2, taskManager.getPendingTaskIds().size()); }
@Test public void testMaxOnDemandTasks() { SingularityRequestBuilder bldr = new SingularityRequestBuilder(requestId, RequestType.ON_DEMAND); bldr.setInstances(Optional.of(1)); requestResource.postRequest(bldr.build(), singularityUser); deploy("on_demand_deploy"); deployChecker.checkDeploys(); requestManager.addToPendingQueue(new SingularityPendingRequest(requestId, "on_demand_deploy", System.currentTimeMillis(), Optional.<String>absent(), PendingType.ONEOFF, Optional.<List<String>>absent(), Optional.<String>absent(), Optional.<Boolean>absent(), Optional.<String>absent(), Optional.<String>absent())); requestManager.addToPendingQueue(new SingularityPendingRequest(requestId, "on_demand_deploy", System.currentTimeMillis(), Optional.<String>absent(), PendingType.ONEOFF, Optional.<List<String>>absent(), Optional.<String>absent(), Optional.<Boolean>absent(), Optional.<String>absent(), Optional.<String>absent())); scheduler.drainPendingQueue(); resourceOffers(); Assert.assertEquals(1, taskManager.getActiveTaskIds().size()); }
@Test public void testDeployFailsForInvalidRequestState() { initRequest(); SingularityRequest request = requestResource.getRequest(requestId, singularityUser).getRequest(); initFirstDeploy(); deploy(secondDeployId, Optional.<Boolean>absent(), Optional.of(1), Optional.of(false), false); requestManager.pause(request, System.currentTimeMillis(), Optional.<String>absent(), Optional.<String>absent()); deployChecker.checkDeploys(); Assert.assertEquals(DeployState.FAILED, deployManager.getDeployResult(requestId, secondDeployId).get().getDeployState()); }
@Test public void testDeployTimesOut() { initRequest(); final long hourAgo = System.currentTimeMillis() - TimeUnit.HOURS.toMillis(1); final String deployId = "timeout_test"; SingularityDeployBuilder db = new SingularityDeployBuilder(requestId, deployId); db.setDeployHealthTimeoutSeconds(Optional.of(TimeUnit.MINUTES.toSeconds(1))); initDeploy(db, hourAgo); deployChecker.checkDeploys(); Assert.assertEquals(DeployState.OVERDUE, deployManager.getDeployResult(requestId, deployId).get().getDeployState()); }
@Test public void testRemovedRequestData() { long now = System.currentTimeMillis(); initRequest(); SingularityDeployBuilder db = new SingularityDeployBuilder(requestId, firstDeployId); db.setMaxTaskRetries(Optional.of(1)); initDeploy(db, now); deployChecker.checkDeploys(); Assert.assertEquals(DeployState.WAITING, deployManager.getPendingDeploys().get(0).getCurrentDeployState()); requestManager.startDeletingRequest(request, Optional.absent(), Optional.<String>absent(), Optional.<String>absent(), Optional.<String>absent()); requestManager.markDeleted(request, now, Optional.<String>absent(), Optional.<String>absent()); deployChecker.checkDeploys(); SingularityDeployResult deployResult = deployManager.getDeployResult(requestId, firstDeployId).get(); Assert.assertEquals(DeployState.FAILED, deployResult.getDeployState()); Assert.assertTrue(deployResult.getMessage().get().contains("MISSING")); }
@Test public void testDeployFailsAfterMaxTaskRetries() { initRequest(); SingularityDeployBuilder db = new SingularityDeployBuilder(requestId, firstDeployId); db.setMaxTaskRetries(Optional.of(1)); SingularityDeploy deploy = initDeploy(db, System.currentTimeMillis()); deployChecker.checkDeploys(); Assert.assertTrue(!deployManager.getDeployResult(requestId, firstDeployId).isPresent()); SingularityTask task = launchTask(request, deploy, System.currentTimeMillis(), 1, TaskState.TASK_FAILED); deployChecker.checkDeploys(); Assert.assertEquals(deployManager.getPendingDeploys().get(0).getCurrentDeployState(), DeployState.WAITING); SingularityTask taskTryTwo = launchTask(request, deploy, System.currentTimeMillis(), 1, TaskState.TASK_FAILED); deployChecker.checkDeploys(); Assert.assertEquals(deployManager.getDeployResult(requestId, firstDeployId).get().getDeployState(), DeployState.FAILED); }
@Test public void testDeploySucceedsWithTaskRetries() { initRequest(); SingularityDeployBuilder db = new SingularityDeployBuilder(requestId, firstDeployId); db.setMaxTaskRetries(Optional.of(1)); SingularityDeploy deploy = initDeploy(db, System.currentTimeMillis()); deployChecker.checkDeploys(); Assert.assertTrue(!deployManager.getDeployResult(requestId, firstDeployId).isPresent()); SingularityTask task = launchTask(request, deploy, System.currentTimeMillis(), 1, TaskState.TASK_FAILED); deployChecker.checkDeploys(); Assert.assertEquals(deployManager.getPendingDeploys().get(0).getCurrentDeployState(), DeployState.WAITING); SingularityTask taskTryTwo = launchTask(request, deploy, System.currentTimeMillis(), 1, TaskState.TASK_RUNNING); deployChecker.checkDeploys(); Assert.assertEquals(deployManager.getDeployResult(requestId, firstDeployId).get().getDeployState(), DeployState.SUCCEEDED); }
@Test public void testSkipDeployHealthchecks() { initRequest(); final String deployId = "deploy_test"; SingularityDeployBuilder db = new SingularityDeployBuilder(requestId, deployId); db.setHealthcheckUri(Optional.of("http://uri")); db.setSkipHealthchecksOnDeploy(Optional.of(true)); SingularityDeploy deploy = initDeploy(db, System.currentTimeMillis()); deployChecker.checkDeploys(); Assert.assertTrue(!deployManager.getDeployResult(requestId, deployId).isPresent()); launchTask(request, deploy, System.currentTimeMillis(), 1, TaskState.TASK_RUNNING); deployChecker.checkDeploys(); Assert.assertEquals(DeployState.SUCCEEDED, deployManager.getDeployResult(requestId, deployId).get().getDeployState()); }
@Test public void testStartupTimeout() { initRequest(); final long hourAgo = System.currentTimeMillis() - TimeUnit.HOURS.toMillis(1); final String deployId = "startup_timeout_test"; HealthcheckOptions options = new HealthcheckOptionsBuilder("http://uri").setMaxRetries(Optional.of(2)).setStartupTimeoutSeconds(Optional.of((int) TimeUnit.MINUTES.toSeconds(30))).build(); SingularityDeployBuilder db = new SingularityDeployBuilder(requestId, deployId).setHealthcheck(Optional.of(options)); db.setDeployHealthTimeoutSeconds(Optional.of(TimeUnit.DAYS.toMillis(1))); SingularityDeploy deploy = initDeploy(db, hourAgo); deployChecker.checkDeploys(); Assert.assertTrue(!deployManager.getDeployResult(requestId, deployId).isPresent()); SingularityTask task = launchTask(request, deploy, hourAgo, hourAgo + 1, 1, TaskState.TASK_RUNNING); deployChecker.checkDeploys(); Assert.assertTrue(!deployManager.getDeployResult(requestId, deployId).isPresent()); taskManager.saveHealthcheckResult(new SingularityTaskHealthcheckResult(Optional.<Integer>absent(), Optional.of(1000L), hourAgo + 1, Optional.<String> absent(), Optional.of("ERROR"), task.getTaskId(), Optional.of(true))); deployChecker.checkDeploys(); Assert.assertEquals(DeployState.FAILED, deployManager.getDeployResult(requestId, deployId).get().getDeployState()); }
@Test public void testHealthchecksTimeout() { initRequest(); final long hourAgo = System.currentTimeMillis() - TimeUnit.HOURS.toMillis(1); final String deployId = "timeout_test"; HealthcheckOptions options = new HealthcheckOptionsBuilder("http://uri").setMaxRetries(Optional.of(2)).build(); SingularityDeployBuilder db = new SingularityDeployBuilder(requestId, deployId).setHealthcheck(Optional.of(options)); db.setDeployHealthTimeoutSeconds(Optional.of(TimeUnit.DAYS.toMillis(1))); SingularityDeploy deploy = initDeploy(db, hourAgo); deployChecker.checkDeploys(); Assert.assertTrue(!deployManager.getDeployResult(requestId, deployId).isPresent()); SingularityTask task = launchTask(request, deploy, hourAgo, hourAgo + 1, 1, TaskState.TASK_RUNNING); deployChecker.checkDeploys(); Assert.assertTrue(!deployManager.getDeployResult(requestId, deployId).isPresent()); taskManager.saveHealthcheckResult(new SingularityTaskHealthcheckResult(Optional.of(503), Optional.of(1000L), hourAgo + 1, Optional.<String> absent(), Optional.<String> absent(), task.getTaskId(), Optional.<Boolean>absent())); deployChecker.checkDeploys(); Assert.assertEquals(DeployState.FAILED, deployManager.getDeployResult(requestId, deployId).get().getDeployState()); }
@Test public void testOnDemandDoesntGetRescheduled() { saveRequest(new SingularityRequestBuilder(requestId, RequestType.ON_DEMAND).build()); deploy(firstDeployId); deployChecker.checkDeploys(); Assert.assertTrue(requestManager.getPendingRequests().isEmpty()); Assert.assertTrue(taskManager.getPendingTaskIds().isEmpty()); startup.checkSchedulerForInconsistentState(); Assert.assertTrue(requestManager.getPendingRequests().isEmpty()); Assert.assertTrue(taskManager.getPendingTaskIds().isEmpty()); requestManager.addToPendingQueue(new SingularityPendingRequest(requestId, firstDeployId, System.currentTimeMillis(), Optional.<String> absent(), PendingType.ONEOFF, Optional.<Boolean> absent(), Optional.<String> absent())); startup.checkSchedulerForInconsistentState(); Assert.assertTrue(requestManager.getPendingRequests().get(0).getPendingType() == PendingType.ONEOFF); }
@Test public void testDeployFails() { initRequest(); SingularityRequest request = requestResource.getRequest(requestId, singularityUser).getRequest(); initFirstDeploy(); SingularityTask firstTask = launchTask(request, firstDeploy, 1, TaskState.TASK_RUNNING); deploy(secondDeployId, Optional.<Boolean>absent(), Optional.of(1), Optional.of(false), false); deployChecker.checkDeploys(); scheduler.drainPendingQueue(); Assert.assertEquals(1, taskManager.getPendingTaskIds().size()); resourceOffers(); Assert.assertEquals(1, taskManager.getActiveTaskIdsForDeploy(requestId, secondDeployId).size()); SingularityTaskId firstNewTaskId = taskManager.getActiveTaskIdsForDeploy(requestId, secondDeployId).get(0); statusUpdate(taskManager.getTask(firstNewTaskId).get(), TaskState.TASK_FAILED); deployChecker.checkDeploys(); Assert.assertFalse(taskManager.getCleanupTaskIds().contains(firstTask.getTaskId())); Assert.assertEquals(DeployState.FAILED, deployManager.getDeployResult(requestId, secondDeployId).get().getDeployState()); }
@Test public void testRunOnceDontMoveDuringDecomission() { SingularityRequestBuilder bldr = new SingularityRequestBuilder(requestId, RequestType.RUN_ONCE); request = bldr.build(); saveRequest(request); deployResource.deploy(new SingularityDeployRequest(new SingularityDeployBuilder(requestId, "d1").setCommand(Optional.of("cmd")).build(), Optional.absent(), Optional.absent()), singularityUser); scheduler.drainPendingQueue(); deployChecker.checkDeploys(); validateTaskDoesntMoveDuringDecommission(); }
@Test public void testMultipleRunOnceTasks() { SingularityRequestBuilder bldr = new SingularityRequestBuilder(requestId, RequestType.RUN_ONCE); request = bldr.build(); saveRequest(request); deployResource.deploy(new SingularityDeployRequest(new SingularityDeployBuilder(requestId, "d1").setCommand(Optional.of("cmd")).build(), Optional.absent(), Optional.absent()), singularityUser); deployChecker.checkDeploys(); Assert.assertEquals(1, requestManager.getSizeOfPendingQueue()); deployResource.deploy(new SingularityDeployRequest(new SingularityDeployBuilder(requestId, "d2").setCommand(Optional.of("cmd")).build(), Optional.absent(), Optional.absent()), singularityUser); deployChecker.checkDeploys(); Assert.assertEquals(2, requestManager.getSizeOfPendingQueue()); scheduler.drainPendingQueue(); resourceOffers(); Assert.assertEquals(2, taskManager.getActiveTaskIds().size()); }
@Test public void testLongRunningTaskKills() { initScheduledRequest(); initFirstDeploy(); launchTask(request, firstDeploy, 1, TaskState.TASK_RUNNING); initSecondDeploy(); deployChecker.checkDeploys(); Assert.assertTrue(taskManager.getKilledTaskIdRecords().isEmpty()); Assert.assertTrue(!taskManager.getCleanupTasks().isEmpty()); cleaner.drainCleanupQueue(); Assert.assertTrue(taskManager.getKilledTaskIdRecords().isEmpty()); Assert.assertTrue(!taskManager.getCleanupTasks().isEmpty()); requestManager.activate(request.toBuilder().setKillOldNonLongRunningTasksAfterMillis(Optional.<Long>of(0L)).build(), RequestHistoryType.CREATED, System.currentTimeMillis(), Optional.<String>absent(), Optional.<String>absent()); cleaner.drainCleanupQueue(); Assert.assertTrue(!taskManager.getKilledTaskIdRecords().isEmpty()); Assert.assertTrue(taskManager.getCleanupTasks().isEmpty()); }
@Test public void testScheduledJobLivesThroughDeploy() { initScheduledRequest(); initFirstDeploy(); createAndSchedulePendingTask(firstDeployId); Assert.assertTrue(!taskManager.getPendingTaskIds().isEmpty()); deploy("d2"); scheduler.drainPendingQueue(); deployChecker.checkDeploys(); scheduler.drainPendingQueue(); Assert.assertTrue(!taskManager.getPendingTaskIds().isEmpty()); }
@Test public void testFinishedRequestCanBeDeployed() { initScheduledRequest(); initFirstDeploy(); schedule = "*/1 * * * * ? 1995"; // cause it to be pending requestResource.postRequest(request.toBuilder().setQuartzSchedule(Optional.of(schedule)).build(), singularityUser); scheduler.drainPendingQueue(); Assert.assertTrue(requestResource.getActiveRequests(singularityUser, false, false, false, 10, Collections.emptyList()).isEmpty()); Assert.assertTrue(requestManager.getRequest(requestId).get().getState() == RequestState.FINISHED); SingularityDeployBuilder db = new SingularityDeployBuilder(requestId, secondDeployId); initDeploy(db, System.currentTimeMillis()); deployChecker.checkDeploys(); Assert.assertEquals(RequestState.ACTIVE, requestManager.getRequest(requestId).get().getState()); Assert.assertEquals(1, requestManager.getPendingRequests().size()); }
@Test public void testDeployCleanupOverwritesTaskBounceCleanup() { initRequest(); initFirstDeploy(); final SingularityTask oldTask = startTask(firstDeploy); taskResource .killTask(oldTask.getTaskId().getId(), Optional.of(new SingularityKillTaskRequest(Optional.absent(), Optional.absent(), Optional.absent(), Optional.of(true), Optional.absent())), singularityUser); final Optional<SingularityTaskCleanup> taskCleanup = taskManager.getTaskCleanup(oldTask.getTaskId().getId()); Assert.assertTrue(taskCleanup.isPresent()); Assert.assertEquals(TaskCleanupType.USER_REQUESTED_TASK_BOUNCE, taskCleanup.get().getCleanupType()); initSecondDeploy(); startTask(secondDeploy); deployChecker.checkDeploys(); Assert.assertEquals(DeployState.SUCCEEDED, deployManager.getDeployResult(requestId, secondDeployId).get().getDeployState()); Assert.assertEquals(TaskCleanupType.DEPLOY_STEP_FINISHED, taskManager.getTaskCleanup(oldTask.getTaskId().getId()).get().getCleanupType()); cleaner.drainCleanupQueue(); Assert.assertFalse(taskManager.getTaskCleanup(oldTask.getTaskId().getId()).isPresent()); }
@Test public void testRunOnceDoesntGetRescheduled() { saveRequest(new SingularityRequestBuilder(requestId, RequestType.RUN_ONCE).build()); deploy(firstDeployId); deployChecker.checkDeploys(); scheduler.drainPendingQueue(); resourceOffers(); Assert.assertTrue(requestManager.getPendingRequests().isEmpty()); Assert.assertTrue(taskManager.getPendingTaskIds().isEmpty()); startup.checkSchedulerForInconsistentState(); // assert that SingularityStartup does not enqueue a SingularityPendingRequest (pendingType=NOT_STARTED) for the RUN_ONCE request Assert.assertTrue(requestManager.getPendingRequests().isEmpty()); } }