private SingularityRequestParent getAndLogRequestAndDeployStatus(SingularityRequestParent singularityRequestParent) { String activeDeployId = singularityRequestParent.getActiveDeploy().isPresent() ? singularityRequestParent.getActiveDeploy().get().getId() : "No Active Deploy"; String pendingDeployId = singularityRequestParent.getPendingDeploy().isPresent() ? singularityRequestParent.getPendingDeploy().get().getId() : "No Pending deploy"; LOG.info("Deploy status: Singularity request {} -> pending deploy: '{}', active deploy: '{}'", singularityRequestParent.getRequest().getId(), pendingDeployId, activeDeployId); return singularityRequestParent; }
public static SingularityPendingRequestParent fromSingularityRequestParent(SingularityRequestParent singularityRequestParent, SingularityPendingRequest pendingRequest) { return new SingularityPendingRequestParent(singularityRequestParent.getRequest(), singularityRequestParent.getState(), singularityRequestParent.getRequestDeployState(), singularityRequestParent.getActiveDeploy(), singularityRequestParent.getPendingDeploy(), singularityRequestParent.getPendingDeployState(), pendingRequest, singularityRequestParent.getExpiringBounce(), singularityRequestParent.getExpiringPause(), singularityRequestParent.getExpiringScale(), singularityRequestParent.getExpiringSkipHealthchecks(), singularityRequestParent.getTaskIds(), singularityRequestParent.getLastHistory(), singularityRequestParent.getMostRecentTask()); }
@Test public void testDeployFailsForInvalidRequestState() { initRequest(); SingularityRequest request = requestResource.getRequest(requestId, singularityUser).getRequest(); initFirstDeploy(); deploy(secondDeployId, Optional.<Boolean>absent(), Optional.of(1), Optional.of(false), false); requestManager.pause(request, System.currentTimeMillis(), Optional.<String>absent(), Optional.<String>absent()); deployChecker.checkDeploys(); Assert.assertEquals(DeployState.FAILED, deployManager.getDeployResult(requestId, secondDeployId).get().getDeployState()); }
assertEquals(request, requestParent.get().getRequest());
@Test public void itDoesntFlagPendingRequestsForUnderOrOverProvisioning() { initRequest(); initFirstDeploy(); SingularityRequest request = requestResource.getRequest(requestId, singularityUser).getRequest(); requestManager.activate(request.toBuilder().setInstances(Optional.of(0)).build(), RequestHistoryType.UPDATED, System.currentTimeMillis(), Optional.<String> absent(), Optional.<String> absent()); requestManager.addToPendingQueue(new SingularityPendingRequest(request.getId(), firstDeployId, System.currentTimeMillis(), Optional.<String> absent(), PendingType.ONEOFF, Optional.<Boolean> absent(), Optional.<String> absent())); Assert.assertEquals(0, taskManager.getActiveTaskIds().size()); SingularityState state = stateManager.getState(true, false); Assert.assertEquals(0, state.getOverProvisionedRequests()); Assert.assertEquals(0, state.getUnderProvisionedRequests()); } }
@Test(expected = WebApplicationException.class) public void testCannotUpdateRequestDuringPendingDeployWithNewData() { initRequest(); SingularityRequest request = requestResource.getRequest(requestId, singularityUser).getRequest(); SingularityRequest newRequest = request.toBuilder().setInstances(Optional.of(1)).build(); String deployId = "test_new_request_data"; SingularityDeploy deploy = new SingularityDeployBuilder(request.getId(), deployId).setCommand(Optional.of("sleep 100")).build(); deployResource.deploy(new SingularityDeployRequest(deploy, Optional.absent(), Optional.absent(), Optional.of(newRequest)), singularityUser); requestResource.postRequest(newRequest, singularityUser); }
resourceOffers(2); // set up slaves so scale validate will pass SingularityRequest request = requestResource.getRequest(requestId, singularityUser).getRequest();
@Test public void testDeployFails() { initRequest(); SingularityRequest request = requestResource.getRequest(requestId, singularityUser).getRequest(); initFirstDeploy(); SingularityTask firstTask = launchTask(request, firstDeploy, 1, TaskState.TASK_RUNNING); deploy(secondDeployId, Optional.<Boolean>absent(), Optional.of(1), Optional.of(false), false); deployChecker.checkDeploys(); scheduler.drainPendingQueue(); Assert.assertEquals(1, taskManager.getPendingTaskIds().size()); resourceOffers(); Assert.assertEquals(1, taskManager.getActiveTaskIdsForDeploy(requestId, secondDeployId).size()); SingularityTaskId firstNewTaskId = taskManager.getActiveTaskIdsForDeploy(requestId, secondDeployId).get(0); statusUpdate(taskManager.getTask(firstNewTaskId).get(), TaskState.TASK_FAILED); deployChecker.checkDeploys(); Assert.assertFalse(taskManager.getCleanupTaskIds().contains(firstTask.getTaskId())); Assert.assertEquals(DeployState.FAILED, deployManager.getDeployResult(requestId, secondDeployId).get().getDeployState()); }
@Test public void itSetsRequestStateToDeletedAfterAllTasksAreCleanedUp() { initRequest(); SingularityRequest request = requestResource.getRequest(requestId, singularityUser).getRequest(); requestResource.postRequest(request.toBuilder().setInstances(Optional.of(2)).build(), singularityUser); initFirstDeploy(); launchTask(request, firstDeploy, 1, TaskState.TASK_RUNNING); launchTask(request, firstDeploy, 2, TaskState.TASK_RUNNING); Assert.assertEquals(requestId, requestManager.getActiveRequests().iterator().next().getRequest().getId()); Assert.assertEquals(2, taskManager.getActiveTaskIds().size()); requestManager.startDeletingRequest(request, Optional.absent(), Optional.absent(), Optional.absent(), Optional.absent()); Assert.assertEquals(requestId, requestManager.getCleanupRequests().get(0).getRequestId()); Assert.assertEquals(RequestState.DELETING, requestManager.getRequest(requestId).get().getState()); cleaner.drainCleanupQueue(); Assert.assertEquals(0, taskManager.getCleanupTaskIds().size()); killKilledTasks(); cleaner.drainCleanupQueue(); Assert.assertFalse(requestManager.getRequest(requestId).isPresent()); }
@Test public void testCancelDeploy() { initRequest(); SingularityRequest request = requestResource.getRequest(requestId, singularityUser).getRequest(); initFirstDeploy(); SingularityTask firstTask = launchTask(request, firstDeploy, 1, TaskState.TASK_RUNNING); deploy(secondDeployId, Optional.<Boolean>absent(), Optional.of(1), Optional.of(false), false); deployChecker.checkDeploys(); scheduler.drainPendingQueue(); Assert.assertEquals(1, taskManager.getPendingTaskIds().size()); resourceOffers(); Assert.assertEquals(1, taskManager.getActiveTaskIdsForDeploy(requestId, secondDeployId).size()); SingularityTaskId firstNewTaskId = taskManager.getActiveTaskIdsForDeploy(requestId, secondDeployId).get(0); statusUpdate(taskManager.getTask(firstNewTaskId).get(), TaskState.TASK_RUNNING); deployResource.cancelDeploy(singularityUser, requestId, secondDeployId); deployChecker.checkDeploys(); Assert.assertTrue(taskManager.getCleanupTaskIds().contains(firstNewTaskId)); Assert.assertFalse(taskManager.getCleanupTaskIds().contains(firstTask.getTaskId())); Assert.assertEquals(DeployState.CANCELED, deployManager.getDeployResult(requestId, secondDeployId).get().getDeployState()); }
@Test public void itSetsRequestStateToDeletedIfTaskCleanupFails() { initRequest(); SingularityRequest request = requestResource.getRequest(requestId, singularityUser).getRequest(); requestResource.postRequest(request.toBuilder().setInstances(Optional.of(2)).build(), singularityUser); initFirstDeploy(); SingularityTask firstTask = launchTask(request, firstDeploy, 1, TaskState.TASK_RUNNING); launchTask(request, firstDeploy, 2, TaskState.TASK_RUNNING); Assert.assertEquals(requestId, requestManager.getActiveRequests().iterator().next().getRequest().getId()); Assert.assertEquals(2, taskManager.getActiveTaskIds().size()); requestManager.startDeletingRequest(request, Optional.absent(), Optional.absent(), Optional.absent(), Optional.absent()); Assert.assertEquals(requestId, requestManager.getCleanupRequests().get(0).getRequestId()); Assert.assertEquals(RequestState.DELETING, requestManager.getRequest(requestId).get().getState()); statusUpdate(firstTask, TaskState.TASK_FAILED); Assert.assertEquals(1, taskManager.getActiveTaskIds().size()); cleaner.drainCleanupQueue(); Assert.assertEquals(0, taskManager.getCleanupTaskIds().size()); killKilledTasks(); cleaner.drainCleanupQueue(); Assert.assertFalse(requestManager.getRequest(requestId).isPresent()); }
@Test public void testUsesNewRequestDataFromPendingDeploy() { initRequest(); initFirstDeploy(); saveAndSchedule(request.toBuilder().setInstances(Optional.of(2))); scheduler.drainPendingQueue(); Assert.assertEquals(2, taskManager.getPendingTaskIds().size()); Assert.assertEquals(2, requestManager.getRequest(requestId).get().getRequest().getInstancesSafe()); SingularityRequest request = requestResource.getRequest(requestId, singularityUser).getRequest(); SingularityRequest newRequest = request.toBuilder().setInstances(Optional.of(1)).build(); String deployId = "test_new_request_data"; SingularityDeploy deploy = new SingularityDeployBuilder(request.getId(), deployId).setCommand(Optional.of("sleep 100")).build(); deployResource.deploy(new SingularityDeployRequest(deploy, Optional.absent(), Optional.absent(), Optional.of(newRequest)), singularityUser); deployChecker.checkDeploys(); scheduler.drainPendingQueue(); List<SingularityPendingTaskId> pendingTaskIdsForNewDeploy = new ArrayList<>(); for (SingularityPendingTaskId pendingTaskId : taskManager.getPendingTaskIds()) { if (pendingTaskId.getDeployId().equals(deployId)) { pendingTaskIdsForNewDeploy.add(pendingTaskId); } } Assert.assertEquals(1, pendingTaskIdsForNewDeploy.size()); Assert.assertEquals(2, requestManager.getRequest(requestId).get().getRequest().getInstancesSafe()); resourceOffers(); for (SingularityTaskId taskId : taskManager.getActiveTaskIdsForDeploy(requestId, deployId)) { statusUpdate(taskManager.getTask(taskId).get(), TaskState.TASK_RUNNING); } deployChecker.checkDeploys(); Assert.assertEquals(1, requestManager.getRequest(requestId).get().getRequest().getInstancesSafe()); }
@Test public void testDeployAllInstancesAtOnce() { initRequest(); SingularityRequest request = requestResource.getRequest(requestId, singularityUser).getRequest(); requestResource.postRequest(request.toBuilder().setInstances(Optional.of(2)).build(), singularityUser); initFirstDeploy(); launchTask(request, firstDeploy, 1, TaskState.TASK_RUNNING); launchTask(request, firstDeploy, 2, TaskState.TASK_RUNNING); deploy(secondDeployId); deployChecker.checkDeploys(); scheduler.drainPendingQueue(); Assert.assertEquals(2, taskManager.getActiveTaskIds().size()); Assert.assertEquals(2, taskManager.getPendingTaskIds().size()); resourceOffers(); Assert.assertEquals(2, taskManager.getActiveTaskIdsForDeploy(requestId, secondDeployId).size()); for (SingularityTaskId taskId : taskManager.getActiveTaskIdsForDeploy(requestId, secondDeployId)) { statusUpdate(taskManager.getTask(taskId).get(), TaskState.TASK_RUNNING); } deployChecker.checkDeploys(); Assert.assertEquals(2, taskManager.getCleanupTaskIds().size()); Assert.assertEquals(DeployState.SUCCEEDED, deployManager.getDeployResult(requestId, secondDeployId).get().getDeployState()); Assert.assertEquals(2, taskManager.getActiveTaskIdsForDeploy(requestId, secondDeployId).size()); }
@Test public void testRunNowOnDemandJobMayRetryOnFailure() { initRequestWithType(RequestType.ON_DEMAND, false); SingularityRequest request = requestResource.getRequest(requestId, singularityUser).getRequest(); SingularityRequest newRequest = request.toBuilder().setNumRetriesOnFailure(Optional.of(2)).build(); requestResource.postRequest(newRequest, singularityUser); initFirstDeploy(); requestResource.scheduleImmediately(singularityUser, requestId, new SingularityRunNowRequestBuilder().build()); resourceOffers(); SingularityTask task = taskManager.getActiveTasks().get(0); statusUpdate(task, TaskState.TASK_FAILED); scheduler.drainPendingQueue(); SingularityDeployStatistics deployStatistics = deployManager.getDeployStatistics(task.getTaskId().getRequestId(), task.getTaskId().getDeployId()).get(); Assert.assertEquals(MesosTaskState.TASK_FAILED, deployStatistics.getLastTaskState().get().toTaskState().get()); Assert.assertEquals(PendingType.RETRY, taskManager.getPendingTaskIds().get(0).getPendingType()); Assert.assertEquals(1, deployStatistics.getNumFailures()); Assert.assertEquals(1, deployStatistics.getNumSequentialRetries()); }
@Test public void testRunNowScheduledJobDoesNotRetry() { initScheduledRequest(); SingularityRequest request = requestResource.getRequest(requestId, singularityUser).getRequest(); SingularityRequest newRequest = request.toBuilder().setNumRetriesOnFailure(Optional.of(2)).build(); requestResource.postRequest(newRequest, singularityUser); initFirstDeploy(); requestResource.scheduleImmediately(singularityUser, requestId, new SingularityRunNowRequestBuilder().build()); resourceOffers(); SingularityTask task = taskManager.getActiveTasks().get(0); statusUpdate(task, TaskState.TASK_FAILED); scheduler.drainPendingQueue(); SingularityDeployStatistics deployStatistics = deployManager.getDeployStatistics(task.getTaskId().getRequestId(), task.getTaskId().getDeployId()).get(); Assert.assertEquals(MesosTaskState.TASK_FAILED, deployStatistics.getLastTaskState().get().toTaskState().get()); Assert.assertEquals(PendingType.TASK_DONE, taskManager.getPendingTaskIds().get(0).getPendingType()); Assert.assertEquals(1, deployStatistics.getNumFailures()); Assert.assertEquals(0, deployStatistics.getNumSequentialRetries()); Assert.assertEquals(Optional.<Long>absent(), deployStatistics.getAverageRuntimeMillis()); }
initRequest(); SingularityRequest request = requestResource.getRequest(requestId, singularityUser).getRequest();
initRequest(); SingularityRequest request = requestResource.getRequest(requestId, singularityUser).getRequest();
@Test public void testScaleDownDuringDeploy() { initRequest(); SingularityRequest request = requestResource.getRequest(requestId, singularityUser).getRequest(); requestResource.postRequest(request.toBuilder().setInstances(Optional.of(2)).build(), singularityUser); initFirstDeploy(); launchTask(request, firstDeploy, 1, TaskState.TASK_RUNNING); launchTask(request, firstDeploy, 2, TaskState.TASK_RUNNING); deploy(secondDeployId); deployChecker.checkDeploys(); scheduler.drainPendingQueue(); resourceOffers(); Assert.assertEquals(2, taskManager.getActiveTaskIdsForDeploy(requestId, secondDeployId).size()); for (SingularityTaskId taskId : taskManager.getActiveTaskIdsForDeploy(requestId, secondDeployId)) { statusUpdate(taskManager.getTask(taskId).get(), TaskState.TASK_RUNNING); } requestResource.postRequest(request.toBuilder().setInstances(Optional.of(1)).build(), singularityUser); scheduler.drainPendingQueue(); Assert.assertEquals(1, taskManager.getCleanupTaskIds().size()); deployChecker.checkDeploys(); Assert.assertEquals(2, taskManager.getCleanupTaskIds().size()); // Extra task from the new deploy should get cleaned up as well scheduler.drainPendingQueue(); Assert.assertEquals(3, taskManager.getCleanupTaskIds().size()); }
@Test public void itDoesntCountCleaningTasks() { initRequest(); initFirstDeploy(); SingularityRequest request = requestResource.getRequest(requestId, singularityUser).getRequest(); saveAndSchedule(request.toBuilder().setInstances(Optional.of(3))); resourceOffers(); Assert.assertEquals(3, taskManager.getActiveTaskIds().size()); Assert.assertEquals(0, stateManager.getState(true, false).getOverProvisionedRequests()); Assert.assertEquals(0, stateManager.getState(true, false).getUnderProvisionedRequests()); SingularityTask task = taskManager.getActiveTasks().get(0); statusUpdate(task, TaskState.TASK_KILLED); scheduler.drainPendingQueue(); taskManager.createTaskCleanup(new SingularityTaskCleanup(Optional.absent(), TaskCleanupType.BOUNCING, 1L, task.getTaskId(), Optional.absent(), Optional.absent(), Optional.absent())); Assert.assertEquals(2, taskManager.getActiveTaskIds().size()); Assert.assertEquals(0, stateManager.getState(true, false).getOverProvisionedRequests()); Assert.assertEquals(1, stateManager.getState(true, false).getUnderProvisionedRequests()); launchTask(request, firstDeploy, 4, TaskState.TASK_RUNNING); launchTask(request, firstDeploy, 5, TaskState.TASK_RUNNING); Assert.assertEquals(4, taskManager.getActiveTaskIds().size()); Assert.assertEquals(0, stateManager.getState(true, false).getUnderProvisionedRequests()); Assert.assertEquals(1, stateManager.getState(true, false).getOverProvisionedRequests()); }
SingularityRequest request = requestResource.getRequest(requestId, singularityUser).getRequest(); requestResource.postRequest(request.toBuilder().setInstances(Optional.of(2)).build(), singularityUser); initFirstDeploy();