@Test public void testDeployFailsForInvalidRequestState() { initRequest(); SingularityRequest request = requestResource.getRequest(requestId, singularityUser).getRequest(); initFirstDeploy(); deploy(secondDeployId, Optional.<Boolean>absent(), Optional.of(1), Optional.of(false), false); requestManager.pause(request, System.currentTimeMillis(), Optional.<String>absent(), Optional.<String>absent()); deployChecker.checkDeploys(); Assert.assertEquals(DeployState.FAILED, deployManager.getDeployResult(requestId, secondDeployId).get().getDeployState()); }
public static SingularityPendingRequestParent fromSingularityRequestParent(SingularityRequestParent singularityRequestParent, SingularityPendingRequest pendingRequest) { return new SingularityPendingRequestParent(singularityRequestParent.getRequest(), singularityRequestParent.getState(), singularityRequestParent.getRequestDeployState(), singularityRequestParent.getActiveDeploy(), singularityRequestParent.getPendingDeploy(), singularityRequestParent.getPendingDeployState(), pendingRequest, singularityRequestParent.getExpiringBounce(), singularityRequestParent.getExpiringPause(), singularityRequestParent.getExpiringScale(), singularityRequestParent.getExpiringSkipHealthchecks(), singularityRequestParent.getTaskIds(), singularityRequestParent.getLastHistory(), singularityRequestParent.getMostRecentTask()); }
CompletableFuture<Optional<SingularityExpiringScale>> maybeExpiringScale = CompletableFuture.supplyAsync(() -> requestManager.getExpiringScale(requestWithState.getRequest().getId())).exceptionally((throwable) -> Optional.absent()); CompletableFuture<Optional<SingularityExpiringSkipHealthchecks>> maybeExpiringSkipHealthchecks = CompletableFuture.supplyAsync(() -> requestManager.getExpiringSkipHealthchecks(requestWithState.getRequest().getId())).exceptionally((throwable) -> Optional.absent()); return new SingularityRequestParent( requestWithState.getRequest(), requestWithState.getState(), Optional.fromNullable(deployStates.get(requestWithState.getRequest().getId())), mostRecentTasks.computeIfAbsent(requestWithState.getRequest().getId(), (id) -> getMostRecentTask(requestWithState.getRequest()))); } else { return new SingularityRequestParent( requestWithState.getRequest(), requestWithState.getState(), Optional.fromNullable(deployStates.get(requestWithState.getRequest().getId())), Optional.absent(), Optional.absent(), Optional.absent(), Optional.absent(), Optional.absent(), Optional.absent(), Optional.absent(), Optional.absent(), Optional.absent(), Optional.absent());
private SingularityRequestParent getAndLogRequestAndDeployStatus(SingularityRequestParent singularityRequestParent) { String activeDeployId = singularityRequestParent.getActiveDeploy().isPresent() ? singularityRequestParent.getActiveDeploy().get().getId() : "No Active Deploy"; String pendingDeployId = singularityRequestParent.getPendingDeploy().isPresent() ? singularityRequestParent.getPendingDeploy().get().getId() : "No Pending deploy"; LOG.info("Deploy status: Singularity request {} -> pending deploy: '{}', active deploy: '{}'", singularityRequestParent.getRequest().getId(), pendingDeployId, activeDeployId); return singularityRequestParent; }
assertEquals(request, requestParent.get().getRequest());
protected SingularityRequestParent fillEntireRequest(SingularityRequestWithState requestWithState, Optional<SingularityRequest> newRequestData) { final String requestId = requestWithState.getRequest().getId(); final Optional<SingularityRequestDeployState> requestDeployState = deployManager.getRequestDeployState(requestId); Optional<SingularityDeploy> activeDeploy = Optional.absent(); Optional<SingularityDeploy> pendingDeploy = Optional.absent(); if (requestDeployState.isPresent()) { activeDeploy = fillDeploy(requestDeployState.get().getActiveDeploy()); pendingDeploy = fillDeploy(requestDeployState.get().getPendingDeploy()); } Optional<SingularityPendingDeploy> pendingDeployState = deployManager.getPendingDeploy(requestId); return new SingularityRequestParent(newRequestData.or(requestWithState.getRequest()), requestWithState.getState(), requestDeployState, activeDeploy, pendingDeploy, pendingDeployState, requestManager.getExpiringBounce(requestId), requestManager.getExpiringPause(requestId), requestManager.getExpiringScale(requestId), requestManager.getExpiringSkipHealthchecks(requestId), requestHelper.getTaskIdsByStatusForRequest(requestId), requestHistoryHelper.getLastHistory(requestId), requestHelper.getMostRecentTask(requestWithState.getRequest())); }
@Test public void itDoesntFlagPendingRequestsForUnderOrOverProvisioning() { initRequest(); initFirstDeploy(); SingularityRequest request = requestResource.getRequest(requestId, singularityUser).getRequest(); requestManager.activate(request.toBuilder().setInstances(Optional.of(0)).build(), RequestHistoryType.UPDATED, System.currentTimeMillis(), Optional.<String> absent(), Optional.<String> absent()); requestManager.addToPendingQueue(new SingularityPendingRequest(request.getId(), firstDeployId, System.currentTimeMillis(), Optional.<String> absent(), PendingType.ONEOFF, Optional.<Boolean> absent(), Optional.<String> absent())); Assert.assertEquals(0, taskManager.getActiveTaskIds().size()); SingularityState state = stateManager.getState(true, false); Assert.assertEquals(0, state.getOverProvisionedRequests()); Assert.assertEquals(0, state.getUnderProvisionedRequests()); } }
protected SingularityRequestParent fillEntireRequest(SingularityRequestWithState requestWithState, Optional<SingularityRequest> newRequestData) { final String requestId = requestWithState.getRequest().getId(); final Optional<SingularityRequestDeployState> requestDeployState = deployManager.getRequestDeployState(requestId); Optional<SingularityDeploy> activeDeploy = Optional.absent(); Optional<SingularityDeploy> pendingDeploy = Optional.absent(); if (requestDeployState.isPresent()) { activeDeploy = fillDeploy(requestDeployState.get().getActiveDeploy()); pendingDeploy = fillDeploy(requestDeployState.get().getPendingDeploy()); } Optional<SingularityPendingDeploy> pendingDeployState = deployManager.getPendingDeploy(requestId); return new SingularityRequestParent(newRequestData.or(requestWithState.getRequest()), requestWithState.getState(), requestDeployState, activeDeploy, pendingDeploy, pendingDeployState, requestManager.getExpiringBounce(requestId), requestManager.getExpiringPause(requestId), requestManager.getExpiringScale(requestId), requestManager.getExpiringSkipHealthchecks(requestId), requestHelper.getTaskIdsByStatusForRequest(requestId), requestHistoryHelper.getLastHistory(requestId), requestHelper.getMostRecentTask(requestWithState.getRequest())); }
@Test(expected = WebApplicationException.class) public void testCannotUpdateRequestDuringPendingDeployWithNewData() { initRequest(); SingularityRequest request = requestResource.getRequest(requestId, singularityUser).getRequest(); SingularityRequest newRequest = request.toBuilder().setInstances(Optional.of(1)).build(); String deployId = "test_new_request_data"; SingularityDeploy deploy = new SingularityDeployBuilder(request.getId(), deployId).setCommand(Optional.of("sleep 100")).build(); deployResource.deploy(new SingularityDeployRequest(deploy, Optional.absent(), Optional.absent(), Optional.of(newRequest)), singularityUser); requestResource.postRequest(newRequest, singularityUser); }
CompletableFuture<Optional<SingularityExpiringScale>> maybeExpiringScale = CompletableFuture.supplyAsync(() -> requestManager.getExpiringScale(requestWithState.getRequest().getId())).exceptionally((throwable) -> Optional.absent()); CompletableFuture<Optional<SingularityExpiringSkipHealthchecks>> maybeExpiringSkipHealthchecks = CompletableFuture.supplyAsync(() -> requestManager.getExpiringSkipHealthchecks(requestWithState.getRequest().getId())).exceptionally((throwable) -> Optional.absent()); return new SingularityRequestParent( requestWithState.getRequest(), requestWithState.getState(), Optional.fromNullable(deployStates.get(requestWithState.getRequest().getId())), mostRecentTasks.computeIfAbsent(requestWithState.getRequest().getId(), (id) -> getMostRecentTask(requestWithState.getRequest()))); } else { return new SingularityRequestParent( requestWithState.getRequest(), requestWithState.getState(), Optional.fromNullable(deployStates.get(requestWithState.getRequest().getId())), Optional.absent(), Optional.absent(), Optional.absent(), Optional.absent(), Optional.absent(), Optional.absent(), Optional.absent(), Optional.absent(), Optional.absent(), Optional.absent());
resourceOffers(2); // set up slaves so scale validate will pass SingularityRequest request = requestResource.getRequest(requestId, singularityUser).getRequest();
@Test public void testDeployFails() { initRequest(); SingularityRequest request = requestResource.getRequest(requestId, singularityUser).getRequest(); initFirstDeploy(); SingularityTask firstTask = launchTask(request, firstDeploy, 1, TaskState.TASK_RUNNING); deploy(secondDeployId, Optional.<Boolean>absent(), Optional.of(1), Optional.of(false), false); deployChecker.checkDeploys(); scheduler.drainPendingQueue(); Assert.assertEquals(1, taskManager.getPendingTaskIds().size()); resourceOffers(); Assert.assertEquals(1, taskManager.getActiveTaskIdsForDeploy(requestId, secondDeployId).size()); SingularityTaskId firstNewTaskId = taskManager.getActiveTaskIdsForDeploy(requestId, secondDeployId).get(0); statusUpdate(taskManager.getTask(firstNewTaskId).get(), TaskState.TASK_FAILED); deployChecker.checkDeploys(); Assert.assertFalse(taskManager.getCleanupTaskIds().contains(firstTask.getTaskId())); Assert.assertEquals(DeployState.FAILED, deployManager.getDeployResult(requestId, secondDeployId).get().getDeployState()); }
@Test public void itSetsRequestStateToDeletedAfterAllTasksAreCleanedUp() { initRequest(); SingularityRequest request = requestResource.getRequest(requestId, singularityUser).getRequest(); requestResource.postRequest(request.toBuilder().setInstances(Optional.of(2)).build(), singularityUser); initFirstDeploy(); launchTask(request, firstDeploy, 1, TaskState.TASK_RUNNING); launchTask(request, firstDeploy, 2, TaskState.TASK_RUNNING); Assert.assertEquals(requestId, requestManager.getActiveRequests().iterator().next().getRequest().getId()); Assert.assertEquals(2, taskManager.getActiveTaskIds().size()); requestManager.startDeletingRequest(request, Optional.absent(), Optional.absent(), Optional.absent(), Optional.absent()); Assert.assertEquals(requestId, requestManager.getCleanupRequests().get(0).getRequestId()); Assert.assertEquals(RequestState.DELETING, requestManager.getRequest(requestId).get().getState()); cleaner.drainCleanupQueue(); Assert.assertEquals(0, taskManager.getCleanupTaskIds().size()); killKilledTasks(); cleaner.drainCleanupQueue(); Assert.assertFalse(requestManager.getRequest(requestId).isPresent()); }
@Test public void testCancelDeploy() { initRequest(); SingularityRequest request = requestResource.getRequest(requestId, singularityUser).getRequest(); initFirstDeploy(); SingularityTask firstTask = launchTask(request, firstDeploy, 1, TaskState.TASK_RUNNING); deploy(secondDeployId, Optional.<Boolean>absent(), Optional.of(1), Optional.of(false), false); deployChecker.checkDeploys(); scheduler.drainPendingQueue(); Assert.assertEquals(1, taskManager.getPendingTaskIds().size()); resourceOffers(); Assert.assertEquals(1, taskManager.getActiveTaskIdsForDeploy(requestId, secondDeployId).size()); SingularityTaskId firstNewTaskId = taskManager.getActiveTaskIdsForDeploy(requestId, secondDeployId).get(0); statusUpdate(taskManager.getTask(firstNewTaskId).get(), TaskState.TASK_RUNNING); deployResource.cancelDeploy(singularityUser, requestId, secondDeployId); deployChecker.checkDeploys(); Assert.assertTrue(taskManager.getCleanupTaskIds().contains(firstNewTaskId)); Assert.assertFalse(taskManager.getCleanupTaskIds().contains(firstTask.getTaskId())); Assert.assertEquals(DeployState.CANCELED, deployManager.getDeployResult(requestId, secondDeployId).get().getDeployState()); }
@Test public void itSetsRequestStateToDeletedIfTaskCleanupFails() { initRequest(); SingularityRequest request = requestResource.getRequest(requestId, singularityUser).getRequest(); requestResource.postRequest(request.toBuilder().setInstances(Optional.of(2)).build(), singularityUser); initFirstDeploy(); SingularityTask firstTask = launchTask(request, firstDeploy, 1, TaskState.TASK_RUNNING); launchTask(request, firstDeploy, 2, TaskState.TASK_RUNNING); Assert.assertEquals(requestId, requestManager.getActiveRequests().iterator().next().getRequest().getId()); Assert.assertEquals(2, taskManager.getActiveTaskIds().size()); requestManager.startDeletingRequest(request, Optional.absent(), Optional.absent(), Optional.absent(), Optional.absent()); Assert.assertEquals(requestId, requestManager.getCleanupRequests().get(0).getRequestId()); Assert.assertEquals(RequestState.DELETING, requestManager.getRequest(requestId).get().getState()); statusUpdate(firstTask, TaskState.TASK_FAILED); Assert.assertEquals(1, taskManager.getActiveTaskIds().size()); cleaner.drainCleanupQueue(); Assert.assertEquals(0, taskManager.getCleanupTaskIds().size()); killKilledTasks(); cleaner.drainCleanupQueue(); Assert.assertFalse(requestManager.getRequest(requestId).isPresent()); }
@Test public void testUsesNewRequestDataFromPendingDeploy() { initRequest(); initFirstDeploy(); saveAndSchedule(request.toBuilder().setInstances(Optional.of(2))); scheduler.drainPendingQueue(); Assert.assertEquals(2, taskManager.getPendingTaskIds().size()); Assert.assertEquals(2, requestManager.getRequest(requestId).get().getRequest().getInstancesSafe()); SingularityRequest request = requestResource.getRequest(requestId, singularityUser).getRequest(); SingularityRequest newRequest = request.toBuilder().setInstances(Optional.of(1)).build(); String deployId = "test_new_request_data"; SingularityDeploy deploy = new SingularityDeployBuilder(request.getId(), deployId).setCommand(Optional.of("sleep 100")).build(); deployResource.deploy(new SingularityDeployRequest(deploy, Optional.absent(), Optional.absent(), Optional.of(newRequest)), singularityUser); deployChecker.checkDeploys(); scheduler.drainPendingQueue(); List<SingularityPendingTaskId> pendingTaskIdsForNewDeploy = new ArrayList<>(); for (SingularityPendingTaskId pendingTaskId : taskManager.getPendingTaskIds()) { if (pendingTaskId.getDeployId().equals(deployId)) { pendingTaskIdsForNewDeploy.add(pendingTaskId); } } Assert.assertEquals(1, pendingTaskIdsForNewDeploy.size()); Assert.assertEquals(2, requestManager.getRequest(requestId).get().getRequest().getInstancesSafe()); resourceOffers(); for (SingularityTaskId taskId : taskManager.getActiveTaskIdsForDeploy(requestId, deployId)) { statusUpdate(taskManager.getTask(taskId).get(), TaskState.TASK_RUNNING); } deployChecker.checkDeploys(); Assert.assertEquals(1, requestManager.getRequest(requestId).get().getRequest().getInstancesSafe()); }
resourceOffers(2); // set up slaves so scale validate will pass SingularityRequest request = requestResource.getRequest(requestId, singularityUser).getRequest();
@Test public void testDeployAllInstancesAtOnce() { initRequest(); SingularityRequest request = requestResource.getRequest(requestId, singularityUser).getRequest(); requestResource.postRequest(request.toBuilder().setInstances(Optional.of(2)).build(), singularityUser); initFirstDeploy(); launchTask(request, firstDeploy, 1, TaskState.TASK_RUNNING); launchTask(request, firstDeploy, 2, TaskState.TASK_RUNNING); deploy(secondDeployId); deployChecker.checkDeploys(); scheduler.drainPendingQueue(); Assert.assertEquals(2, taskManager.getActiveTaskIds().size()); Assert.assertEquals(2, taskManager.getPendingTaskIds().size()); resourceOffers(); Assert.assertEquals(2, taskManager.getActiveTaskIdsForDeploy(requestId, secondDeployId).size()); for (SingularityTaskId taskId : taskManager.getActiveTaskIdsForDeploy(requestId, secondDeployId)) { statusUpdate(taskManager.getTask(taskId).get(), TaskState.TASK_RUNNING); } deployChecker.checkDeploys(); Assert.assertEquals(2, taskManager.getCleanupTaskIds().size()); Assert.assertEquals(DeployState.SUCCEEDED, deployManager.getDeployResult(requestId, secondDeployId).get().getDeployState()); Assert.assertEquals(2, taskManager.getActiveTaskIdsForDeploy(requestId, secondDeployId).size()); }
@Test public void testRunNowOnDemandJobMayRetryOnFailure() { initRequestWithType(RequestType.ON_DEMAND, false); SingularityRequest request = requestResource.getRequest(requestId, singularityUser).getRequest(); SingularityRequest newRequest = request.toBuilder().setNumRetriesOnFailure(Optional.of(2)).build(); requestResource.postRequest(newRequest, singularityUser); initFirstDeploy(); requestResource.scheduleImmediately(singularityUser, requestId, new SingularityRunNowRequestBuilder().build()); resourceOffers(); SingularityTask task = taskManager.getActiveTasks().get(0); statusUpdate(task, TaskState.TASK_FAILED); scheduler.drainPendingQueue(); SingularityDeployStatistics deployStatistics = deployManager.getDeployStatistics(task.getTaskId().getRequestId(), task.getTaskId().getDeployId()).get(); Assert.assertEquals(MesosTaskState.TASK_FAILED, deployStatistics.getLastTaskState().get().toTaskState().get()); Assert.assertEquals(PendingType.RETRY, taskManager.getPendingTaskIds().get(0).getPendingType()); Assert.assertEquals(1, deployStatistics.getNumFailures()); Assert.assertEquals(1, deployStatistics.getNumSequentialRetries()); }
@Test public void testRunNowScheduledJobDoesNotRetry() { initScheduledRequest(); SingularityRequest request = requestResource.getRequest(requestId, singularityUser).getRequest(); SingularityRequest newRequest = request.toBuilder().setNumRetriesOnFailure(Optional.of(2)).build(); requestResource.postRequest(newRequest, singularityUser); initFirstDeploy(); requestResource.scheduleImmediately(singularityUser, requestId, new SingularityRunNowRequestBuilder().build()); resourceOffers(); SingularityTask task = taskManager.getActiveTasks().get(0); statusUpdate(task, TaskState.TASK_FAILED); scheduler.drainPendingQueue(); SingularityDeployStatistics deployStatistics = deployManager.getDeployStatistics(task.getTaskId().getRequestId(), task.getTaskId().getDeployId()).get(); Assert.assertEquals(MesosTaskState.TASK_FAILED, deployStatistics.getLastTaskState().get().toTaskState().get()); Assert.assertEquals(PendingType.TASK_DONE, taskManager.getPendingTaskIds().get(0).getPendingType()); Assert.assertEquals(1, deployStatistics.getNumFailures()); Assert.assertEquals(0, deployStatistics.getNumSequentialRetries()); Assert.assertEquals(Optional.<Long>absent(), deployStatistics.getAverageRuntimeMillis()); }