@Override public boolean apply(SingularityRequestWithState input) { for (RequestState state : states) { if (input.getState() == state) { return true; } } return false; }
public static String getRequestState(Optional<SingularityRequestWithState> maybeRequestWithState) { if (maybeRequestWithState.isPresent()) { return maybeRequestWithState.get().getState().name(); } return "MISSING"; }
public static SingularityPendingRequestParent minimalFromRequestWithState(SingularityRequestWithState requestWithState, SingularityPendingRequest pendingRequest) { return new SingularityPendingRequestParent(requestWithState.getRequest(), requestWithState.getState(), Optional.absent(), Optional.absent(), Optional.absent(), Optional.absent(), pendingRequest, Optional.absent(), Optional.absent(), Optional.absent(), Optional.absent(), Optional.absent(), Optional.absent(), Optional.absent()); }
public static boolean isActive(Optional<SingularityRequestWithState> maybeRequestWithState) { return maybeRequestWithState.isPresent() && maybeRequestWithState.get().getState().isRunnable(); }
private void submitScaleRequest(SingularityRequestWithState oldRequestWithState, Integer newRequestedInstances) { SingularityRequest oldRequest = oldRequestWithState.getRequest(); SingularityRequest newRequest = oldRequest.toBuilder().setInstances(Optional.of((newRequestedInstances))).build(); Optional<SingularityRequestHistory.RequestHistoryType> historyType = Optional.of(SingularityRequestHistory.RequestHistoryType.SCALED); Optional<String> message = Optional.of(String.format("Auto scale number of instances to spread to all %d available slaves", newRequestedInstances)); requestHelper.updateRequest(newRequest, Optional.of(oldRequest), oldRequestWithState.getState(), historyType, Optional.<String>absent(), oldRequest.getSkipHealthchecks(), message, Optional.<SingularityBounceRequest>absent()); }
@Override protected void handleExpiringObject(SingularityExpiringPause expiringObject, SingularityRequestWithState requestWithState, String message) { if (requestWithState.getState() != RequestState.PAUSED) { LOG.warn("Discarding {} because request {} is in state {}", expiringObject, requestWithState.getRequest().getId(), requestWithState.getState()); return; } LOG.info("Unpausing request {} because of {}", requestWithState.getRequest().getId(), expiringObject); requestHelper.unpause(requestWithState.getRequest(), expiringObject.getUser(), Optional.of(message), Optional.absent()); }
public SingularityCreateResult update(SingularityRequest request, long timestamp, Optional<String> user, Optional<String> message) { return save(request, getRequest(request.getId()).get().getState(), RequestHistoryType.UPDATED, timestamp, user, message); }
@Override protected void handleExpiringObject(SingularityExpiringSkipHealthchecks expiringObject, SingularityRequestWithState requestWithState, String message) { final SingularityRequest oldRequest = requestWithState.getRequest(); final SingularityRequest newRequest = oldRequest.toBuilder().setSkipHealthchecks(expiringObject.getRevertToSkipHealthchecks()).build(); try { requestHelper.updateRequest(newRequest, Optional.of(oldRequest), requestWithState.getState(), Optional.<RequestHistoryType>absent(), expiringObject.getUser(), Optional.<Boolean>absent(), Optional.of(message), Optional.<SingularityBounceRequest>absent()); } catch (WebApplicationException wae) { LOG.error("While trying to apply {} for {}", expiringObject, expiringObject.getRequestId(), wae); } } }
private void updatePossiblyUnderProvisionedAndOverProvisionedIds(SingularityRequestWithState requestWithState, Map<String, Long> numInstances, List<String> overProvisionedRequestIds, Set<String> possiblyUnderProvisionedRequestIds) { if (requestWithState.getState().isRunnable() && requestWithState.getRequest().isAlwaysRunning()) { SingularityRequest request = requestWithState.getRequest(); final int expectedInstances = request.getInstancesSafe(); final Long numActualInstances = numInstances.get(request.getId()); if (numActualInstances == null || numActualInstances < expectedInstances) { possiblyUnderProvisionedRequestIds.add(request.getId()); } else if (numActualInstances > expectedInstances) { overProvisionedRequestIds.add(request.getId()); } } }
public SingularityRequestParent unpause(String requestId, Optional<SingularityUnpauseRequest> unpauseRequest, SingularityUser user) { SingularityRequestWithState requestWithState = fetchRequestWithState(requestId, user); authorizationHelper.checkForAuthorization(requestWithState.getRequest(), user, SingularityAuthorizationScope.WRITE); checkConflict(requestWithState.getState() == RequestState.PAUSED, "Request %s is not in PAUSED state, it is in %s", requestId, requestWithState.getState()); Optional<String> message = Optional.absent(); Optional<Boolean> skipHealthchecks = Optional.absent(); if (unpauseRequest.isPresent()) { message = unpauseRequest.get().getMessage(); skipHealthchecks = unpauseRequest.get().getSkipHealthchecks(); } requestManager.deleteExpiringObject(SingularityExpiringPause.class, requestId); final long now = requestHelper.unpause(requestWithState.getRequest(), user.getEmail(), message, skipHealthchecks); return fillEntireRequest(new SingularityRequestWithState(requestWithState.getRequest(), RequestState.ACTIVE, now)); }
public SingularityRequestParent exitCooldown(String requestId, Optional<SingularityExitCooldownRequest> exitCooldownRequest, SingularityUser user) { final SingularityRequestWithState requestWithState = fetchRequestWithState(requestId, user); authorizationHelper.checkForAuthorization(requestWithState.getRequest(), user, SingularityAuthorizationScope.WRITE); checkConflict(requestWithState.getState() == RequestState.SYSTEM_COOLDOWN, "Request %s is not in SYSTEM_COOLDOWN state, it is in %s", requestId, requestWithState.getState()); final Optional<String> maybeDeployId = deployManager.getInUseDeployId(requestId); final long now = System.currentTimeMillis(); Optional<String> message = Optional.absent(); Optional<Boolean> skipHealthchecks = Optional.absent(); if (exitCooldownRequest.isPresent()) { message = exitCooldownRequest.get().getMessage(); skipHealthchecks = exitCooldownRequest.get().getSkipHealthchecks(); } requestManager.exitCooldown(requestWithState.getRequest(), now, Optional.of(user.getId()), message); if (maybeDeployId.isPresent() && !requestWithState.getRequest().isOneOff()) { requestManager.addToPendingQueue(new SingularityPendingRequest(requestId, maybeDeployId.get(), now, Optional.of(user.getId()), PendingType.IMMEDIATE, skipHealthchecks, message)); } return fillEntireRequest(requestWithState); }
protected SingularityRequestParent fillEntireRequest(SingularityRequestWithState requestWithState, Optional<SingularityRequest> newRequestData) { final String requestId = requestWithState.getRequest().getId(); final Optional<SingularityRequestDeployState> requestDeployState = deployManager.getRequestDeployState(requestId); Optional<SingularityDeploy> activeDeploy = Optional.absent(); Optional<SingularityDeploy> pendingDeploy = Optional.absent(); if (requestDeployState.isPresent()) { activeDeploy = fillDeploy(requestDeployState.get().getActiveDeploy()); pendingDeploy = fillDeploy(requestDeployState.get().getPendingDeploy()); } Optional<SingularityPendingDeploy> pendingDeployState = deployManager.getPendingDeploy(requestId); return new SingularityRequestParent(newRequestData.or(requestWithState.getRequest()), requestWithState.getState(), requestDeployState, activeDeploy, pendingDeploy, pendingDeployState, requestManager.getExpiringBounce(requestId), requestManager.getExpiringPause(requestId), requestManager.getExpiringScale(requestId), requestManager.getExpiringSkipHealthchecks(requestId), requestHelper.getTaskIdsByStatusForRequest(requestId), requestHistoryHelper.getLastHistory(requestId), requestHelper.getMostRecentTask(requestWithState.getRequest())); }
@Test public void testCooldownOnlyWhenTasksRapidlyFail() { initRequest(); initFirstDeploy(); configuration.setCooldownAfterFailures(1); SingularityTask firstTask = startTask(firstDeploy); statusUpdate(firstTask, TaskState.TASK_FAILED, Optional.of(System.currentTimeMillis() - TimeUnit.HOURS.toMillis(5))); Assert.assertTrue(requestManager.getRequest(requestId).get().getState() == RequestState.ACTIVE); SingularityTask secondTask = startTask(firstDeploy); statusUpdate(secondTask, TaskState.TASK_FAILED); Assert.assertTrue(requestManager.getRequest(requestId).get().getState() == RequestState.SYSTEM_COOLDOWN); }
@Test public void itCorrectlyUpdatesRequestDeletingStateHistory() { initRequest(); Assert.assertEquals(RequestState.ACTIVE, requestManager.getRequest(requestId).get().getState()); Assert.assertEquals(1, requestManager.getRequestHistory(requestId).size()); requestManager.startDeletingRequest(request, Optional.absent(), Optional.<String>absent(), Optional.<String>absent(), Optional.of("the cake is a lie")); Assert.assertEquals(RequestState.DELETING, requestManager.getRequest(requestId).get().getState()); Assert.assertEquals(2, requestManager.getRequestHistory(requestId).size()); cleaner.drainCleanupQueue(); Assert.assertEquals(3, requestManager.getRequestHistory(requestId).size()); List<RequestHistoryType> historyTypes = new ArrayList<>(); for (SingularityRequestHistory request : requestManager.getRequestHistory(requestId)) { historyTypes.add(request.getEventType()); } Assert.assertTrue(historyTypes.contains(RequestHistoryType.CREATED)); Assert.assertTrue(historyTypes.contains(RequestHistoryType.DELETING)); Assert.assertTrue(historyTypes.contains(RequestHistoryType.DELETED)); }
@Test public void testCooldownAfterSequentialFailures() { initRequest(); initFirstDeploy(); Assert.assertTrue(requestManager.getRequest(requestId).get().getState() == RequestState.ACTIVE); configuration.setCooldownAfterFailures(2); SingularityTask firstTask = startTask(firstDeploy); SingularityTask secondTask = startTask(firstDeploy); statusUpdate(firstTask, TaskState.TASK_FAILED); Assert.assertTrue(requestManager.getRequest(requestId).get().getState() == RequestState.ACTIVE); statusUpdate(secondTask, TaskState.TASK_FAILED); Assert.assertTrue(requestManager.getRequest(requestId).get().getState() == RequestState.SYSTEM_COOLDOWN); cooldownChecker.checkCooldowns(); Assert.assertTrue(requestManager.getRequest(requestId).get().getState() == RequestState.SYSTEM_COOLDOWN); SingularityTask thirdTask = startTask(firstDeploy); statusUpdate(thirdTask, TaskState.TASK_FINISHED); Assert.assertTrue(requestManager.getRequest(requestId).get().getState() == RequestState.ACTIVE); }
@Test public void testFinishedRequestCanBeDeployed() { initScheduledRequest(); initFirstDeploy(); schedule = "*/1 * * * * ? 1995"; // cause it to be pending requestResource.postRequest(request.toBuilder().setQuartzSchedule(Optional.of(schedule)).build(), singularityUser); scheduler.drainPendingQueue(); Assert.assertTrue(requestResource.getActiveRequests(singularityUser, false, false, false, 10, Collections.emptyList()).isEmpty()); Assert.assertTrue(requestManager.getRequest(requestId).get().getState() == RequestState.FINISHED); SingularityDeployBuilder db = new SingularityDeployBuilder(requestId, secondDeployId); initDeploy(db, System.currentTimeMillis()); deployChecker.checkDeploys(); Assert.assertEquals(RequestState.ACTIVE, requestManager.getRequest(requestId).get().getState()); Assert.assertEquals(1, requestManager.getPendingRequests().size()); }
@Test public void testTaskDestroy() { initRequest(); initFirstDeploy(); SingularityTask firstTask = startTask(firstDeploy, 1); SingularityTask secondTask = startTask(firstDeploy, 2); SingularityTask thirdTask = startTask(firstDeploy, 3); taskResource.killTask(secondTask.getTaskId().getId(), Optional.of( new SingularityKillTaskRequest(Optional.of(true), Optional.of("kill -9 bb"), Optional.absent(), Optional.absent(), Optional.absent())), singularityUser); cleaner.drainCleanupQueue(); killKilledTasks(); Assert.assertEquals(2, taskManager.getNumActiveTasks()); System.out.println(requestManager.getCleanupRequests()); Assert.assertEquals(0, requestManager.getCleanupRequests().size()); Assert.assertEquals(RequestState.ACTIVE, requestManager.getRequest(requestId).get().getState()); }
@Test public void testSchedulerHandlesFinishedTasks() { initScheduledRequest(); initFirstDeploy(); schedule = "*/1 * * * * ? 1995"; // cause it to be pending requestResource.postRequest(request.toBuilder().setQuartzSchedule(Optional.of(schedule)).build(), singularityUser); scheduler.drainPendingQueue(); Assert.assertTrue(requestResource.getActiveRequests(singularityUser, false, false, false, 10, Collections.emptyList()).isEmpty()); Assert.assertTrue(requestManager.getRequest(requestId).get().getState() == RequestState.FINISHED); Assert.assertTrue(taskManager.getPendingTaskIds().isEmpty()); schedule = "*/1 * * * * ?"; requestResource.postRequest(request.toBuilder().setQuartzSchedule(Optional.of(schedule)).build(), singularityUser); scheduler.drainPendingQueue(); Assert.assertTrue(!requestResource.getActiveRequests(singularityUser, false, false, false, 10, Collections.emptyList()).isEmpty()); Assert.assertTrue(requestManager.getRequest(requestId).get().getState() == RequestState.ACTIVE); Assert.assertTrue(!taskManager.getPendingTaskIds().isEmpty()); }
@Test public void itSetsRequestStateToDeletedAfterAllTasksAreCleanedUp() { initRequest(); SingularityRequest request = requestResource.getRequest(requestId, singularityUser).getRequest(); requestResource.postRequest(request.toBuilder().setInstances(Optional.of(2)).build(), singularityUser); initFirstDeploy(); launchTask(request, firstDeploy, 1, TaskState.TASK_RUNNING); launchTask(request, firstDeploy, 2, TaskState.TASK_RUNNING); Assert.assertEquals(requestId, requestManager.getActiveRequests().iterator().next().getRequest().getId()); Assert.assertEquals(2, taskManager.getActiveTaskIds().size()); requestManager.startDeletingRequest(request, Optional.absent(), Optional.absent(), Optional.absent(), Optional.absent()); Assert.assertEquals(requestId, requestManager.getCleanupRequests().get(0).getRequestId()); Assert.assertEquals(RequestState.DELETING, requestManager.getRequest(requestId).get().getState()); cleaner.drainCleanupQueue(); Assert.assertEquals(0, taskManager.getCleanupTaskIds().size()); killKilledTasks(); cleaner.drainCleanupQueue(); Assert.assertFalse(requestManager.getRequest(requestId).isPresent()); }
public SingularityPendingRequestParent scheduleImmediately(SingularityUser user, String requestId, SingularityRunNowRequest runNowRequest, boolean minimalReturn) { final Optional<SingularityRunNowRequest> maybeRunNowRequest = Optional.fromNullable(runNowRequest); SingularityRequestWithState requestWithState = fetchRequestWithState(requestId, user); authorizationHelper.checkForAuthorization(requestWithState.getRequest(), user, SingularityAuthorizationScope.WRITE); checkConflict(requestWithState.getState() != RequestState.PAUSED, "Request %s is paused. Unable to run now (it must be manually unpaused first)", requestWithState.getRequest().getId()); final SingularityPendingRequest pendingRequest = validator.checkRunNowRequest( getAndCheckDeployId(requestId), user.getEmail(), requestWithState.getRequest(), maybeRunNowRequest, taskManager.getActiveTaskIdsForRequest(requestId), taskManager.getPendingTaskIdsForRequest(requestId)); SingularityCreateResult result = requestManager.addToPendingQueue(pendingRequest); checkConflict(result != SingularityCreateResult.EXISTED, "%s is already pending, please try again soon", requestId); if (minimalReturn) { return SingularityPendingRequestParent.minimalFromRequestWithState(requestWithState, pendingRequest); } else { return SingularityPendingRequestParent.fromSingularityRequestParent(fillEntireRequest(requestWithState), pendingRequest); } }