@Test public void testPurgingDoesntApplyIfDatabasePresent() { initRequest(); initFirstDeploy(); requestManager.startDeletingRequest(request, Optional.absent(), user, Optional.<String> absent(), Optional.<String> absent()); requestManager.deleteHistoryParent(requestId); requestManager.activate(request, RequestHistoryType.CREATED, System.currentTimeMillis() - TimeUnit.HOURS.toMillis(3), Optional.<String> absent(), Optional.<String> absent()); configuration.setDatabaseConfiguration(new DataSourceFactory()); configuration.setDeleteStaleRequestsFromZkWhenNoDatabaseAfterHours(1); requestHistoryPersister.runActionOnPoll(); Assert.assertTrue(!requestManager.getRequestHistory(requestId).isEmpty()); }
private Optional<PendingType> handleCompletedTaskWithStatistics(Optional<SingularityTask> task, SingularityTaskId taskId, long timestamp, ExtendedTaskState state, SingularityDeployStatistics deployStatistics, SingularityCreateResult taskHistoryUpdateCreateResult, Protos.TaskStatus status) { final Optional<SingularityRequestWithState> maybeRequestWithState = requestManager.getRequest(taskId.getRequestId()); final Optional<SingularityPendingDeploy> maybePendingDeploy = deployManager.getPendingDeploy(taskId.getRequestId()); LOG.info("Request {} is entering cooldown due to task {}", request.getId(), taskId); requestState = RequestState.SYSTEM_COOLDOWN; requestManager.cooldown(request, System.currentTimeMillis()); mailer.sendRequestInCooldownMail(request); requestManager.exitCooldown(request, System.currentTimeMillis(), Optional.<String>absent(), Optional.<String>absent()); Optional.absent(), resources, Collections.emptyList(), Optional.absent(), Collections.emptyMap(), Collections.emptyList(), Optional.absent()); requestManager.addToPendingQueue(pendingRequest);
private void checkReschedule(SingularityRequest newRequest, Optional<SingularityRequest> maybeOldRequest, Optional<String> user, long timestamp, Optional<Boolean> skipHealthchecks, Optional<String> message, Optional<SingularityBounceRequest> maybeBounceRequest) { if (!maybeOldRequest.isPresent()) { return; } if (shouldReschedule(newRequest, maybeOldRequest.get())) { Optional<String> maybeDeployId = deployManager.getInUseDeployId(newRequest.getId()); if (maybeDeployId.isPresent()) { if (maybeBounceRequest.isPresent()) { Optional<String> actionId = maybeBounceRequest.get().getActionId().or(Optional.of(UUID.randomUUID().toString())); Optional<Boolean> removeFromLoadBalancer = Optional.absent(); SingularityCreateResult createResult = requestManager.createCleanupRequest( new SingularityRequestCleanup(user, maybeBounceRequest.get().getIncremental().or(true) ? RequestCleanupType.INCREMENTAL_BOUNCE : RequestCleanupType.BOUNCE, System.currentTimeMillis(), Optional.<Boolean> absent(), removeFromLoadBalancer, newRequest.getId(), Optional.of(maybeDeployId.get()), skipHealthchecks, message, actionId, maybeBounceRequest.get().getRunShellCommandBeforeKill())); if (createResult != SingularityCreateResult.EXISTED) { requestManager.bounce(newRequest, System.currentTimeMillis(), user, Optional.of("Bouncing due to bounce after scale")); final SingularityBounceRequest validatedBounceRequest = validator.checkBounceRequest(maybeBounceRequest.get()); requestManager.saveExpiringObject(new SingularityExpiringBounce(newRequest.getId(), maybeDeployId.get(), user, System.currentTimeMillis(), validatedBounceRequest, actionId.get())); } else { requestManager.addToPendingQueue(new SingularityPendingRequest(newRequest.getId(), maybeDeployId.get(), timestamp, user, PendingType.UPDATED_REQUEST, skipHealthchecks, message)); } } else { requestManager.addToPendingQueue(new SingularityPendingRequest(newRequest.getId(), maybeDeployId.get(), timestamp, user, PendingType.UPDATED_REQUEST, skipHealthchecks, message)); } } } }
public void startDeletingRequest(SingularityRequest request, Optional<Boolean> removeFromLoadBalancer, Optional<String> user, Optional<String> actionId, Optional<String> message) { final long now = System.currentTimeMillis(); // delete it no matter if the delete request already exists. createCleanupRequest(new SingularityRequestCleanup(user, RequestCleanupType.DELETING, now, Optional.of(Boolean.TRUE), removeFromLoadBalancer, request.getId(), Optional.<String> absent(), Optional.<Boolean> absent(), message, actionId, Optional.<SingularityShellCommand>absent())); markDeleting(request, System.currentTimeMillis(), user, message); LOG.info("Request {} enqueued for deletion by {} - {}", request.getId(), user, message); }
public long unpause(SingularityRequest request, Optional<String> user, Optional<String> message, Optional<Boolean> skipHealthchecks) { mailer.sendRequestUnpausedMail(request, user, message); Optional<String> maybeDeployId = deployManager.getInUseDeployId(request.getId()); final long now = System.currentTimeMillis(); requestManager.unpause(request, now, user, message); if (maybeDeployId.isPresent() && !request.isOneOff()) { requestManager.addToPendingQueue(new SingularityPendingRequest(request.getId(), maybeDeployId.get(), now, user, PendingType.UNPAUSED, skipHealthchecks, message)); } return now; }
@Test public void itSetsRequestStateToDeletedAfterAllTasksAreCleanedUp() { initRequest(); SingularityRequest request = requestResource.getRequest(requestId, singularityUser).getRequest(); requestResource.postRequest(request.toBuilder().setInstances(Optional.of(2)).build(), singularityUser); initFirstDeploy(); launchTask(request, firstDeploy, 1, TaskState.TASK_RUNNING); launchTask(request, firstDeploy, 2, TaskState.TASK_RUNNING); Assert.assertEquals(requestId, requestManager.getActiveRequests().iterator().next().getRequest().getId()); Assert.assertEquals(2, taskManager.getActiveTaskIds().size()); requestManager.startDeletingRequest(request, Optional.absent(), Optional.absent(), Optional.absent(), Optional.absent()); Assert.assertEquals(requestId, requestManager.getCleanupRequests().get(0).getRequestId()); Assert.assertEquals(RequestState.DELETING, requestManager.getRequest(requestId).get().getState()); cleaner.drainCleanupQueue(); Assert.assertEquals(0, taskManager.getCleanupTaskIds().size()); killKilledTasks(); cleaner.drainCleanupQueue(); Assert.assertFalse(requestManager.getRequest(requestId).isPresent()); }
Optional.of(new SingularityBounceRequest(Optional.of(true), Optional.absent(), Optional.of(1L), Optional.absent(), Optional.of("msg"), Optional.absent())), singularityUser); Assert.assertTrue(requestManager.cleanupRequestExists(requestId)); Assert.assertEquals("msg", requestManager.getCleanupRequests().get(0).getMessage().get()); Assert.assertTrue(requestManager.getCleanupRequests().get(0).getActionId().isPresent()); String actionId = requestManager.getCleanupRequests().get(0).getActionId().get(); Assert.assertTrue(!requestManager.cleanupRequestExists(requestId)); Assert.assertTrue(taskManager.getCleanupTaskIds().size() == 3); Assert.assertTrue(!requestManager.getExpiringBounce(requestId).isPresent()); Assert.assertTrue(requestManager.getPendingRequests().isEmpty()); Assert.assertTrue(taskManager.getPendingTaskIds().isEmpty()); Assert.assertTrue(taskManager.getActiveTaskIds().size() == 3); Assert.assertTrue(!requestManager.cleanupRequestExists(requestId)); Assert.assertTrue(taskManager.getCleanupTasks().isEmpty());
requestManager.addToPendingQueue(builder.build()); } else { LOG.warn("Could not determine pending type for deploy {}.", deployId); requestManager.addToPendingQueue(new SingularityPendingRequest(request.getId(), pendingDeploy.getDeployMarker().getDeployId(), deployResult.getTimestamp(), pendingDeploy.getDeployMarker().getUser(), pendingType, deploy.isPresent() ? deploy.get().getSkipHealthchecksOnDeploy() : Optional.absent(), pendingDeploy.getDeployMarker().getMessage())); requestManager.markBounceComplete(request.getId()); requestManager.removeExpiringBounce(request.getId()); requestManager.activate( request, RequestHistoryType.UPDATED, && maybeRequestDeployState.get().getActiveDeploy().isPresent() && !(requestWithState.getState() == RequestState.PAUSED || requestWithState.getState() == RequestState.DEPLOYING_TO_UNPAUSE)) { requestManager.addToPendingQueue(new SingularityPendingRequest( request.getId(), maybeRequestDeployState.get().getActiveDeploy().get().getDeployId(), requestManager.addToPendingQueue(new SingularityPendingRequest( request.getId(), pendingDeploy.getDeployMarker().getDeployId(), requestManager.addToPendingQueue(new SingularityPendingRequest(request.getId(), pendingDeploy.getDeployMarker().getDeployId(), deployResult.getTimestamp(), pendingDeploy.getDeployMarker().getUser(), PendingType.UPDATED_REQUEST, request.getSkipHealthchecks(), pendingDeploy.getDeployMarker().getMessage())); requestManager.activate(request, RequestHistoryType.DEPLOYED_TO_UNPAUSE, deployResult.getTimestamp(), pendingDeploy.getDeployMarker().getUser(), Optional.<String> absent());
@Test public void testPause() { initRequest(); initFirstDeploy(); SingularityTask taskOne = startTask(firstDeploy); requestResource.pause(requestId, Optional.absent(), singularityUser); cleaner.drainCleanupQueue(); Assert.assertEquals(1, taskManager.getKilledTaskIdRecords().size()); statusUpdate(taskOne, TaskState.TASK_KILLED); resourceOffers(); Assert.assertEquals(0, taskManager.getActiveTaskIds().size()); Assert.assertEquals(0, taskManager.getPendingTasks().size()); Assert.assertEquals(RequestState.PAUSED, requestManager.getRequest(requestId).get().getState()); Assert.assertEquals(requestId, requestManager.getPausedRequests(false).iterator().next().getRequest().getId()); requestResource.unpause(requestId, Optional.absent(), singularityUser); resourceOffers(); Assert.assertEquals(1, taskManager.getActiveTaskIds().size()); Assert.assertEquals(0, taskManager.getPendingTasks().size()); Assert.assertEquals(RequestState.ACTIVE, requestManager.getRequest(requestId).get().getState()); Assert.assertEquals(requestId, requestManager.getActiveRequests(false).iterator().next().getRequest().getId()); }
protected SingularityRequestParent fillEntireRequest(SingularityRequestWithState requestWithState, Optional<SingularityRequest> newRequestData) { final String requestId = requestWithState.getRequest().getId(); final Optional<SingularityRequestDeployState> requestDeployState = deployManager.getRequestDeployState(requestId); Optional<SingularityDeploy> activeDeploy = Optional.absent(); Optional<SingularityDeploy> pendingDeploy = Optional.absent(); if (requestDeployState.isPresent()) { activeDeploy = fillDeploy(requestDeployState.get().getActiveDeploy()); pendingDeploy = fillDeploy(requestDeployState.get().getPendingDeploy()); } Optional<SingularityPendingDeploy> pendingDeployState = deployManager.getPendingDeploy(requestId); return new SingularityRequestParent(newRequestData.or(requestWithState.getRequest()), requestWithState.getState(), requestDeployState, activeDeploy, pendingDeploy, pendingDeployState, requestManager.getExpiringBounce(requestId), requestManager.getExpiringPause(requestId), requestManager.getExpiringScale(requestId), requestManager.getExpiringSkipHealthchecks(requestId), requestHelper.getTaskIdsByStatusForRequest(requestId), requestHistoryHelper.getLastHistory(requestId), requestHelper.getMostRecentTask(requestWithState.getRequest())); }
@Test public void testImmediateRequestsAreConsistentlyDeleted() { long now = System.currentTimeMillis(); initRequestWithType(RequestType.SCHEDULED, false); startFirstDeploy(); SingularityPendingRequest pendingDeployRequest = new SingularityPendingRequest(requestId, firstDeploy.getId(), now, Optional.absent(), PendingType.NEW_DEPLOY, firstDeploy.getSkipHealthchecksOnDeploy(), Optional.absent()); SingularityPendingRequest pendingRunNowRequest = new SingularityPendingRequest(requestId, firstDeploy.getId(), now + 200, Optional.absent(), PendingType.IMMEDIATE, firstDeploy.getSkipHealthchecksOnDeploy(), Optional.absent()); requestManager.addToPendingQueue(pendingDeployRequest); requestManager.addToPendingQueue(pendingRunNowRequest); // Pending queue has two requests: NEW_DEPLOY & IMMEDIATE Assert.assertEquals(2, requestManager.getPendingRequests().size()); finishNewTaskChecks(); requestManager.deletePendingRequest(pendingDeployRequest); // Just the immediate run Assert.assertEquals(1, requestManager.getPendingRequests().size()); requestManager.deletePendingRequest(pendingRunNowRequest); // Immediate run was successfully deleted Assert.assertEquals(0, requestManager.getPendingRequests().size()); }
private void bounce(SingularityRequestCleanup requestCleanup, final List<SingularityTaskId> activeTaskIds) { final long start = System.currentTimeMillis(); final List<SingularityTaskId> matchingTaskIds = new ArrayList<>(); for (SingularityTaskId activeTaskId : activeTaskIds) { if (activeTaskId.getRequestId().equals(requestCleanup.getRequestId()) && activeTaskId.getDeployId().equals(requestCleanup.getDeployId().get())) { matchingTaskIds.add(activeTaskId); } } for (SingularityTaskId matchingTaskId : matchingTaskIds) { LOG.debug("Adding task {} to cleanup (bounce)", matchingTaskId.getId()); Optional<SingularityTaskShellCommandRequestId> runBeforeKillId = Optional.absent(); if (requestCleanup.getRunShellCommandBeforeKill().isPresent()) { SingularityTaskShellCommandRequest shellRequest = new SingularityTaskShellCommandRequest(matchingTaskId, requestCleanup.getUser(), System.currentTimeMillis(), requestCleanup.getRunShellCommandBeforeKill().get()); taskManager.saveTaskShellCommandRequestToQueue(shellRequest); runBeforeKillId = Optional.of(shellRequest.getId()); } taskManager.createTaskCleanup(new SingularityTaskCleanup(requestCleanup.getUser(), requestCleanup.getCleanupType().getTaskCleanupType().get(), start, matchingTaskId, requestCleanup.getMessage(), requestCleanup.getActionId(), runBeforeKillId)); } if (matchingTaskIds.isEmpty() && requestCleanup.getDeployId().isPresent()) { Optional<SingularityExpiringBounce> expiringBounce = requestManager.getExpiringBounce(requestCleanup.getRequestId()); if (expiringBounce.isPresent() && expiringBounce.get().getDeployId().equalsIgnoreCase(requestCleanup.getDeployId().get())) { LOG.info("No running tasks for request {}. Marking bounce {} complete and starting new tasks", expiringBounce.get().getRequestId(), expiringBounce.get()); requestManager.removeExpiringBounce(requestCleanup.getRequestId()); } requestManager.markBounceComplete(requestCleanup.getRequestId()); } requestManager.addToPendingQueue(new SingularityPendingRequest(requestCleanup.getRequestId(), requestCleanup.getDeployId().get(), requestCleanup.getTimestamp(), requestCleanup.getUser(), PendingType.BOUNCE, Optional.absent(), Optional.absent(), requestCleanup.getSkipHealthchecks(), requestCleanup.getMessage(), requestCleanup.getActionId())); LOG.info("Added {} tasks for request {} to cleanup bounce queue in {}", matchingTaskIds.size(), requestCleanup.getRequestId(), JavaUtils.duration(start)); }
curator.create().creatingParentsIfNeeded().forPath(String.format("%s%s", "/requests/pending/oneOffRequest-oneOffDeploy", now), objectMapper.writeValueAsBytes(oneOffRequest)); Assert.assertEquals("3 existing requests under old paths", 3, requestManager.getPendingRequests().size()); System.out.println(curator.getChildren().forPath("/requests/pending")); Assert.assertEquals("3 existing requests under new paths", 3, requestManager.getPendingRequests().size()); System.out.println(curator.getChildren().forPath("/requests/pending")); requestManager.deletePendingRequest(newDeploy); Assertions.assertThat(requestManager.getPendingRequests()) .as("Non-renamed, non-timestamped nodes can be deleted") .hasSize(2) .contains(oneOffRequest.toString(), immediateRequest.toString()); requestManager.deletePendingRequest(oneOffRequest); Assertions.assertThat(requestManager.getPendingRequests()) .as("Non-renamed immediate (i.e., run-now scheduled without a runId) nodes can be deleted") .hasSize(1) .contains(immediateRequest.toString()); requestManager.deletePendingRequest(immediateRequest); Assertions.assertThat(requestManager.getPendingRequests()) .as("Renamed one-off (i.e., run-now on-demand) nodes can be deleted") .hasSize(0);
if (requestManager.isBouncing(taskId.getRequestId())) { List<SingularityTaskId> activeTaskIds = taskManager.getActiveTaskIdsForRequest(taskId.getRequestId()); boolean foundBouncingTask = false; Optional<SingularityExpiringBounce> expiringBounce = requestManager.getExpiringBounce(taskId.getRequestId()); requestManager.removeExpiringBounce(taskId.getRequestId()); requestManager.markBounceComplete(taskId.getRequestId());
@Override protected void handleExpiringObject(SingularityExpiringBounce expiringObject, SingularityRequestWithState requestWithState, String message) { for (SingularityTaskCleanup taskCleanup : taskManager.getCleanupTasks()) { if (taskCleanup.getTaskId().getRequestId().equals(expiringObject.getRequestId()) && taskCleanup.getActionId().isPresent() && expiringObject.getActionId().equals(taskCleanup.getActionId().get())) { LOG.info("Discarding cleanup for {} ({}) because of {}", taskCleanup.getTaskId(), taskCleanup, expiringObject); taskManager.deleteCleanupTask(taskCleanup.getTaskId().getId()); if (!taskManager.getTaskCleanup(taskCleanup.getTaskId().getId()).isPresent()) { LOG.info("No other task cleanups found, removing task cleanup update for {}", taskCleanup.getTaskId()); List<SingularityTaskHistoryUpdate> historyUpdates = taskManager.getTaskHistoryUpdates(taskCleanup.getTaskId()); Collections.sort(historyUpdates); if (Iterables.getLast(historyUpdates).getTaskState() == ExtendedTaskState.TASK_CLEANING) { Optional<SingularityTaskHistoryUpdate> maybePreviousHistoryUpdate = historyUpdates.size() > 1 ? Optional.of(historyUpdates.get(historyUpdates.size() - 2)) : Optional.<SingularityTaskHistoryUpdate>absent(); taskManager.deleteTaskHistoryUpdate(taskCleanup.getTaskId(), ExtendedTaskState.TASK_CLEANING, maybePreviousHistoryUpdate); } } } } Optional<SingularityPendingRequest> pendingRequest = requestManager.getPendingRequest(expiringObject.getRequestId(), expiringObject.getDeployId()); if (pendingRequest.isPresent() && pendingRequest.get().getActionId().isPresent() && pendingRequest.get().getActionId().get().equals(expiringObject.getActionId())) { LOG.info("Discarding pending request for {} ({}) because of {}", expiringObject.getRequestId(), pendingRequest.get(), expiringObject); requestManager.deletePendingRequest(pendingRequest.get()); } requestManager.addToPendingQueue(new SingularityPendingRequest(expiringObject.getRequestId(), expiringObject.getDeployId(), System.currentTimeMillis(), expiringObject.getUser(), PendingType.CANCEL_BOUNCE, Optional.absent(), Optional.absent(), Optional.absent(), Optional.of(message), Optional.of(expiringObject.getActionId()))); }
Optional<Boolean> removeFromLoadBalancer = Optional.absent(); SingularityCreateResult result = requestManager.createCleanupRequest(new SingularityRequestCleanup(user.getEmail(), RequestCleanupType.PAUSING, now, killTasks, removeFromLoadBalancer, requestId, Optional.<String> absent(), Optional.<Boolean> absent(), message, actionId, runBeforeKill)); requestManager.pause(requestWithState.getRequest(), now, user.getEmail(), message); requestManager.saveExpiringObject(new SingularityExpiringPause(requestId, user.getEmail(), System.currentTimeMillis(), pauseRequest.get(), actionId.get()));
checkConflict(!(requestManager.markAsBouncing(requestId) == SingularityCreateResult.EXISTED), "%s is already bouncing", requestId); requestManager.createCleanupRequest( new SingularityRequestCleanup(user.getEmail(), isIncrementalBounce ? RequestCleanupType.INCREMENTAL_BOUNCE : RequestCleanupType.BOUNCE, System.currentTimeMillis(), Optional.absent(), Optional.absent(), requestId, Optional.of(deployId), skipHealthchecks, message, actionId, runBeforeKill)); requestManager.bounce(requestWithState.getRequest(), System.currentTimeMillis(), Optional.of(user.getId()), message); requestManager.saveExpiringObject(new SingularityExpiringBounce(requestId, deployId, Optional.of(user.getId()), System.currentTimeMillis(), validatedBounceRequest, actionId.get()));
protected void saveAndSchedule(SingularityRequestBuilder bldr) { SingularityRequest build = bldr.build(); requestManager.activate(build, RequestHistoryType.UPDATED, System.currentTimeMillis(), Optional.<String> absent(), Optional.<String> absent()); requestManager.addToPendingQueue(new SingularityPendingRequest(build.getId(), firstDeployId, System.currentTimeMillis(), Optional.<String> absent(), PendingType.UPDATED_REQUEST, Optional.<Boolean> absent(), Optional.<String> absent())); scheduler.drainPendingQueue(); }
@Test public void testObsoletePendingRequestsRemoved() { initRequest(); initFirstDeploy(); SingularityTask taskOne = startTask(firstDeploy); requestResource.pause(requestId, Optional.absent(), singularityUser); requestManager.addToPendingQueue(new SingularityPendingRequest(requestId, firstDeployId, System.currentTimeMillis(), Optional.<String>absent(), PendingType.NEW_DEPLOY, Optional.<Boolean>absent(), Optional.<String>absent())); Assert.assertEquals(requestManager.getPendingRequests().size(), 1); scheduler.drainPendingQueue(); Assert.assertEquals(requestManager.getPendingRequests().size(), 0); }
@Test public void badPauseExpires() { initRequest(); requestManager.createCleanupRequest(new SingularityRequestCleanup(Optional.<String>absent(), RequestCleanupType.PAUSING, System.currentTimeMillis(), Optional.<Boolean>absent(), Optional.absent(), requestId, Optional.<String>absent(), Optional.<Boolean> absent(), Optional.<String>absent(), Optional.<String>absent(), Optional.<SingularityShellCommand>absent())); cleaner.drainCleanupQueue(); Assert.assertTrue(!requestManager.getCleanupRequests().isEmpty()); configuration.setCleanupEverySeconds(0); sleep(1); cleaner.drainCleanupQueue(); Assert.assertTrue(requestManager.getCleanupRequests().isEmpty()); }