private long getAllowedMillis(SingularityDeploy deploy) { long seconds = deploy.getDeployHealthTimeoutSeconds().or(configuration.getDeployHealthyBySeconds()); if (deploy.getHealthcheck().isPresent() && !deploy.getSkipHealthchecksOnDeploy().or(false)) { seconds += deployHealthHelper.getMaxHealthcheckTimeoutSeconds(deploy.getHealthcheck().get()); } else { seconds += deploy.getConsiderHealthyAfterRunningForSeconds().or(configuration.getConsiderTaskHealthyAfterRunningForSeconds()); } return TimeUnit.SECONDS.toMillis(seconds); }
private boolean shouldCheckHealthchecks(final SingularityRequest request, final Optional<SingularityDeploy> deploy, final Collection<SingularityTaskId> activeTasks, final boolean isDeployPending) { if (disasterManager.isDisabled(SingularityAction.RUN_HEALTH_CHECKS)) { return false; } if (!deploy.isPresent()) { return false; } if (!deploy.get().getHealthcheck().isPresent()) { return false; } if (isDeployPending && deploy.get().getSkipHealthchecksOnDeploy().or(false)) { return false; } if (request.getSkipHealthchecks().or(Boolean.FALSE)) { return false; } for (SingularityTask task : taskManager.getTasks(activeTasks).values()) { if (task.getTaskRequest().getPendingTask().getSkipHealthchecks().or(Boolean.FALSE)) { return false; } } return true; }
private void maybeUpdatePendingRequest(SingularityPendingDeploy pendingDeploy, Optional<SingularityDeploy> deploy, SingularityRequest request, Optional<SingularityUpdatePendingDeployRequest> updatePendingDeployRequest, Optional<SingularityLoadBalancerUpdate> lbUpdate) { if (updatePendingDeployRequest.isPresent() && pendingDeploy.getDeployProgress().isPresent()) { SingularityDeployProgress newProgress = pendingDeploy.getDeployProgress().get().withNewTargetInstances(Math.min(updatePendingDeployRequest.get().getTargetActiveInstances(), request.getInstancesSafe())); updatePendingDeploy(pendingDeploy, lbUpdate.or(pendingDeploy.getLastLoadBalancerUpdate()), DeployState.WAITING, Optional.of(newProgress)); requestManager .addToPendingQueue(new SingularityPendingRequest(request.getId(), pendingDeploy.getDeployMarker().getDeployId(), System.currentTimeMillis(), pendingDeploy.getDeployMarker().getUser(), PendingType.NEXT_DEPLOY_STEP, deploy.isPresent() ? deploy.get().getSkipHealthchecksOnDeploy() : Optional.<Boolean> absent(), pendingDeploy.getDeployMarker().getMessage())); } }
private boolean healthchecksSkipped(SingularityTaskId taskId, List<SingularityRequestHistory> requestHistories, SingularityDeploy deploy) { if (deploy.getSkipHealthchecksOnDeploy().or(false)) { return true; } Optional<SingularityTask> maybeTask = taskManager.getTask(taskId); if (maybeTask.isPresent()) { if (maybeTask.get().getTaskRequest().getPendingTask().getSkipHealthchecks().or(false)) { return true; } Optional<Long> runningStartTime = getRunningAt(taskManager.getTaskHistoryUpdates(taskId)); if (runningStartTime.isPresent()) { Optional<SingularityRequestHistory> previousHistory = Optional.absent(); for (SingularityRequestHistory history : requestHistories) { if (history.getCreatedAt() < runningStartTime.get() && (!previousHistory.isPresent() || previousHistory.get().getCreatedAt() < history.getCreatedAt())) { previousHistory = Optional.of(history); } } if (previousHistory.isPresent() && previousHistory.get().getRequest().getSkipHealthchecks().or(false)) { return true; } } } return false; }
private SingularityDeployResult checkCanMoveToNextDeployStep(SingularityRequest request, Optional<SingularityDeploy> deploy, SingularityPendingDeploy pendingDeploy, Optional<SingularityUpdatePendingDeployRequest> updatePendingDeployRequest) { SingularityDeployProgress deployProgress = pendingDeploy.getDeployProgress().get(); if (canMoveToNextStep(deployProgress) || updatePendingDeployRequest.isPresent()) { SingularityDeployProgress newProgress = deployProgress.withNewTargetInstances(getNewTargetInstances(deployProgress, request, updatePendingDeployRequest)); updatePendingDeploy(pendingDeploy, pendingDeploy.getLastLoadBalancerUpdate(), DeployState.WAITING, Optional.of(newProgress)); requestManager.addToPendingQueue( new SingularityPendingRequest(request.getId(), pendingDeploy.getDeployMarker().getDeployId(), System.currentTimeMillis(), pendingDeploy.getDeployMarker().getUser(), PendingType.NEXT_DEPLOY_STEP, deploy.isPresent() ? deploy.get().getSkipHealthchecksOnDeploy() : Optional.<Boolean> absent(), pendingDeploy.getDeployMarker().getMessage())); } return new SingularityDeployResult(DeployState.WAITING); }
private boolean shouldHealthcheck(final SingularityTask task, final Optional<SingularityRequestWithState> request, Optional<SingularityPendingDeploy> pendingDeploy) { if (disasterManager.isDisabled(SingularityAction.RUN_HEALTH_CHECKS)) { return false; } if (!task.getTaskRequest().getRequest().isLongRunning() || !task.getTaskRequest().getDeploy().getHealthcheck().isPresent() || task.getTaskRequest().getDeploy().getHealthcheck().get().getHealthcheckResultFilePath().isPresent()) { return false; } if (task.getTaskRequest().getPendingTask().getSkipHealthchecks().or(false)) { return false; } if (pendingDeploy.isPresent() && pendingDeploy.get().getDeployMarker().getDeployId().equals(task.getTaskId().getDeployId()) && task.getTaskRequest().getDeploy().getSkipHealthchecksOnDeploy().or(false)) { return false; } if (request.isPresent() && request.get().getRequest().getSkipHealthchecks().or(false)) { return false; } Optional<SingularityTaskHealthcheckResult> lastHealthcheck = taskManager.getLastHealthcheck(task.getTaskId()); if (lastHealthcheck.isPresent() && !lastHealthcheck.get().isFailed()) { LOG.debug("Not submitting a new healthcheck for {} because it already passed a healthcheck", task.getTaskId()); return false; } return true; }
private SingularityDeployResult getDeployResult(final SingularityRequest request, final RequestState requestState, final Optional<SingularityDeployMarker> cancelRequest, final SingularityPendingDeploy pendingDeploy, final Optional<SingularityUpdatePendingDeployRequest> updatePendingDeployRequest, final Optional<SingularityDeploy> deploy, final Collection<SingularityTaskId> deployActiveTasks, final Collection<SingularityTaskId> otherActiveTasks, final Collection<SingularityTaskId> inactiveDeployMatchingTasks) { if (!request.isDeployable() || (configuration.isAllowDeployOfPausedRequests() && requestState == RequestState.PAUSED)) { LOG.info("Succeeding a deploy {} because the request {} was not deployable", pendingDeploy, request); return new SingularityDeployResult(DeployState.SUCCEEDED, "Request not deployable"); } if (!pendingDeploy.getDeployProgress().isPresent()) { return new SingularityDeployResult(DeployState.FAILED, "No deploy progress data present in Zookeeper. Please reattempt your deploy"); } Set<SingularityTaskId> newInactiveDeployTasks = getNewInactiveDeployTasks(pendingDeploy, inactiveDeployMatchingTasks); if (!newInactiveDeployTasks.isEmpty()) { if (canRetryTasks(deploy, inactiveDeployMatchingTasks)) { SingularityDeployProgress newProgress = pendingDeploy.getDeployProgress().get().withFailedTasks(new HashSet<>(inactiveDeployMatchingTasks)); updatePendingDeploy(pendingDeploy, pendingDeploy.getLastLoadBalancerUpdate(), DeployState.WAITING, Optional.of(newProgress)); requestManager.addToPendingQueue( new SingularityPendingRequest(request.getId(), pendingDeploy.getDeployMarker().getDeployId(), System.currentTimeMillis(), pendingDeploy.getDeployMarker().getUser(), PendingType.NEXT_DEPLOY_STEP, deploy.isPresent() ? deploy.get().getSkipHealthchecksOnDeploy() : Optional.<Boolean> absent(), pendingDeploy.getDeployMarker().getMessage())); return new SingularityDeployResult(DeployState.WAITING); } if (request.isLoadBalanced() && shouldCancelLoadBalancer(pendingDeploy)) { LOG.info("Attempting to cancel pending load balancer request, failing deploy {} regardless", pendingDeploy); sendCancelToLoadBalancer(pendingDeploy); } int maxRetries = deploy.get().getMaxTaskRetries().or(configuration.getDefaultDeployMaxTaskRetries()); return getDeployResultWithFailures(request, deploy, pendingDeploy, DeployState.FAILED, String.format("%s task(s) for this deploy failed", inactiveDeployMatchingTasks.size() - maxRetries), inactiveDeployMatchingTasks); } return checkDeployProgress(request, cancelRequest, pendingDeploy, updatePendingDeployRequest, deploy, deployActiveTasks, otherActiveTasks); }
deployRequest.getDeploy().getSkipHealthchecksOnDeploy(), deployRequest.getMessage()));
@Test public void testRequestsInPendingQueueAreOrderedByTimestamp() { long now = System.currentTimeMillis(); initRequestWithType(RequestType.SCHEDULED, false); startFirstDeploy(); SingularityPendingRequest pendingDeployRequest = new SingularityPendingRequest(requestId, firstDeploy.getId(), now, Optional.absent(), PendingType.NEW_DEPLOY, firstDeploy.getSkipHealthchecksOnDeploy(), Optional.absent()); SingularityPendingRequest pendingRunNowRequest = new SingularityPendingRequest(requestId, firstDeploy.getId(), now + 200, Optional.absent(), PendingType.IMMEDIATE, firstDeploy.getSkipHealthchecksOnDeploy(), Optional.absent()); requestManager.addToPendingQueue(pendingDeployRequest); requestManager.addToPendingQueue(pendingRunNowRequest); Assert.assertEquals(2, requestManager.getPendingRequests().size()); // Was added first Assert.assertEquals(PendingType.NEW_DEPLOY, requestManager.getPendingRequests().get(0).getPendingType()); // Was added second Assert.assertEquals(PendingType.IMMEDIATE, requestManager.getPendingRequests().get(1).getPendingType()); resourceOffers(); }
@Test public void testImmediateRequestsAreConsistentlyDeleted() { long now = System.currentTimeMillis(); initRequestWithType(RequestType.SCHEDULED, false); startFirstDeploy(); SingularityPendingRequest pendingDeployRequest = new SingularityPendingRequest(requestId, firstDeploy.getId(), now, Optional.absent(), PendingType.NEW_DEPLOY, firstDeploy.getSkipHealthchecksOnDeploy(), Optional.absent()); SingularityPendingRequest pendingRunNowRequest = new SingularityPendingRequest(requestId, firstDeploy.getId(), now + 200, Optional.absent(), PendingType.IMMEDIATE, firstDeploy.getSkipHealthchecksOnDeploy(), Optional.absent()); requestManager.addToPendingQueue(pendingDeployRequest); requestManager.addToPendingQueue(pendingRunNowRequest); // Pending queue has two requests: NEW_DEPLOY & IMMEDIATE Assert.assertEquals(2, requestManager.getPendingRequests().size()); finishNewTaskChecks(); requestManager.deletePendingRequest(pendingDeployRequest); // Just the immediate run Assert.assertEquals(1, requestManager.getPendingRequests().size()); requestManager.deletePendingRequest(pendingRunNowRequest); // Immediate run was successfully deleted Assert.assertEquals(0, requestManager.getPendingRequests().size()); }
@Test public void testImmediateRunReplacesScheduledTask() { initScheduledRequest(); SingularityDeploy deploy = SingularityDeploy.newBuilder(requestId, firstDeployId) .setCommand(Optional.of("sleep 100")) .build(); SingularityDeployRequest singularityDeployRequest = new SingularityDeployRequest(deploy, Optional.absent(), Optional.absent(), Optional.absent()); deployResource.deploy(singularityDeployRequest, singularityUser); scheduler.drainPendingQueue(); SingularityPendingTask task1 = createAndSchedulePendingTask(firstDeployId); Assert.assertEquals(1, taskManager.getPendingTaskIds().size()); Assert.assertEquals(PendingType.NEW_DEPLOY, taskManager.getPendingTaskIds().get(0).getPendingType()); requestManager.addToPendingQueue(new SingularityPendingRequest(requestId, deploy.getId(), System.currentTimeMillis(), Optional.absent(), PendingType.IMMEDIATE, deploy.getSkipHealthchecksOnDeploy(), Optional.absent())); scheduler.drainPendingQueue(); Assert.assertEquals(1, taskManager.getPendingTaskIds().size()); Assert.assertEquals(PendingType.IMMEDIATE, taskManager.getPendingTaskIds().get(0).getPendingType()); }
requestManager.addToPendingQueue(new SingularityPendingRequest(request.getId(), pendingDeploy.getDeployMarker().getDeployId(), deployResult.getTimestamp(), pendingDeploy.getDeployMarker().getUser(), pendingType, deploy.isPresent() ? deploy.get().getSkipHealthchecksOnDeploy() : Optional.absent(), pendingDeploy.getDeployMarker().getMessage()));
private long getAllowedMillis(SingularityDeploy deploy) { long seconds = deploy.getDeployHealthTimeoutSeconds().or(configuration.getDeployHealthyBySeconds()); if (deploy.getHealthcheck().isPresent() && !deploy.getSkipHealthchecksOnDeploy().or(false)) { seconds += deployHealthHelper.getMaxHealthcheckTimeoutSeconds(deploy.getHealthcheck().get()); } else { seconds += deploy.getConsiderHealthyAfterRunningForSeconds().or(configuration.getConsiderTaskHealthyAfterRunningForSeconds()); } return TimeUnit.SECONDS.toMillis(seconds); }
private boolean shouldCheckHealthchecks(final SingularityRequest request, final Optional<SingularityDeploy> deploy, final Collection<SingularityTaskId> activeTasks, final boolean isDeployPending) { if (disasterManager.isDisabled(SingularityAction.RUN_HEALTH_CHECKS)) { return false; } if (!deploy.isPresent()) { return false; } if (!deploy.get().getHealthcheck().isPresent()) { return false; } if (isDeployPending && deploy.get().getSkipHealthchecksOnDeploy().or(false)) { return false; } if (request.getSkipHealthchecks().or(Boolean.FALSE)) { return false; } for (SingularityTask task : taskManager.getTasks(activeTasks).values()) { if (task.getTaskRequest().getPendingTask().getSkipHealthchecks().or(Boolean.FALSE)) { return false; } } return true; }
private void maybeUpdatePendingRequest(SingularityPendingDeploy pendingDeploy, Optional<SingularityDeploy> deploy, SingularityRequest request, Optional<SingularityUpdatePendingDeployRequest> updatePendingDeployRequest, Optional<SingularityLoadBalancerUpdate> lbUpdate) { if (updatePendingDeployRequest.isPresent() && pendingDeploy.getDeployProgress().isPresent()) { SingularityDeployProgress newProgress = pendingDeploy.getDeployProgress().get().withNewTargetInstances(Math.min(updatePendingDeployRequest.get().getTargetActiveInstances(), request.getInstancesSafe())); updatePendingDeploy(pendingDeploy, lbUpdate.or(pendingDeploy.getLastLoadBalancerUpdate()), DeployState.WAITING, Optional.of(newProgress)); requestManager .addToPendingQueue(new SingularityPendingRequest(request.getId(), pendingDeploy.getDeployMarker().getDeployId(), System.currentTimeMillis(), pendingDeploy.getDeployMarker().getUser(), PendingType.NEXT_DEPLOY_STEP, deploy.isPresent() ? deploy.get().getSkipHealthchecksOnDeploy() : Optional.<Boolean> absent(), pendingDeploy.getDeployMarker().getMessage())); } }
private boolean healthchecksSkipped(SingularityTaskId taskId, List<SingularityRequestHistory> requestHistories, SingularityDeploy deploy) { if (deploy.getSkipHealthchecksOnDeploy().or(false)) { return true; } Optional<SingularityTask> maybeTask = taskManager.getTask(taskId); if (maybeTask.isPresent()) { if (maybeTask.get().getTaskRequest().getPendingTask().getSkipHealthchecks().or(false)) { return true; } Optional<Long> runningStartTime = getRunningAt(taskManager.getTaskHistoryUpdates(taskId)); if (runningStartTime.isPresent()) { Optional<SingularityRequestHistory> previousHistory = Optional.absent(); for (SingularityRequestHistory history : requestHistories) { if (history.getCreatedAt() < runningStartTime.get() && (!previousHistory.isPresent() || previousHistory.get().getCreatedAt() < history.getCreatedAt())) { previousHistory = Optional.of(history); } } if (previousHistory.isPresent() && previousHistory.get().getRequest().getSkipHealthchecks().or(false)) { return true; } } } return false; }
private SingularityDeployResult checkCanMoveToNextDeployStep(SingularityRequest request, Optional<SingularityDeploy> deploy, SingularityPendingDeploy pendingDeploy, Optional<SingularityUpdatePendingDeployRequest> updatePendingDeployRequest) { SingularityDeployProgress deployProgress = pendingDeploy.getDeployProgress().get(); if (canMoveToNextStep(deployProgress) || updatePendingDeployRequest.isPresent()) { SingularityDeployProgress newProgress = deployProgress.withNewTargetInstances(getNewTargetInstances(deployProgress, request, updatePendingDeployRequest)); updatePendingDeploy(pendingDeploy, pendingDeploy.getLastLoadBalancerUpdate(), DeployState.WAITING, Optional.of(newProgress)); requestManager.addToPendingQueue( new SingularityPendingRequest(request.getId(), pendingDeploy.getDeployMarker().getDeployId(), System.currentTimeMillis(), pendingDeploy.getDeployMarker().getUser(), PendingType.NEXT_DEPLOY_STEP, deploy.isPresent() ? deploy.get().getSkipHealthchecksOnDeploy() : Optional.<Boolean> absent(), pendingDeploy.getDeployMarker().getMessage())); } return new SingularityDeployResult(DeployState.WAITING); }
@Test public void testRequestsInPendingQueueAreOrderedByTimestamp() { long now = System.currentTimeMillis(); initRequestWithType(RequestType.SCHEDULED, false); startFirstDeploy(); SingularityPendingRequest pendingDeployRequest = new SingularityPendingRequest(requestId, firstDeploy.getId(), now, Optional.absent(), PendingType.NEW_DEPLOY, firstDeploy.getSkipHealthchecksOnDeploy(), Optional.absent()); SingularityPendingRequest pendingRunNowRequest = new SingularityPendingRequest(requestId, firstDeploy.getId(), now + 200, Optional.absent(), PendingType.IMMEDIATE, firstDeploy.getSkipHealthchecksOnDeploy(), Optional.absent()); requestManager.addToPendingQueue(pendingDeployRequest); requestManager.addToPendingQueue(pendingRunNowRequest); Assert.assertEquals(2, requestManager.getPendingRequests().size()); // Was added first Assert.assertEquals(PendingType.NEW_DEPLOY, requestManager.getPendingRequests().get(0).getPendingType()); // Was added second Assert.assertEquals(PendingType.IMMEDIATE, requestManager.getPendingRequests().get(1).getPendingType()); resourceOffers(); }
@Test public void testImmediateRequestsAreConsistentlyDeleted() { long now = System.currentTimeMillis(); initRequestWithType(RequestType.SCHEDULED, false); startFirstDeploy(); SingularityPendingRequest pendingDeployRequest = new SingularityPendingRequest(requestId, firstDeploy.getId(), now, Optional.absent(), PendingType.NEW_DEPLOY, firstDeploy.getSkipHealthchecksOnDeploy(), Optional.absent()); SingularityPendingRequest pendingRunNowRequest = new SingularityPendingRequest(requestId, firstDeploy.getId(), now + 200, Optional.absent(), PendingType.IMMEDIATE, firstDeploy.getSkipHealthchecksOnDeploy(), Optional.absent()); requestManager.addToPendingQueue(pendingDeployRequest); requestManager.addToPendingQueue(pendingRunNowRequest); // Pending queue has two requests: NEW_DEPLOY & IMMEDIATE Assert.assertEquals(2, requestManager.getPendingRequests().size()); finishNewTaskChecks(); requestManager.deletePendingRequest(pendingDeployRequest); // Just the immediate run Assert.assertEquals(1, requestManager.getPendingRequests().size()); requestManager.deletePendingRequest(pendingRunNowRequest); // Immediate run was successfully deleted Assert.assertEquals(0, requestManager.getPendingRequests().size()); }
@Test public void testImmediateRunReplacesScheduledTask() { initScheduledRequest(); SingularityDeploy deploy = SingularityDeploy.newBuilder(requestId, firstDeployId) .setCommand(Optional.of("sleep 100")) .build(); SingularityDeployRequest singularityDeployRequest = new SingularityDeployRequest(deploy, Optional.absent(), Optional.absent(), Optional.absent()); deployResource.deploy(singularityDeployRequest, singularityUser); scheduler.drainPendingQueue(); SingularityPendingTask task1 = createAndSchedulePendingTask(firstDeployId); Assert.assertEquals(1, taskManager.getPendingTaskIds().size()); Assert.assertEquals(PendingType.NEW_DEPLOY, taskManager.getPendingTaskIds().get(0).getPendingType()); requestManager.addToPendingQueue(new SingularityPendingRequest(requestId, deploy.getId(), System.currentTimeMillis(), Optional.absent(), PendingType.IMMEDIATE, deploy.getSkipHealthchecksOnDeploy(), Optional.absent())); scheduler.drainPendingQueue(); Assert.assertEquals(1, taskManager.getPendingTaskIds().size()); Assert.assertEquals(PendingType.IMMEDIATE, taskManager.getPendingTaskIds().get(0).getPendingType()); }