private Set<SingularityTaskId> getNewInactiveDeployTasks(SingularityPendingDeploy pendingDeploy, Collection<SingularityTaskId> inactiveDeployMatchingTasks) { Set<SingularityTaskId> newInactiveDeployTasks = new HashSet<>(); newInactiveDeployTasks.addAll(inactiveDeployMatchingTasks); if (pendingDeploy.getDeployProgress().isPresent()) { newInactiveDeployTasks.removeAll(pendingDeploy.getDeployProgress().get().getFailedDeployTasks()); } return newInactiveDeployTasks; }
private void updatePendingDeploy(SingularityPendingDeploy pendingDeploy, Optional<SingularityLoadBalancerUpdate> lbUpdate, DeployState deployState) { updatePendingDeploy(pendingDeploy, lbUpdate, deployState, pendingDeploy.getDeployProgress()); }
private List<SingularityDeployFailure> getDeployFailures(SingularityRequest request, Optional<SingularityDeploy> deploy, SingularityPendingDeploy pendingDeploy, DeployState state, Collection<SingularityTaskId> matchingTasks) { List<SingularityDeployFailure> failures = new ArrayList<>(); failures.addAll(deployHealthHelper.getTaskFailures(deploy, matchingTasks)); if (state == DeployState.OVERDUE) { int targetInstances = pendingDeploy.getDeployProgress().isPresent() ? pendingDeploy.getDeployProgress().get().getTargetActiveInstances() :request.getInstancesSafe(); if (failures.isEmpty() && matchingTasks.size() < targetInstances) { failures.add(new SingularityDeployFailure(SingularityDeployFailureReason.TASK_COULD_NOT_BE_SCHEDULED, Optional.<SingularityTaskId>absent(), Optional.of(String.format("Only %s of %s tasks could be launched for deploy, there may not be enough resources to launch the remaining tasks", matchingTasks.size(), targetInstances)))); } } return failures; } }
private long getStartTime(SingularityPendingDeploy pendingDeploy) { if (pendingDeploy.getDeployProgress().isPresent()) { return pendingDeploy.getDeployProgress().get().getTimestamp(); } else { return pendingDeploy.getDeployMarker().getTimestamp(); } }
private TaskCleanupType getCleanupType(SingularityPendingDeploy pendingDeploy, SingularityRequest request, SingularityDeployResult deployResult) { if (pendingDeploy.getDeployProgress().isPresent() && pendingDeploy.getDeployProgress().get().getDeployInstanceCountPerStep() != request.getInstancesSafe()) { // For incremental deploys, return a special cleanup type if (deployResult.getDeployState() == DeployState.FAILED) { return TaskCleanupType.INCREMENTAL_DEPLOY_FAILED; } else if (deployResult.getDeployState() == DeployState.CANCELED) { return TaskCleanupType.INCREMENTAL_DEPLOY_CANCELLED; } } return deployResult.getDeployState().getCleanupType(); }
private boolean isDeployOverdue(SingularityPendingDeploy pendingDeploy, Optional<SingularityDeploy> deploy) { if (!deploy.isPresent()) { LOG.warn("Can't determine if deploy {} is overdue because it was missing", pendingDeploy); return false; } if (pendingDeploy.getDeployProgress().isPresent() && pendingDeploy.getDeployProgress().get().isStepComplete()) { return false; } final long startTime = getStartTime(pendingDeploy); final long deployDuration = System.currentTimeMillis() - startTime; final long allowedTime = getAllowedMillis(deploy.get()); if (deployDuration > allowedTime) { LOG.warn("Deploy {} is overdue (duration: {}), allowed: {}", pendingDeploy, DurationFormatUtils.formatDurationHMS(deployDuration), DurationFormatUtils.formatDurationHMS(allowedTime)); return true; } else { LOG.trace("Deploy {} is not yet overdue (duration: {}), allowed: {}", pendingDeploy, DurationFormatUtils.formatDurationHMS(deployDuration), DurationFormatUtils.formatDurationHMS(allowedTime)); return false; } }
private void maybeUpdatePendingRequest(SingularityPendingDeploy pendingDeploy, Optional<SingularityDeploy> deploy, SingularityRequest request, Optional<SingularityUpdatePendingDeployRequest> updatePendingDeployRequest, Optional<SingularityLoadBalancerUpdate> lbUpdate) { if (updatePendingDeployRequest.isPresent() && pendingDeploy.getDeployProgress().isPresent()) { SingularityDeployProgress newProgress = pendingDeploy.getDeployProgress().get().withNewTargetInstances(Math.min(updatePendingDeployRequest.get().getTargetActiveInstances(), request.getInstancesSafe())); updatePendingDeploy(pendingDeploy, lbUpdate.or(pendingDeploy.getLastLoadBalancerUpdate()), DeployState.WAITING, Optional.of(newProgress)); requestManager .addToPendingQueue(new SingularityPendingRequest(request.getId(), pendingDeploy.getDeployMarker().getDeployId(), System.currentTimeMillis(), pendingDeploy.getDeployMarker().getUser(), PendingType.NEXT_DEPLOY_STEP, deploy.isPresent() ? deploy.get().getSkipHealthchecksOnDeploy() : Optional.<Boolean> absent(), pendingDeploy.getDeployMarker().getMessage())); } }
private int numInstancesExpected(SingularityRequest request, SingularityPendingRequest pendingRequest, Optional<SingularityPendingDeploy> maybePendingDeploy) { if (!maybePendingDeploy.isPresent() || (maybePendingDeploy.get().getCurrentDeployState() == DeployState.CANCELED) || !maybePendingDeploy.get().getDeployProgress().isPresent()) { return request.getInstancesSafe(); } SingularityDeployProgress deployProgress = maybePendingDeploy.get().getDeployProgress().get(); if (maybePendingDeploy.get().getDeployMarker().getDeployId().equals(pendingRequest.getDeployId())) { return deployProgress.getTargetActiveInstances(); } else { if (deployProgress.isStepComplete()) { return Math.max(request.getInstancesSafe() - deployProgress.getTargetActiveInstances(), 0); } else { return request.getInstancesSafe() - (Math.max(deployProgress.getTargetActiveInstances() - deployProgress.getDeployInstanceCountPerStep(), 0)); } } }
private LoadBalancerRequestId getLoadBalancerRequestId(SingularityPendingDeploy pendingDeploy) { return new LoadBalancerRequestId( String.format("%s-%s-%s", pendingDeploy.getDeployMarker().getRequestId(), pendingDeploy.getDeployMarker().getDeployId(), pendingDeploy.getDeployProgress().get().getTargetActiveInstances()), LoadBalancerRequestType.DEPLOY, Optional.<Integer> absent()); }
private SingularityDeployResult markStepFinished(SingularityPendingDeploy pendingDeploy, Optional<SingularityDeploy> deploy, Collection<SingularityTaskId> deployActiveTasks, Collection<SingularityTaskId> otherActiveTasks, SingularityRequest request, Optional<SingularityUpdatePendingDeployRequest> updatePendingDeployRequest) { SingularityDeployProgress deployProgress = pendingDeploy.getDeployProgress().get(); if (updatePendingDeployRequest.isPresent() && getNewTargetInstances(deployProgress, request, updatePendingDeployRequest) != deployProgress.getTargetActiveInstances()) { maybeUpdatePendingRequest(pendingDeploy, deploy, request, updatePendingDeployRequest); return new SingularityDeployResult(DeployState.WAITING); } SingularityDeployProgress newProgress = deployProgress.withNewActiveInstances(deployActiveTasks.size()).withCompletedStep(); DeployState deployState = isLastStepFinished(newProgress, request) ? DeployState.SUCCEEDED : DeployState.WAITING; String message = deployState == DeployState.SUCCEEDED ? "New deploy succeeded" : "New deploy is progressing, this task is being replaced"; updatePendingDeploy(pendingDeploy, pendingDeploy.getLastLoadBalancerUpdate(), deployState, Optional.of(newProgress)); for (SingularityTaskId taskId : tasksToShutDown(deployProgress, otherActiveTasks, request)) { taskManager.createTaskCleanup( new SingularityTaskCleanup(Optional.<String> absent(), TaskCleanupType.DEPLOY_STEP_FINISHED, System.currentTimeMillis(), taskId, Optional.of(message), Optional.<String> absent(), Optional.<SingularityTaskShellCommandRequestId>absent())); } return new SingularityDeployResult(deployState); }
private SingularityDeployResult checkCanMoveToNextDeployStep(SingularityRequest request, Optional<SingularityDeploy> deploy, SingularityPendingDeploy pendingDeploy, Optional<SingularityUpdatePendingDeployRequest> updatePendingDeployRequest) { SingularityDeployProgress deployProgress = pendingDeploy.getDeployProgress().get(); if (canMoveToNextStep(deployProgress) || updatePendingDeployRequest.isPresent()) { SingularityDeployProgress newProgress = deployProgress.withNewTargetInstances(getNewTargetInstances(deployProgress, request, updatePendingDeployRequest)); updatePendingDeploy(pendingDeploy, pendingDeploy.getLastLoadBalancerUpdate(), DeployState.WAITING, Optional.of(newProgress)); requestManager.addToPendingQueue( new SingularityPendingRequest(request.getId(), pendingDeploy.getDeployMarker().getDeployId(), System.currentTimeMillis(), pendingDeploy.getDeployMarker().getUser(), PendingType.NEXT_DEPLOY_STEP, deploy.isPresent() ? deploy.get().getSkipHealthchecksOnDeploy() : Optional.<Boolean> absent(), pendingDeploy.getDeployMarker().getMessage())); } return new SingularityDeployResult(DeployState.WAITING); }
private SingularityDeployResult enqueueAndProcessLbRequest(SingularityRequest request, Optional<SingularityDeploy> deploy, SingularityPendingDeploy pendingDeploy, Optional<SingularityUpdatePendingDeployRequest> updatePendingDeployRequest, Collection<SingularityTaskId> deployActiveTasks, Collection<SingularityTaskId> otherActiveTasks) { Collection<SingularityTaskId> toShutDown = tasksToShutDown(pendingDeploy.getDeployProgress().get(), otherActiveTasks, request); final Map<SingularityTaskId, SingularityTask> tasks = taskManager.getTasks(Iterables.concat(deployActiveTasks, toShutDown)); final LoadBalancerRequestId lbRequestId = getLoadBalancerRequestId(pendingDeploy); List<SingularityTaskId> toRemoveFromLb = new ArrayList<>(); for (SingularityTaskId taskId : toShutDown) { Optional<SingularityLoadBalancerUpdate> maybeAddUpdate = taskManager.getLoadBalancerState(taskId, LoadBalancerRequestType.ADD); if (maybeAddUpdate.isPresent() && maybeAddUpdate.get().getLoadBalancerState() == BaragonRequestState.SUCCESS) { toRemoveFromLb.add(taskId); } } updateLoadBalancerStateForTasks(deployActiveTasks, LoadBalancerRequestType.ADD, SingularityLoadBalancerUpdate.preEnqueue(lbRequestId)); updateLoadBalancerStateForTasks(toRemoveFromLb, LoadBalancerRequestType.REMOVE, SingularityLoadBalancerUpdate.preEnqueue(lbRequestId)); SingularityLoadBalancerUpdate enqueueResult = lbClient.enqueue(lbRequestId, request, deploy.get(), getTasks(deployActiveTasks, tasks), getTasks(toShutDown, tasks)); return processLbState(request, deploy, pendingDeploy, updatePendingDeployRequest, deployActiveTasks, otherActiveTasks, toShutDown, enqueueResult); }
final long now = System.currentTimeMillis(); if (maybePendingDeploy.isPresent() && maybePendingDeploy.get().getDeployProgress().isPresent()) {
private SingularityDeployResult getDeployResult(final SingularityRequest request, final RequestState requestState, final Optional<SingularityDeployMarker> cancelRequest, final SingularityPendingDeploy pendingDeploy, final Optional<SingularityUpdatePendingDeployRequest> updatePendingDeployRequest, final Optional<SingularityDeploy> deploy, final Collection<SingularityTaskId> deployActiveTasks, final Collection<SingularityTaskId> otherActiveTasks, final Collection<SingularityTaskId> inactiveDeployMatchingTasks) { if (!request.isDeployable() || (configuration.isAllowDeployOfPausedRequests() && requestState == RequestState.PAUSED)) { LOG.info("Succeeding a deploy {} because the request {} was not deployable", pendingDeploy, request); return new SingularityDeployResult(DeployState.SUCCEEDED, "Request not deployable"); } if (!pendingDeploy.getDeployProgress().isPresent()) { return new SingularityDeployResult(DeployState.FAILED, "No deploy progress data present in Zookeeper. Please reattempt your deploy"); } Set<SingularityTaskId> newInactiveDeployTasks = getNewInactiveDeployTasks(pendingDeploy, inactiveDeployMatchingTasks); if (!newInactiveDeployTasks.isEmpty()) { if (canRetryTasks(deploy, inactiveDeployMatchingTasks)) { SingularityDeployProgress newProgress = pendingDeploy.getDeployProgress().get().withFailedTasks(new HashSet<>(inactiveDeployMatchingTasks)); updatePendingDeploy(pendingDeploy, pendingDeploy.getLastLoadBalancerUpdate(), DeployState.WAITING, Optional.of(newProgress)); requestManager.addToPendingQueue( new SingularityPendingRequest(request.getId(), pendingDeploy.getDeployMarker().getDeployId(), System.currentTimeMillis(), pendingDeploy.getDeployMarker().getUser(), PendingType.NEXT_DEPLOY_STEP, deploy.isPresent() ? deploy.get().getSkipHealthchecksOnDeploy() : Optional.<Boolean> absent(), pendingDeploy.getDeployMarker().getMessage())); return new SingularityDeployResult(DeployState.WAITING); } if (request.isLoadBalanced() && shouldCancelLoadBalancer(pendingDeploy)) { LOG.info("Attempting to cancel pending load balancer request, failing deploy {} regardless", pendingDeploy); sendCancelToLoadBalancer(pendingDeploy); } int maxRetries = deploy.get().getMaxTaskRetries().or(configuration.getDefaultDeployMaxTaskRetries()); return getDeployResultWithFailures(request, deploy, pendingDeploy, DeployState.FAILED, String.format("%s task(s) for this deploy failed", inactiveDeployMatchingTasks.size() - maxRetries), inactiveDeployMatchingTasks); } return checkDeployProgress(request, cancelRequest, pendingDeploy, updatePendingDeployRequest, deploy, deployActiveTasks, otherActiveTasks); }
SingularityDeployProgress deployProgressStepOne = deployManager.getPendingDeploys().get(0).getDeployProgress().get(); Assert.assertTrue(deployProgressStepOne.isStepComplete()); Assert.assertEquals(1, deployProgressStepOne.getTargetActiveInstances()); SingularityDeployProgress deployProgressStepTwo = deployManager.getPendingDeploys().get(0).getDeployProgress().get(); Assert.assertFalse(deployProgressStepTwo.isStepComplete()); Assert.assertEquals(2, deployProgressStepTwo.getTargetActiveInstances());
SingularityDeployProgress deployProgressStepOne = deployManager.getPendingDeploys().get(0).getDeployProgress().get(); Assert.assertTrue(deployProgressStepOne.isStepComplete()); Assert.assertEquals(1, deployProgressStepOne.getTargetActiveInstances()); deployProgressStepOne = deployManager.getPendingDeploys().get(0).getDeployProgress().get(); Assert.assertTrue(deployProgressStepOne.isStepComplete()); Assert.assertEquals(1, deployProgressStepOne.getTargetActiveInstances()); SingularityDeployProgress deployProgressStepTwo = deployManager.getPendingDeploys().get(0).getDeployProgress().get(); Assert.assertFalse(deployProgressStepTwo.isStepComplete()); Assert.assertEquals(2, deployProgressStepTwo.getTargetActiveInstances());
deployChecker.checkDeploys(); SingularityDeployProgress deployProgress = deployManager.getPendingDeploy(requestId).get().getDeployProgress().get(); Assert.assertEquals(1, deployProgress.getTargetActiveInstances()); Assert.assertEquals(1, deployProgress.getCurrentActiveInstances()); deployProgress = deployManager.getPendingDeploy(requestId).get().getDeployProgress().get(); Assert.assertEquals(2, deployProgress.getTargetActiveInstances()); Assert.assertEquals(2, deployProgress.getCurrentActiveInstances()); deployProgress = deployManager.getPendingDeploy(requestId).get().getDeployProgress().get(); Assert.assertEquals(3, deployProgress.getTargetActiveInstances()); Assert.assertEquals(3, deployProgress.getCurrentActiveInstances());
List<SingularityTaskId> cleanupTaskIds = taskManager.getCleanupTaskIds(); Assert.assertTrue(cleanupTaskIds.contains(firstTask.getTaskId()) && cleanupTaskIds.contains(secondTask.getTaskId())); SingularityDeployProgress deployProgressStepOne = deployManager.getPendingDeploys().get(0).getDeployProgress().get(); Assert.assertTrue(deployProgressStepOne.isStepComplete()); Assert.assertEquals(2, deployProgressStepOne.getTargetActiveInstances()); SingularityDeployProgress deployProgressStepTwo = deployManager.getPendingDeploys().get(0).getDeployProgress().get(); Assert.assertFalse(deployProgressStepTwo.isStepComplete()); Assert.assertEquals(4, deployProgressStepTwo.getTargetActiveInstances());
SingularityDeployProgress deployProgressStepOne = pendingDeploy.getDeployProgress().get(); Assert.assertTrue(deployProgressStepOne.isStepComplete()); Assert.assertEquals(1, deployProgressStepOne.getTargetActiveInstances()); Assert.assertFalse(pendingDeploy.getDeployProgress().get().isStepComplete()); Assert.assertEquals(2, pendingDeploy.getDeployProgress().get().getTargetActiveInstances());
SingularityDeployProgress deployProgressStepOne = deployManager.getPendingDeploys().get(0).getDeployProgress().get(); Assert.assertTrue(deployProgressStepOne.isStepComplete()); Assert.assertEquals(1, deployProgressStepOne.getTargetActiveInstances());