private void updatePendingDeploy(SingularityPendingDeploy pendingDeploy, Optional<SingularityLoadBalancerUpdate> lbUpdate, DeployState deployState, Optional<SingularityDeployProgress> deployProgress) { SingularityPendingDeploy copy = new SingularityPendingDeploy(pendingDeploy.getDeployMarker(), lbUpdate, deployState, deployProgress, pendingDeploy.getUpdatedRequest()); deployManager.savePendingDeploy(copy); }
private int numInstancesExpected(SingularityRequest request, SingularityPendingRequest pendingRequest, Optional<SingularityPendingDeploy> maybePendingDeploy) { if (!maybePendingDeploy.isPresent() || (maybePendingDeploy.get().getCurrentDeployState() == DeployState.CANCELED) || !maybePendingDeploy.get().getDeployProgress().isPresent()) { return request.getInstancesSafe(); } SingularityDeployProgress deployProgress = maybePendingDeploy.get().getDeployProgress().get(); if (maybePendingDeploy.get().getDeployMarker().getDeployId().equals(pendingRequest.getDeployId())) { return deployProgress.getTargetActiveInstances(); } else { if (deployProgress.isStepComplete()) { return Math.max(request.getInstancesSafe() - deployProgress.getTargetActiveInstances(), 0); } else { return request.getInstancesSafe() - (Math.max(deployProgress.getTargetActiveInstances() - deployProgress.getDeployInstanceCountPerStep(), 0)); } } }
private void maybeUpdatePendingRequest(SingularityPendingDeploy pendingDeploy, Optional<SingularityDeploy> deploy, SingularityRequest request, Optional<SingularityUpdatePendingDeployRequest> updatePendingDeployRequest, Optional<SingularityLoadBalancerUpdate> lbUpdate) { if (updatePendingDeployRequest.isPresent() && pendingDeploy.getDeployProgress().isPresent()) { SingularityDeployProgress newProgress = pendingDeploy.getDeployProgress().get().withNewTargetInstances(Math.min(updatePendingDeployRequest.get().getTargetActiveInstances(), request.getInstancesSafe())); updatePendingDeploy(pendingDeploy, lbUpdate.or(pendingDeploy.getLastLoadBalancerUpdate()), DeployState.WAITING, Optional.of(newProgress)); requestManager .addToPendingQueue(new SingularityPendingRequest(request.getId(), pendingDeploy.getDeployMarker().getDeployId(), System.currentTimeMillis(), pendingDeploy.getDeployMarker().getUser(), PendingType.NEXT_DEPLOY_STEP, deploy.isPresent() ? deploy.get().getSkipHealthchecksOnDeploy() : Optional.<Boolean> absent(), pendingDeploy.getDeployMarker().getMessage())); } }
private boolean shouldCancelLoadBalancer(final SingularityPendingDeploy pendingDeploy) { return pendingDeploy.getLastLoadBalancerUpdate().isPresent() && !pendingDeploy.getCurrentDeployState().isDeployFinished(); }
private long getStartTime(SingularityPendingDeploy pendingDeploy) { if (pendingDeploy.getDeployProgress().isPresent()) { return pendingDeploy.getDeployProgress().get().getTimestamp(); } else { return pendingDeploy.getDeployMarker().getTimestamp(); } }
private Set<SingularityTaskId> getNewInactiveDeployTasks(SingularityPendingDeploy pendingDeploy, Collection<SingularityTaskId> inactiveDeployMatchingTasks) { Set<SingularityTaskId> newInactiveDeployTasks = new HashSet<>(); newInactiveDeployTasks.addAll(inactiveDeployMatchingTasks); if (pendingDeploy.getDeployProgress().isPresent()) { newInactiveDeployTasks.removeAll(pendingDeploy.getDeployProgress().get().getFailedDeployTasks()); } return newInactiveDeployTasks; }
@Override public SingularityDeployKey apply(@Nonnull SingularityPendingDeploy input) { return SingularityDeployKey.fromDeployMarker(input.getDeployMarker()); } };
private SingularityRequest updatedRequest(Optional<SingularityPendingDeploy> maybePendingDeploy, SingularityPendingRequest pendingRequest, SingularityRequestWithState currentRequest) { if (maybePendingDeploy.isPresent() && pendingRequest.getDeployId().equals(maybePendingDeploy.get().getDeployMarker().getDeployId())) { return maybePendingDeploy.get().getUpdatedRequest().or(currentRequest.getRequest()); } else { return currentRequest.getRequest(); } } }
.setDeployId(deployId) .setTimestamp(deployResult.getTimestamp()) .setUser(pendingDeploy.getDeployMarker().getUser()) .setCmdLineArgsList(runNowRequest.getCommandLineArgs()) .setRunId(runNowRequest.getRunId().or(Optional.of(UUID.randomUUID().toString()))) .setSkipHealthchecks(runNowRequest.getSkipHealthchecks().or(request.getSkipHealthchecks())) .setMessage(runNowRequest.getMessage() .or(pendingDeploy.getDeployMarker().getMessage())) .setResources(runNowRequest.getResources()) .setRunAsUserOverride(runNowRequest.getRunAsUserOverride()) requestManager.addToPendingQueue(new SingularityPendingRequest(request.getId(), pendingDeploy.getDeployMarker().getDeployId(), deployResult.getTimestamp(), pendingDeploy.getDeployMarker().getUser(), pendingType, deploy.isPresent() ? deploy.get().getSkipHealthchecksOnDeploy() : Optional.absent(), pendingDeploy.getDeployMarker().getMessage())); deployManager.saveDeployResult(pendingDeploy.getDeployMarker(), deploy, deployResult); maybeRequestDeployState.get().getActiveDeploy().get().getDeployId(), deployResult.getTimestamp(), pendingDeploy.getDeployMarker().getUser(), deployResult.getDeployState() == DeployState.CANCELED ? PendingType.DEPLOY_CANCELLED : PendingType.DEPLOY_FAILED, request.getSkipHealthchecks(), pendingDeploy.getDeployMarker().getMessage())); .filter((t) -> t.getDeployId().equals(pendingDeploy.getDeployMarker().getDeployId())) .collect(Collectors.toList()); requestManager.addToPendingQueue(new SingularityPendingRequest( request.getId(),
private SingularityDeployResult markStepFinished(SingularityPendingDeploy pendingDeploy, Optional<SingularityDeploy> deploy, Collection<SingularityTaskId> deployActiveTasks, Collection<SingularityTaskId> otherActiveTasks, SingularityRequest request, Optional<SingularityUpdatePendingDeployRequest> updatePendingDeployRequest) { SingularityDeployProgress deployProgress = pendingDeploy.getDeployProgress().get(); if (updatePendingDeployRequest.isPresent() && getNewTargetInstances(deployProgress, request, updatePendingDeployRequest) != deployProgress.getTargetActiveInstances()) { maybeUpdatePendingRequest(pendingDeploy, deploy, request, updatePendingDeployRequest); return new SingularityDeployResult(DeployState.WAITING); } SingularityDeployProgress newProgress = deployProgress.withNewActiveInstances(deployActiveTasks.size()).withCompletedStep(); DeployState deployState = isLastStepFinished(newProgress, request) ? DeployState.SUCCEEDED : DeployState.WAITING; String message = deployState == DeployState.SUCCEEDED ? "New deploy succeeded" : "New deploy is progressing, this task is being replaced"; updatePendingDeploy(pendingDeploy, pendingDeploy.getLastLoadBalancerUpdate(), deployState, Optional.of(newProgress)); for (SingularityTaskId taskId : tasksToShutDown(deployProgress, otherActiveTasks, request)) { taskManager.createTaskCleanup( new SingularityTaskCleanup(Optional.<String> absent(), TaskCleanupType.DEPLOY_STEP_FINISHED, System.currentTimeMillis(), taskId, Optional.of(message), Optional.<String> absent(), Optional.<SingularityTaskShellCommandRequestId>absent())); } return new SingularityDeployResult(deployState); }
private SingularityDeployResult processLbState(SingularityRequest request, Optional<SingularityDeploy> deploy, SingularityPendingDeploy pendingDeploy, Optional<SingularityUpdatePendingDeployRequest> updatePendingDeployRequest, Collection<SingularityTaskId> deployActiveTasks, Collection<SingularityTaskId> otherActiveTasks, Collection<SingularityTaskId> tasksToShutDown, SingularityLoadBalancerUpdate lbUpdate) { List<SingularityTaskId> toRemoveFromLb = new ArrayList<>(); for (SingularityTaskId taskId : tasksToShutDown) { Optional<SingularityLoadBalancerUpdate> maybeRemoveUpdate = taskManager.getLoadBalancerState(taskId, LoadBalancerRequestType.REMOVE); if (maybeRemoveUpdate.isPresent() && maybeRemoveUpdate.get().getLoadBalancerRequestId().getId().equals(lbUpdate.getLoadBalancerRequestId().getId())) { toRemoveFromLb.add(taskId); } } updateLoadBalancerStateForTasks(deployActiveTasks, LoadBalancerRequestType.ADD, lbUpdate); updateLoadBalancerStateForTasks(toRemoveFromLb, LoadBalancerRequestType.REMOVE, lbUpdate); DeployState deployState = interpretLoadBalancerState(lbUpdate, pendingDeploy.getCurrentDeployState()); if (deployState == DeployState.SUCCEEDED) { updatePendingDeploy(pendingDeploy, Optional.of(lbUpdate), DeployState.WAITING); // A step has completed, markStepFinished will determine SUCCEEDED/WAITING return markStepFinished(pendingDeploy, deploy, deployActiveTasks, otherActiveTasks, request, updatePendingDeployRequest); } else if (deployState == DeployState.WAITING) { updatePendingDeploy(pendingDeploy, Optional.of(lbUpdate), deployState); maybeUpdatePendingRequest(pendingDeploy, deploy, request, updatePendingDeployRequest, Optional.of(lbUpdate)); return new SingularityDeployResult(DeployState.WAITING); } else { updatePendingDeploy(pendingDeploy, Optional.of(lbUpdate), deployState); maybeUpdatePendingRequest(pendingDeploy, deploy, request, updatePendingDeployRequest, Optional.of(lbUpdate)); return new SingularityDeployResult(deployState, lbUpdate, SingularityDeployFailure.lbUpdateFailed()); } }
deployChecker.checkDeploys(); SingularityPendingDeploy pendingDeploy = deployManager.getPendingDeploy(requestId).get(); Assert.assertEquals(DeployState.WAITING, pendingDeploy.getCurrentDeployState()); Assert.assertEquals(DeployState.WAITING, pendingDeploy.getCurrentDeployState()); SingularityDeployProgress deployProgressStepOne = pendingDeploy.getDeployProgress().get(); Assert.assertTrue(deployProgressStepOne.isStepComplete()); Assert.assertEquals(1, deployProgressStepOne.getTargetActiveInstances()); Assert.assertFalse(pendingDeploy.getDeployProgress().get().isStepComplete()); Assert.assertEquals(2, pendingDeploy.getDeployProgress().get().getTargetActiveInstances()); Assert.assertEquals(DeployState.WAITING, pendingDeploy.getCurrentDeployState());
protected void startDeploy(SingularityDeployMarker deployMarker, long timestamp) { SingularityDeployProgress startingDeployProgress = new SingularityDeployProgress(1, 0, 1, 10, false, true, Collections.<SingularityTaskId>emptySet(), timestamp); deployManager.savePendingDeploy(new SingularityPendingDeploy(deployMarker, Optional.<SingularityLoadBalancerUpdate>absent(), DeployState.WAITING, Optional.of(startingDeployProgress), Optional.<SingularityRequest>absent())); }
private boolean isWaitingForCurrentLbRequest(SingularityPendingDeploy pendingDeploy) { return pendingDeploy.getLastLoadBalancerUpdate().isPresent() && getLoadBalancerRequestId(pendingDeploy).getId().equals(pendingDeploy.getLastLoadBalancerUpdate().get().getLoadBalancerRequestId().getId()) && pendingDeploy.getLastLoadBalancerUpdate().get().getLoadBalancerState() == BaragonRequestState.WAITING; }
final SingularityRequest request = maybePendingDeploy.isPresent() ? maybePendingDeploy.get().getUpdatedRequest().or(maybeRequestWithState.get().getRequest()) : maybeRequestWithState.get().getRequest();
private void updatePendingDeploy(SingularityPendingDeploy pendingDeploy, Optional<SingularityLoadBalancerUpdate> lbUpdate, DeployState deployState) { updatePendingDeploy(pendingDeploy, lbUpdate, deployState, pendingDeploy.getDeployProgress()); }
@Override public String apply(@Nonnull SingularityPendingDeploy input) { return input.getDeployMarker().getRequestId(); } };
private LoadBalancerRequestId getLoadBalancerRequestId(SingularityPendingDeploy pendingDeploy) { return new LoadBalancerRequestId( String.format("%s-%s-%s", pendingDeploy.getDeployMarker().getRequestId(), pendingDeploy.getDeployMarker().getDeployId(), pendingDeploy.getDeployProgress().get().getTargetActiveInstances()), LoadBalancerRequestType.DEPLOY, Optional.<Integer> absent()); }
public List<SingularityTaskRequest> getTaskRequests(List<SingularityPendingTask> tasks) { final Multimap<String, SingularityPendingTask> requestIdToPendingTaskId = ArrayListMultimap.create(tasks.size(), 1); for (SingularityPendingTask task : tasks) { requestIdToPendingTaskId.put(task.getPendingTaskId().getRequestId(), task); } final List<SingularityRequestWithState> matchingRequests = requestManager.getRequests(requestIdToPendingTaskId.keySet()); final Map<SingularityPendingTask, SingularityDeployKey> deployKeys = SingularityDeployKey.fromPendingTasks(requestIdToPendingTaskId.values()); final Map<SingularityDeployKey, SingularityDeploy> matchingDeploys = deployManager.getDeploysForKeys(Sets.newHashSet(deployKeys.values())); final List<SingularityTaskRequest> taskRequests = Lists.newArrayListWithCapacity(matchingRequests.size()); for (SingularityRequestWithState request : matchingRequests) { Optional<SingularityPendingDeploy> maybePendingDeploy = deployManager.getPendingDeploy(request.getRequest().getId()); for (SingularityPendingTask task : requestIdToPendingTaskId.get(request.getRequest().getId())) { SingularityDeploy foundDeploy = matchingDeploys.get(deployKeys.get(task)); if (foundDeploy == null) { LOG.warn("Couldn't find a matching deploy for pending task {}", task); continue; } if (!request.getState().isRunnable()) { LOG.warn("Request was in state {} for pending task {}", request.getState(), task); continue; } Optional<SingularityRequest> updatedRequest = maybePendingDeploy.isPresent() && maybePendingDeploy.get().getDeployMarker().getDeployId().equals(task.getPendingTaskId().getDeployId()) ? maybePendingDeploy.get().getUpdatedRequest() : Optional.<SingularityRequest>absent(); taskRequests.add(new SingularityTaskRequest(updatedRequest.or(request.getRequest()), foundDeploy, task)); } } return taskRequests; }
.setDeployId(deployId) .setTimestamp(deployResult.getTimestamp()) .setUser(pendingDeploy.getDeployMarker().getUser()) .setCmdLineArgsList(runNowRequest.getCommandLineArgs()) .setRunId(runNowRequest.getRunId().or(Optional.of(UUID.randomUUID().toString()))) .setSkipHealthchecks(runNowRequest.getSkipHealthchecks().or(request.getSkipHealthchecks())) .setMessage(runNowRequest.getMessage() .or(pendingDeploy.getDeployMarker().getMessage())) .setResources(runNowRequest.getResources()) .setRunAsUserOverride(runNowRequest.getRunAsUserOverride()) requestManager.addToPendingQueue(new SingularityPendingRequest(request.getId(), pendingDeploy.getDeployMarker().getDeployId(), deployResult.getTimestamp(), pendingDeploy.getDeployMarker().getUser(), pendingType, deploy.isPresent() ? deploy.get().getSkipHealthchecksOnDeploy() : Optional.absent(), pendingDeploy.getDeployMarker().getMessage())); deployManager.saveDeployResult(pendingDeploy.getDeployMarker(), deploy, deployResult); maybeRequestDeployState.get().getActiveDeploy().get().getDeployId(), deployResult.getTimestamp(), pendingDeploy.getDeployMarker().getUser(), deployResult.getDeployState() == DeployState.CANCELED ? PendingType.DEPLOY_CANCELLED : PendingType.DEPLOY_FAILED, request.getSkipHealthchecks(), pendingDeploy.getDeployMarker().getMessage())); .filter((t) -> t.getDeployId().equals(pendingDeploy.getDeployMarker().getDeployId())) .collect(Collectors.toList()); requestManager.addToPendingQueue(new SingularityPendingRequest( request.getId(),