private List<SingularityTaskId> tasksToShutDown(SingularityDeployProgress deployProgress, Collection<SingularityTaskId> otherActiveTasks, SingularityRequest request) { int numTasksToShutDown = Math.max(otherActiveTasks.size() - (request.getInstancesSafe() - deployProgress.getTargetActiveInstances()), 0); List<SingularityTaskId> sortedOtherTasks = new ArrayList<>(otherActiveTasks); Collections.sort(sortedOtherTasks, SingularityTaskId.INSTANCE_NO_COMPARATOR); return sortedOtherTasks.isEmpty() ? sortedOtherTasks : sortedOtherTasks.subList(0, Math.min(numTasksToShutDown, sortedOtherTasks.size())); }
private List<SingularityDeployFailure> getDeployFailures(SingularityRequest request, Optional<SingularityDeploy> deploy, SingularityPendingDeploy pendingDeploy, DeployState state, Collection<SingularityTaskId> matchingTasks) { List<SingularityDeployFailure> failures = new ArrayList<>(); failures.addAll(deployHealthHelper.getTaskFailures(deploy, matchingTasks)); if (state == DeployState.OVERDUE) { int targetInstances = pendingDeploy.getDeployProgress().isPresent() ? pendingDeploy.getDeployProgress().get().getTargetActiveInstances() :request.getInstancesSafe(); if (failures.isEmpty() && matchingTasks.size() < targetInstances) { failures.add(new SingularityDeployFailure(SingularityDeployFailureReason.TASK_COULD_NOT_BE_SCHEDULED, Optional.<SingularityTaskId>absent(), Optional.of(String.format("Only %s of %s tasks could be launched for deploy, there may not be enough resources to launch the remaining tasks", matchingTasks.size(), targetInstances)))); } } return failures; } }
private int getNewTargetInstances(SingularityDeployProgress deployProgress, SingularityRequest request, Optional<SingularityUpdatePendingDeployRequest> updateRequest) { if (updateRequest.isPresent()) { return Math.min(updateRequest.get().getTargetActiveInstances(), request.getInstancesSafe()); } else { return Math.min(deployProgress.getTargetActiveInstances() + deployProgress.getDeployInstanceCountPerStep(), request.getInstancesSafe()); } }
private boolean isLastStepFinished(SingularityDeployProgress deployProgress, SingularityRequest request) { return deployProgress.isStepComplete() && deployProgress.getTargetActiveInstances() >= request.getInstancesSafe(); }
private int numInstancesExpected(SingularityRequest request, SingularityPendingRequest pendingRequest, Optional<SingularityPendingDeploy> maybePendingDeploy) { if (!maybePendingDeploy.isPresent() || (maybePendingDeploy.get().getCurrentDeployState() == DeployState.CANCELED) || !maybePendingDeploy.get().getDeployProgress().isPresent()) { return request.getInstancesSafe(); } SingularityDeployProgress deployProgress = maybePendingDeploy.get().getDeployProgress().get(); if (maybePendingDeploy.get().getDeployMarker().getDeployId().equals(pendingRequest.getDeployId())) { return deployProgress.getTargetActiveInstances(); } else { if (deployProgress.isStepComplete()) { return Math.max(request.getInstancesSafe() - deployProgress.getTargetActiveInstances(), 0); } else { return request.getInstancesSafe() - (Math.max(deployProgress.getTargetActiveInstances() - deployProgress.getDeployInstanceCountPerStep(), 0)); } } }
private LoadBalancerRequestId getLoadBalancerRequestId(SingularityPendingDeploy pendingDeploy) { return new LoadBalancerRequestId( String.format("%s-%s-%s", pendingDeploy.getDeployMarker().getRequestId(), pendingDeploy.getDeployMarker().getDeployId(), pendingDeploy.getDeployProgress().get().getTargetActiveInstances()), LoadBalancerRequestType.DEPLOY, Optional.<Integer> absent()); }
private SingularityDeployResult markStepFinished(SingularityPendingDeploy pendingDeploy, Optional<SingularityDeploy> deploy, Collection<SingularityTaskId> deployActiveTasks, Collection<SingularityTaskId> otherActiveTasks, SingularityRequest request, Optional<SingularityUpdatePendingDeployRequest> updatePendingDeployRequest) { SingularityDeployProgress deployProgress = pendingDeploy.getDeployProgress().get(); if (updatePendingDeployRequest.isPresent() && getNewTargetInstances(deployProgress, request, updatePendingDeployRequest) != deployProgress.getTargetActiveInstances()) { maybeUpdatePendingRequest(pendingDeploy, deploy, request, updatePendingDeployRequest); return new SingularityDeployResult(DeployState.WAITING); } SingularityDeployProgress newProgress = deployProgress.withNewActiveInstances(deployActiveTasks.size()).withCompletedStep(); DeployState deployState = isLastStepFinished(newProgress, request) ? DeployState.SUCCEEDED : DeployState.WAITING; String message = deployState == DeployState.SUCCEEDED ? "New deploy succeeded" : "New deploy is progressing, this task is being replaced"; updatePendingDeploy(pendingDeploy, pendingDeploy.getLastLoadBalancerUpdate(), deployState, Optional.of(newProgress)); for (SingularityTaskId taskId : tasksToShutDown(deployProgress, otherActiveTasks, request)) { taskManager.createTaskCleanup( new SingularityTaskCleanup(Optional.<String> absent(), TaskCleanupType.DEPLOY_STEP_FINISHED, System.currentTimeMillis(), taskId, Optional.of(message), Optional.<String> absent(), Optional.<SingularityTaskShellCommandRequestId>absent())); } return new SingularityDeployResult(deployState); }
if (deployActiveTasks.size() < deployProgress.getTargetActiveInstances()) { maybeUpdatePendingRequest(pendingDeploy, deploy, request, updatePendingDeployRequest); return checkOverdue(request, deploy, pendingDeploy, deployActiveTasks, isDeployOverdue); if (updatePendingDeployRequest.isPresent() && updatePendingDeployRequest.get().getTargetActiveInstances() != deployProgress.getTargetActiveInstances()) { maybeUpdatePendingRequest(pendingDeploy, deploy, request, updatePendingDeployRequest); return new SingularityDeployResult(DeployState.WAITING);
if (pendingDeploy.getDeployProgress().get().getTargetActiveInstances() != request.getInstancesSafe()) { requestManager.addToPendingQueue(new SingularityPendingRequest(request.getId(), pendingDeploy.getDeployMarker().getDeployId(), deployResult.getTimestamp(), pendingDeploy.getDeployMarker().getUser(), PendingType.UPDATED_REQUEST, request.getSkipHealthchecks(), pendingDeploy.getDeployMarker().getMessage()));
SingularityDeployProgress deployProgressStepOne = deployManager.getPendingDeploys().get(0).getDeployProgress().get(); Assert.assertTrue(deployProgressStepOne.isStepComplete()); Assert.assertEquals(1, deployProgressStepOne.getTargetActiveInstances()); Assert.assertEquals(2, deployProgressStepTwo.getTargetActiveInstances());
SingularityDeployProgress deployProgressStepOne = deployManager.getPendingDeploys().get(0).getDeployProgress().get(); Assert.assertTrue(deployProgressStepOne.isStepComplete()); Assert.assertEquals(1, deployProgressStepOne.getTargetActiveInstances()); Assert.assertEquals(1, deployProgressStepOne.getTargetActiveInstances()); Assert.assertEquals(2, deployProgressStepTwo.getTargetActiveInstances());
private List<SingularityTaskId> tasksToShutDown(SingularityDeployProgress deployProgress, Collection<SingularityTaskId> otherActiveTasks, SingularityRequest request) { int numTasksToShutDown = Math.max(otherActiveTasks.size() - (request.getInstancesSafe() - deployProgress.getTargetActiveInstances()), 0); List<SingularityTaskId> sortedOtherTasks = new ArrayList<>(otherActiveTasks); Collections.sort(sortedOtherTasks, SingularityTaskId.INSTANCE_NO_COMPARATOR); return sortedOtherTasks.isEmpty() ? sortedOtherTasks : sortedOtherTasks.subList(0, Math.min(numTasksToShutDown, sortedOtherTasks.size())); }
Assert.assertEquals(1, deployProgress.getTargetActiveInstances()); Assert.assertEquals(1, deployProgress.getCurrentActiveInstances()); Assert.assertEquals(2, deployProgress.getTargetActiveInstances()); Assert.assertEquals(2, deployProgress.getCurrentActiveInstances()); Assert.assertEquals(3, deployProgress.getTargetActiveInstances()); Assert.assertEquals(3, deployProgress.getCurrentActiveInstances());
SingularityDeployProgress deployProgressStepOne = deployManager.getPendingDeploys().get(0).getDeployProgress().get(); Assert.assertTrue(deployProgressStepOne.isStepComplete()); Assert.assertEquals(2, deployProgressStepOne.getTargetActiveInstances()); Assert.assertEquals(4, deployProgressStepTwo.getTargetActiveInstances());
SingularityDeployProgress deployProgressStepOne = deployManager.getPendingDeploys().get(0).getDeployProgress().get(); Assert.assertTrue(deployProgressStepOne.isStepComplete()); Assert.assertEquals(1, deployProgressStepOne.getTargetActiveInstances());
SingularityDeployProgress deployProgressStepOne = pendingDeploy.getDeployProgress().get(); Assert.assertTrue(deployProgressStepOne.isStepComplete()); Assert.assertEquals(1, deployProgressStepOne.getTargetActiveInstances()); Assert.assertEquals(2, pendingDeploy.getDeployProgress().get().getTargetActiveInstances());
private List<SingularityDeployFailure> getDeployFailures(SingularityRequest request, Optional<SingularityDeploy> deploy, SingularityPendingDeploy pendingDeploy, DeployState state, Collection<SingularityTaskId> matchingTasks) { List<SingularityDeployFailure> failures = new ArrayList<>(); failures.addAll(deployHealthHelper.getTaskFailures(deploy, matchingTasks)); if (state == DeployState.OVERDUE) { int targetInstances = pendingDeploy.getDeployProgress().isPresent() ? pendingDeploy.getDeployProgress().get().getTargetActiveInstances() :request.getInstancesSafe(); if (failures.isEmpty() && matchingTasks.size() < targetInstances) { failures.add(new SingularityDeployFailure(SingularityDeployFailureReason.TASK_COULD_NOT_BE_SCHEDULED, Optional.<SingularityTaskId>absent(), Optional.of(String.format("Only %s of %s tasks could be launched for deploy, there may not be enough resources to launch the remaining tasks", matchingTasks.size(), targetInstances)))); } } return failures; } }
private int getNewTargetInstances(SingularityDeployProgress deployProgress, SingularityRequest request, Optional<SingularityUpdatePendingDeployRequest> updateRequest) { if (updateRequest.isPresent()) { return Math.min(updateRequest.get().getTargetActiveInstances(), request.getInstancesSafe()); } else { return Math.min(deployProgress.getTargetActiveInstances() + deployProgress.getDeployInstanceCountPerStep(), request.getInstancesSafe()); } }
private boolean isLastStepFinished(SingularityDeployProgress deployProgress, SingularityRequest request) { return deployProgress.isStepComplete() && deployProgress.getTargetActiveInstances() >= request.getInstancesSafe(); }
private LoadBalancerRequestId getLoadBalancerRequestId(SingularityPendingDeploy pendingDeploy) { return new LoadBalancerRequestId( String.format("%s-%s-%s", pendingDeploy.getDeployMarker().getRequestId(), pendingDeploy.getDeployMarker().getDeployId(), pendingDeploy.getDeployProgress().get().getTargetActiveInstances()), LoadBalancerRequestType.DEPLOY, Optional.<Integer> absent()); }