public SingularityDeployProgress withNewTargetInstances(int instances) { return new SingularityDeployProgress(instances, currentActiveInstances, deployInstanceCountPerStep, deployStepWaitTimeMs, false, autoAdvanceDeploySteps, failedDeployTasks, System.currentTimeMillis()); }
private boolean isLastStepFinished(SingularityDeployProgress deployProgress, SingularityRequest request) { return deployProgress.isStepComplete() && deployProgress.getTargetActiveInstances() >= request.getInstancesSafe(); }
private int numInstancesExpected(SingularityRequest request, SingularityPendingRequest pendingRequest, Optional<SingularityPendingDeploy> maybePendingDeploy) { if (!maybePendingDeploy.isPresent() || (maybePendingDeploy.get().getCurrentDeployState() == DeployState.CANCELED) || !maybePendingDeploy.get().getDeployProgress().isPresent()) { return request.getInstancesSafe(); } SingularityDeployProgress deployProgress = maybePendingDeploy.get().getDeployProgress().get(); if (maybePendingDeploy.get().getDeployMarker().getDeployId().equals(pendingRequest.getDeployId())) { return deployProgress.getTargetActiveInstances(); } else { if (deployProgress.isStepComplete()) { return Math.max(request.getInstancesSafe() - deployProgress.getTargetActiveInstances(), 0); } else { return request.getInstancesSafe() - (Math.max(deployProgress.getTargetActiveInstances() - deployProgress.getDeployInstanceCountPerStep(), 0)); } } }
private boolean canMoveToNextStep(SingularityDeployProgress deployProgress) { return deployProgress.isAutoAdvanceDeploySteps() && deployProgress.getTimestamp() + deployProgress.getDeployStepWaitTimeMs() < System.currentTimeMillis(); }
private int getNewTargetInstances(SingularityDeployProgress deployProgress, SingularityRequest request, Optional<SingularityUpdatePendingDeployRequest> updateRequest) { if (updateRequest.isPresent()) { return Math.min(updateRequest.get().getTargetActiveInstances(), request.getInstancesSafe()); } else { return Math.min(deployProgress.getTargetActiveInstances() + deployProgress.getDeployInstanceCountPerStep(), request.getInstancesSafe()); } }
private List<SingularityTaskId> tasksToShutDown(SingularityDeployProgress deployProgress, Collection<SingularityTaskId> otherActiveTasks, SingularityRequest request) { int numTasksToShutDown = Math.max(otherActiveTasks.size() - (request.getInstancesSafe() - deployProgress.getTargetActiveInstances()), 0); List<SingularityTaskId> sortedOtherTasks = new ArrayList<>(otherActiveTasks); Collections.sort(sortedOtherTasks, SingularityTaskId.INSTANCE_NO_COMPARATOR); return sortedOtherTasks.isEmpty() ? sortedOtherTasks : sortedOtherTasks.subList(0, Math.min(numTasksToShutDown, sortedOtherTasks.size())); }
private SingularityDeployResult markStepFinished(SingularityPendingDeploy pendingDeploy, Optional<SingularityDeploy> deploy, Collection<SingularityTaskId> deployActiveTasks, Collection<SingularityTaskId> otherActiveTasks, SingularityRequest request, Optional<SingularityUpdatePendingDeployRequest> updatePendingDeployRequest) { SingularityDeployProgress deployProgress = pendingDeploy.getDeployProgress().get(); if (updatePendingDeployRequest.isPresent() && getNewTargetInstances(deployProgress, request, updatePendingDeployRequest) != deployProgress.getTargetActiveInstances()) { maybeUpdatePendingRequest(pendingDeploy, deploy, request, updatePendingDeployRequest); return new SingularityDeployResult(DeployState.WAITING); } SingularityDeployProgress newProgress = deployProgress.withNewActiveInstances(deployActiveTasks.size()).withCompletedStep(); DeployState deployState = isLastStepFinished(newProgress, request) ? DeployState.SUCCEEDED : DeployState.WAITING; String message = deployState == DeployState.SUCCEEDED ? "New deploy succeeded" : "New deploy is progressing, this task is being replaced"; updatePendingDeploy(pendingDeploy, pendingDeploy.getLastLoadBalancerUpdate(), deployState, Optional.of(newProgress)); for (SingularityTaskId taskId : tasksToShutDown(deployProgress, otherActiveTasks, request)) { taskManager.createTaskCleanup( new SingularityTaskCleanup(Optional.<String> absent(), TaskCleanupType.DEPLOY_STEP_FINISHED, System.currentTimeMillis(), taskId, Optional.of(message), Optional.<String> absent(), Optional.<SingularityTaskShellCommandRequestId>absent())); } return new SingularityDeployResult(deployState); }
Assert.assertEquals(1, deployProgress.getTargetActiveInstances()); Assert.assertEquals(1, deployProgress.getCurrentActiveInstances()); Assert.assertEquals(2, deployProgress.getTargetActiveInstances()); Assert.assertEquals(2, deployProgress.getCurrentActiveInstances()); Assert.assertEquals(3, deployProgress.getTargetActiveInstances()); Assert.assertEquals(3, deployProgress.getCurrentActiveInstances());
if (pendingDeploy.getDeployProgress().isPresent() && !pendingDeploy.getDeployProgress().get().isStepComplete()) { long deployStepDelta = now - pendingDeploy.getDeployProgress().get().getTimestamp(); if (deployStepDelta > oldestDeployStep) { oldestDeployStep = deployStepDelta;
private boolean isDeployOverdue(SingularityPendingDeploy pendingDeploy, Optional<SingularityDeploy> deploy) { if (!deploy.isPresent()) { LOG.warn("Can't determine if deploy {} is overdue because it was missing", pendingDeploy); return false; } if (pendingDeploy.getDeployProgress().isPresent() && pendingDeploy.getDeployProgress().get().isStepComplete()) { return false; } final long startTime = getStartTime(pendingDeploy); final long deployDuration = System.currentTimeMillis() - startTime; final long allowedTime = getAllowedMillis(deploy.get()); if (deployDuration > allowedTime) { LOG.warn("Deploy {} is overdue (duration: {}), allowed: {}", pendingDeploy, DurationFormatUtils.formatDurationHMS(deployDuration), DurationFormatUtils.formatDurationHMS(allowedTime)); return true; } else { LOG.trace("Deploy {} is not yet overdue (duration: {}), allowed: {}", pendingDeploy, DurationFormatUtils.formatDurationHMS(deployDuration), DurationFormatUtils.formatDurationHMS(allowedTime)); return false; } }
private long getStartTime(SingularityPendingDeploy pendingDeploy) { if (pendingDeploy.getDeployProgress().isPresent()) { return pendingDeploy.getDeployProgress().get().getTimestamp(); } else { return pendingDeploy.getDeployMarker().getTimestamp(); } }
private TaskCleanupType getCleanupType(SingularityPendingDeploy pendingDeploy, SingularityRequest request, SingularityDeployResult deployResult) { if (pendingDeploy.getDeployProgress().isPresent() && pendingDeploy.getDeployProgress().get().getDeployInstanceCountPerStep() != request.getInstancesSafe()) { // For incremental deploys, return a special cleanup type if (deployResult.getDeployState() == DeployState.FAILED) { return TaskCleanupType.INCREMENTAL_DEPLOY_FAILED; } else if (deployResult.getDeployState() == DeployState.CANCELED) { return TaskCleanupType.INCREMENTAL_DEPLOY_CANCELLED; } } return deployResult.getDeployState().getCleanupType(); }
private Set<SingularityTaskId> getNewInactiveDeployTasks(SingularityPendingDeploy pendingDeploy, Collection<SingularityTaskId> inactiveDeployMatchingTasks) { Set<SingularityTaskId> newInactiveDeployTasks = new HashSet<>(); newInactiveDeployTasks.addAll(inactiveDeployMatchingTasks); if (pendingDeploy.getDeployProgress().isPresent()) { newInactiveDeployTasks.removeAll(pendingDeploy.getDeployProgress().get().getFailedDeployTasks()); } return newInactiveDeployTasks; }
private List<SingularityDeployFailure> getDeployFailures(SingularityRequest request, Optional<SingularityDeploy> deploy, SingularityPendingDeploy pendingDeploy, DeployState state, Collection<SingularityTaskId> matchingTasks) { List<SingularityDeployFailure> failures = new ArrayList<>(); failures.addAll(deployHealthHelper.getTaskFailures(deploy, matchingTasks)); if (state == DeployState.OVERDUE) { int targetInstances = pendingDeploy.getDeployProgress().isPresent() ? pendingDeploy.getDeployProgress().get().getTargetActiveInstances() :request.getInstancesSafe(); if (failures.isEmpty() && matchingTasks.size() < targetInstances) { failures.add(new SingularityDeployFailure(SingularityDeployFailureReason.TASK_COULD_NOT_BE_SCHEDULED, Optional.<SingularityTaskId>absent(), Optional.of(String.format("Only %s of %s tasks could be launched for deploy, there may not be enough resources to launch the remaining tasks", matchingTasks.size(), targetInstances)))); } } return failures; } }
private SingularityDeployResult markStepFinished(SingularityPendingDeploy pendingDeploy, Optional<SingularityDeploy> deploy, Collection<SingularityTaskId> deployActiveTasks, Collection<SingularityTaskId> otherActiveTasks, SingularityRequest request, Optional<SingularityUpdatePendingDeployRequest> updatePendingDeployRequest) { SingularityDeployProgress deployProgress = pendingDeploy.getDeployProgress().get(); if (updatePendingDeployRequest.isPresent() && getNewTargetInstances(deployProgress, request, updatePendingDeployRequest) != deployProgress.getTargetActiveInstances()) { maybeUpdatePendingRequest(pendingDeploy, deploy, request, updatePendingDeployRequest); return new SingularityDeployResult(DeployState.WAITING); } SingularityDeployProgress newProgress = deployProgress.withNewActiveInstances(deployActiveTasks.size()).withCompletedStep(); DeployState deployState = isLastStepFinished(newProgress, request) ? DeployState.SUCCEEDED : DeployState.WAITING; String message = deployState == DeployState.SUCCEEDED ? "New deploy succeeded" : "New deploy is progressing, this task is being replaced"; updatePendingDeploy(pendingDeploy, pendingDeploy.getLastLoadBalancerUpdate(), deployState, Optional.of(newProgress)); for (SingularityTaskId taskId : tasksToShutDown(deployProgress, otherActiveTasks, request)) { taskManager.createTaskCleanup( new SingularityTaskCleanup(Optional.<String> absent(), TaskCleanupType.DEPLOY_STEP_FINISHED, System.currentTimeMillis(), taskId, Optional.of(message), Optional.<String> absent(), Optional.<SingularityTaskShellCommandRequestId>absent())); } return new SingularityDeployResult(deployState); }
private int getNewTargetInstances(SingularityDeployProgress deployProgress, SingularityRequest request, Optional<SingularityUpdatePendingDeployRequest> updateRequest) { if (updateRequest.isPresent()) { return Math.min(updateRequest.get().getTargetActiveInstances(), request.getInstancesSafe()); } else { return Math.min(deployProgress.getTargetActiveInstances() + deployProgress.getDeployInstanceCountPerStep(), request.getInstancesSafe()); } }
private boolean canMoveToNextStep(SingularityDeployProgress deployProgress) { return deployProgress.isAutoAdvanceDeploySteps() && deployProgress.getTimestamp() + deployProgress.getDeployStepWaitTimeMs() < System.currentTimeMillis(); }
Assert.assertEquals(1, deployProgress.getTargetActiveInstances()); Assert.assertEquals(1, deployProgress.getCurrentActiveInstances()); Assert.assertEquals(2, deployProgress.getTargetActiveInstances()); Assert.assertEquals(2, deployProgress.getCurrentActiveInstances()); Assert.assertEquals(3, deployProgress.getTargetActiveInstances()); Assert.assertEquals(3, deployProgress.getCurrentActiveInstances());
if (pendingDeploy.getDeployProgress().isPresent() && !pendingDeploy.getDeployProgress().get().isStepComplete()) { long deployStepDelta = now - pendingDeploy.getDeployProgress().get().getTimestamp(); if (deployStepDelta > oldestDeployStep) { oldestDeployStep = deployStepDelta;
private boolean isDeployOverdue(SingularityPendingDeploy pendingDeploy, Optional<SingularityDeploy> deploy) { if (!deploy.isPresent()) { LOG.warn("Can't determine if deploy {} is overdue because it was missing", pendingDeploy); return false; } if (pendingDeploy.getDeployProgress().isPresent() && pendingDeploy.getDeployProgress().get().isStepComplete()) { return false; } final long startTime = getStartTime(pendingDeploy); final long deployDuration = System.currentTimeMillis() - startTime; final long allowedTime = getAllowedMillis(deploy.get()); if (deployDuration > allowedTime) { LOG.warn("Deploy {} is overdue (duration: {}), allowed: {}", pendingDeploy, DurationFormatUtils.formatDurationHMS(deployDuration), DurationFormatUtils.formatDurationHMS(allowedTime)); return true; } else { LOG.trace("Deploy {} is not yet overdue (duration: {}), allowed: {}", pendingDeploy, DurationFormatUtils.formatDurationHMS(deployDuration), DurationFormatUtils.formatDurationHMS(allowedTime)); return false; } }