private boolean shouldCancelLoadBalancer(final SingularityPendingDeploy pendingDeploy) { return pendingDeploy.getLastLoadBalancerUpdate().isPresent() && !pendingDeploy.getCurrentDeployState().isDeployFinished(); }
private int numInstancesExpected(SingularityRequest request, SingularityPendingRequest pendingRequest, Optional<SingularityPendingDeploy> maybePendingDeploy) { if (!maybePendingDeploy.isPresent() || (maybePendingDeploy.get().getCurrentDeployState() == DeployState.CANCELED) || !maybePendingDeploy.get().getDeployProgress().isPresent()) { return request.getInstancesSafe(); } SingularityDeployProgress deployProgress = maybePendingDeploy.get().getDeployProgress().get(); if (maybePendingDeploy.get().getDeployMarker().getDeployId().equals(pendingRequest.getDeployId())) { return deployProgress.getTargetActiveInstances(); } else { if (deployProgress.isStepComplete()) { return Math.max(request.getInstancesSafe() - deployProgress.getTargetActiveInstances(), 0); } else { return request.getInstancesSafe() - (Math.max(deployProgress.getTargetActiveInstances() - deployProgress.getDeployInstanceCountPerStep(), 0)); } } }
private SingularityDeployResult processLbState(SingularityRequest request, Optional<SingularityDeploy> deploy, SingularityPendingDeploy pendingDeploy, Optional<SingularityUpdatePendingDeployRequest> updatePendingDeployRequest, Collection<SingularityTaskId> deployActiveTasks, Collection<SingularityTaskId> otherActiveTasks, Collection<SingularityTaskId> tasksToShutDown, SingularityLoadBalancerUpdate lbUpdate) { List<SingularityTaskId> toRemoveFromLb = new ArrayList<>(); for (SingularityTaskId taskId : tasksToShutDown) { Optional<SingularityLoadBalancerUpdate> maybeRemoveUpdate = taskManager.getLoadBalancerState(taskId, LoadBalancerRequestType.REMOVE); if (maybeRemoveUpdate.isPresent() && maybeRemoveUpdate.get().getLoadBalancerRequestId().getId().equals(lbUpdate.getLoadBalancerRequestId().getId())) { toRemoveFromLb.add(taskId); } } updateLoadBalancerStateForTasks(deployActiveTasks, LoadBalancerRequestType.ADD, lbUpdate); updateLoadBalancerStateForTasks(toRemoveFromLb, LoadBalancerRequestType.REMOVE, lbUpdate); DeployState deployState = interpretLoadBalancerState(lbUpdate, pendingDeploy.getCurrentDeployState()); if (deployState == DeployState.SUCCEEDED) { updatePendingDeploy(pendingDeploy, Optional.of(lbUpdate), DeployState.WAITING); // A step has completed, markStepFinished will determine SUCCEEDED/WAITING return markStepFinished(pendingDeploy, deploy, deployActiveTasks, otherActiveTasks, request, updatePendingDeployRequest); } else if (deployState == DeployState.WAITING) { updatePendingDeploy(pendingDeploy, Optional.of(lbUpdate), deployState); maybeUpdatePendingRequest(pendingDeploy, deploy, request, updatePendingDeployRequest, Optional.of(lbUpdate)); return new SingularityDeployResult(DeployState.WAITING); } else { updatePendingDeploy(pendingDeploy, Optional.of(lbUpdate), deployState); maybeUpdatePendingRequest(pendingDeploy, deploy, request, updatePendingDeployRequest, Optional.of(lbUpdate)); return new SingularityDeployResult(deployState, lbUpdate, SingularityDeployFailure.lbUpdateFailed()); } }
@Test public void testRemovedRequestData() { long now = System.currentTimeMillis(); initRequest(); SingularityDeployBuilder db = new SingularityDeployBuilder(requestId, firstDeployId); db.setMaxTaskRetries(Optional.of(1)); initDeploy(db, now); deployChecker.checkDeploys(); Assert.assertEquals(DeployState.WAITING, deployManager.getPendingDeploys().get(0).getCurrentDeployState()); requestManager.startDeletingRequest(request, Optional.absent(), Optional.<String>absent(), Optional.<String>absent(), Optional.<String>absent()); requestManager.markDeleted(request, now, Optional.<String>absent(), Optional.<String>absent()); deployChecker.checkDeploys(); SingularityDeployResult deployResult = deployManager.getDeployResult(requestId, firstDeployId).get(); Assert.assertEquals(DeployState.FAILED, deployResult.getDeployState()); Assert.assertTrue(deployResult.getMessage().get().contains("MISSING")); }
@Test public void testDeployFailsAfterMaxTaskRetries() { initRequest(); SingularityDeployBuilder db = new SingularityDeployBuilder(requestId, firstDeployId); db.setMaxTaskRetries(Optional.of(1)); SingularityDeploy deploy = initDeploy(db, System.currentTimeMillis()); deployChecker.checkDeploys(); Assert.assertTrue(!deployManager.getDeployResult(requestId, firstDeployId).isPresent()); SingularityTask task = launchTask(request, deploy, System.currentTimeMillis(), 1, TaskState.TASK_FAILED); deployChecker.checkDeploys(); Assert.assertEquals(deployManager.getPendingDeploys().get(0).getCurrentDeployState(), DeployState.WAITING); SingularityTask taskTryTwo = launchTask(request, deploy, System.currentTimeMillis(), 1, TaskState.TASK_FAILED); deployChecker.checkDeploys(); Assert.assertEquals(deployManager.getDeployResult(requestId, firstDeployId).get().getDeployState(), DeployState.FAILED); }
@Test public void testDeploySucceedsWithTaskRetries() { initRequest(); SingularityDeployBuilder db = new SingularityDeployBuilder(requestId, firstDeployId); db.setMaxTaskRetries(Optional.of(1)); SingularityDeploy deploy = initDeploy(db, System.currentTimeMillis()); deployChecker.checkDeploys(); Assert.assertTrue(!deployManager.getDeployResult(requestId, firstDeployId).isPresent()); SingularityTask task = launchTask(request, deploy, System.currentTimeMillis(), 1, TaskState.TASK_FAILED); deployChecker.checkDeploys(); Assert.assertEquals(deployManager.getPendingDeploys().get(0).getCurrentDeployState(), DeployState.WAITING); SingularityTask taskTryTwo = launchTask(request, deploy, System.currentTimeMillis(), 1, TaskState.TASK_RUNNING); deployChecker.checkDeploys(); Assert.assertEquals(deployManager.getDeployResult(requestId, firstDeployId).get().getDeployState(), DeployState.SUCCEEDED); }
deployChecker.checkDeploys(); SingularityPendingDeploy pendingDeploy = deployManager.getPendingDeploy(requestId).get(); Assert.assertEquals(DeployState.WAITING, pendingDeploy.getCurrentDeployState()); Assert.assertEquals(DeployState.WAITING, pendingDeploy.getCurrentDeployState()); Assert.assertEquals(DeployState.WAITING, pendingDeploy.getCurrentDeployState());
private boolean shouldCancelLoadBalancer(final SingularityPendingDeploy pendingDeploy) { return pendingDeploy.getLastLoadBalancerUpdate().isPresent() && !pendingDeploy.getCurrentDeployState().isDeployFinished(); }
private int numInstancesExpected(SingularityRequest request, SingularityPendingRequest pendingRequest, Optional<SingularityPendingDeploy> maybePendingDeploy) { if (!maybePendingDeploy.isPresent() || (maybePendingDeploy.get().getCurrentDeployState() == DeployState.CANCELED) || !maybePendingDeploy.get().getDeployProgress().isPresent()) { return request.getInstancesSafe(); } SingularityDeployProgress deployProgress = maybePendingDeploy.get().getDeployProgress().get(); if (maybePendingDeploy.get().getDeployMarker().getDeployId().equals(pendingRequest.getDeployId())) { return deployProgress.getTargetActiveInstances(); } else { if (deployProgress.isStepComplete()) { return Math.max(request.getInstancesSafe() - deployProgress.getTargetActiveInstances(), 0); } else { return request.getInstancesSafe() - (Math.max(deployProgress.getTargetActiveInstances() - deployProgress.getDeployInstanceCountPerStep(), 0)); } } }
private SingularityDeployResult processLbState(SingularityRequest request, Optional<SingularityDeploy> deploy, SingularityPendingDeploy pendingDeploy, Optional<SingularityUpdatePendingDeployRequest> updatePendingDeployRequest, Collection<SingularityTaskId> deployActiveTasks, Collection<SingularityTaskId> otherActiveTasks, Collection<SingularityTaskId> tasksToShutDown, SingularityLoadBalancerUpdate lbUpdate) { List<SingularityTaskId> toRemoveFromLb = new ArrayList<>(); for (SingularityTaskId taskId : tasksToShutDown) { Optional<SingularityLoadBalancerUpdate> maybeRemoveUpdate = taskManager.getLoadBalancerState(taskId, LoadBalancerRequestType.REMOVE); if (maybeRemoveUpdate.isPresent() && maybeRemoveUpdate.get().getLoadBalancerRequestId().getId().equals(lbUpdate.getLoadBalancerRequestId().getId())) { toRemoveFromLb.add(taskId); } } updateLoadBalancerStateForTasks(deployActiveTasks, LoadBalancerRequestType.ADD, lbUpdate); updateLoadBalancerStateForTasks(toRemoveFromLb, LoadBalancerRequestType.REMOVE, lbUpdate); DeployState deployState = interpretLoadBalancerState(lbUpdate, pendingDeploy.getCurrentDeployState()); if (deployState == DeployState.SUCCEEDED) { updatePendingDeploy(pendingDeploy, Optional.of(lbUpdate), DeployState.WAITING); // A step has completed, markStepFinished will determine SUCCEEDED/WAITING return markStepFinished(pendingDeploy, deploy, deployActiveTasks, otherActiveTasks, request, updatePendingDeployRequest); } else if (deployState == DeployState.WAITING) { updatePendingDeploy(pendingDeploy, Optional.of(lbUpdate), deployState); maybeUpdatePendingRequest(pendingDeploy, deploy, request, updatePendingDeployRequest, Optional.of(lbUpdate)); return new SingularityDeployResult(DeployState.WAITING); } else { updatePendingDeploy(pendingDeploy, Optional.of(lbUpdate), deployState); maybeUpdatePendingRequest(pendingDeploy, deploy, request, updatePendingDeployRequest, Optional.of(lbUpdate)); return new SingularityDeployResult(deployState, lbUpdate, SingularityDeployFailure.lbUpdateFailed()); } }
@Test public void testRemovedRequestData() { long now = System.currentTimeMillis(); initRequest(); SingularityDeployBuilder db = new SingularityDeployBuilder(requestId, firstDeployId); db.setMaxTaskRetries(Optional.of(1)); initDeploy(db, now); deployChecker.checkDeploys(); Assert.assertEquals(DeployState.WAITING, deployManager.getPendingDeploys().get(0).getCurrentDeployState()); requestManager.startDeletingRequest(request, Optional.absent(), Optional.<String>absent(), Optional.<String>absent(), Optional.<String>absent()); requestManager.markDeleted(request, now, Optional.<String>absent(), Optional.<String>absent()); deployChecker.checkDeploys(); SingularityDeployResult deployResult = deployManager.getDeployResult(requestId, firstDeployId).get(); Assert.assertEquals(DeployState.FAILED, deployResult.getDeployState()); Assert.assertTrue(deployResult.getMessage().get().contains("MISSING")); }
@Test public void testDeploySucceedsWithTaskRetries() { initRequest(); SingularityDeployBuilder db = new SingularityDeployBuilder(requestId, firstDeployId); db.setMaxTaskRetries(Optional.of(1)); SingularityDeploy deploy = initDeploy(db, System.currentTimeMillis()); deployChecker.checkDeploys(); Assert.assertTrue(!deployManager.getDeployResult(requestId, firstDeployId).isPresent()); SingularityTask task = launchTask(request, deploy, System.currentTimeMillis(), 1, TaskState.TASK_FAILED); deployChecker.checkDeploys(); Assert.assertEquals(deployManager.getPendingDeploys().get(0).getCurrentDeployState(), DeployState.WAITING); SingularityTask taskTryTwo = launchTask(request, deploy, System.currentTimeMillis(), 1, TaskState.TASK_RUNNING); deployChecker.checkDeploys(); Assert.assertEquals(deployManager.getDeployResult(requestId, firstDeployId).get().getDeployState(), DeployState.SUCCEEDED); }
@Test public void testDeployFailsAfterMaxTaskRetries() { initRequest(); SingularityDeployBuilder db = new SingularityDeployBuilder(requestId, firstDeployId); db.setMaxTaskRetries(Optional.of(1)); SingularityDeploy deploy = initDeploy(db, System.currentTimeMillis()); deployChecker.checkDeploys(); Assert.assertTrue(!deployManager.getDeployResult(requestId, firstDeployId).isPresent()); SingularityTask task = launchTask(request, deploy, System.currentTimeMillis(), 1, TaskState.TASK_FAILED); deployChecker.checkDeploys(); Assert.assertEquals(deployManager.getPendingDeploys().get(0).getCurrentDeployState(), DeployState.WAITING); SingularityTask taskTryTwo = launchTask(request, deploy, System.currentTimeMillis(), 1, TaskState.TASK_FAILED); deployChecker.checkDeploys(); Assert.assertEquals(deployManager.getDeployResult(requestId, firstDeployId).get().getDeployState(), DeployState.FAILED); }
deployChecker.checkDeploys(); SingularityPendingDeploy pendingDeploy = deployManager.getPendingDeploy(requestId).get(); Assert.assertEquals(DeployState.WAITING, pendingDeploy.getCurrentDeployState()); Assert.assertEquals(DeployState.WAITING, pendingDeploy.getCurrentDeployState()); Assert.assertEquals(DeployState.WAITING, pendingDeploy.getCurrentDeployState());