private DeployState interpretLoadBalancerState(SingularityLoadBalancerUpdate lbUpdate, DeployState unknownState) { switch (lbUpdate.getLoadBalancerState()) { case CANCELED: return DeployState.CANCELED; case SUCCESS: return DeployState.SUCCEEDED; case FAILED: case INVALID_REQUEST_NOOP: return DeployState.FAILED; case CANCELING: return DeployState.CANCELING; case UNKNOWN: return unknownState; case WAITING: } return DeployState.WAITING; }
private boolean shouldRemoveLbState(SingularityTaskId taskId, SingularityLoadBalancerUpdate loadBalancerUpdate) { switch (loadBalancerUpdate.getLoadBalancerState()) { case UNKNOWN: case WAITING: case SUCCESS: return true; case INVALID_REQUEST_NOOP: return false; // don't need to remove because Baragon doesnt know about it default: LOG.trace("Task {} had abnormal LB state {}", taskId, loadBalancerUpdate); return false; } }
private boolean shouldEnqueueLbRequest(Optional<SingularityLoadBalancerUpdate> maybeLbUpdate) { if (!maybeLbUpdate.isPresent()) { return true; } switch (maybeLbUpdate.get().getLoadBalancerState()) { case UNKNOWN: case FAILED: case CANCELED: return true; case CANCELING: case SUCCESS: case WAITING: case INVALID_REQUEST_NOOP: } return false; }
public SingularityDeployResult(DeployState deployState, SingularityLoadBalancerUpdate lbUpdate, List<SingularityDeployFailure> deployFailures) { this(deployState, Optional.of(String.format("Load balancer had state %s%s", lbUpdate.getLoadBalancerState(), lbUpdate.getMessage().isPresent() && lbUpdate.getMessage().get().length() > 0 ? String.format(" (%s)", lbUpdate.getMessage().get()) : "")), Optional.of(lbUpdate), deployFailures, System.currentTimeMillis()); }
private LoadBalancerRequestId getLoadBalancerRequestId(String requestId, Optional<SingularityLoadBalancerUpdate> lbDeleteUpdate) { if (!lbDeleteUpdate.isPresent()) { return new LoadBalancerRequestId(String.format("%s-%s", requestId, System.currentTimeMillis()), LoadBalancerRequestType.DELETE, Optional.absent()); } switch (lbDeleteUpdate.get().getLoadBalancerState()) { case FAILED: case CANCELED: return new LoadBalancerRequestId(String.format("%s-%s", requestId, System.currentTimeMillis()), LoadBalancerRequestType.DELETE, Optional.of(lbDeleteUpdate.get().getLoadBalancerRequestId().getAttemptNumber() + 1)); default: return lbDeleteUpdate.get().getLoadBalancerRequestId(); } }
private boolean unknownNotRemoving(SingularityLoadBalancerUpdate update) { return update.getLoadBalancerState() == BaragonRequestState.UNKNOWN && update.getLoadBalancerRequestId().getRequestType() != LoadBalancerRequestType.REMOVE; } }
private DeployHealth getLbHealth(SingularityRequest request, SingularityTaskId taskId) { if (!request.isLoadBalanced()) { return DeployHealth.HEALTHY; } Optional<SingularityLoadBalancerUpdate> update = taskManager.getLoadBalancerState(taskId, LoadBalancerRequestType.ADD); if (!update.isPresent()) { return DeployHealth.WAITING; } switch (update.get().getLoadBalancerState()) { case SUCCESS: return DeployHealth.HEALTHY; case CANCELED: case CANCELING: case UNKNOWN: case INVALID_REQUEST_NOOP: case FAILED: return DeployHealth.UNHEALTHY; case WAITING: return DeployHealth.WAITING; } return DeployHealth.WAITING; }
private LoadBalancerRequestId getLoadBalancerRequestId(SingularityTaskId taskId, Optional<SingularityLoadBalancerUpdate> lbRemoveUpdate) { if (!lbRemoveUpdate.isPresent()) { return new LoadBalancerRequestId(taskId.getId(), LoadBalancerRequestType.REMOVE, Optional.absent()); } switch (lbRemoveUpdate.get().getLoadBalancerState()) { case FAILED: case CANCELED: return new LoadBalancerRequestId(taskId.getId(), LoadBalancerRequestType.REMOVE, Optional.of(lbRemoveUpdate.get().getLoadBalancerRequestId().getAttemptNumber() + 1)); default: return lbRemoveUpdate.get().getLoadBalancerRequestId(); } }
private boolean shouldCheckLbState(final SingularityPendingDeploy pendingDeploy) { return pendingDeploy.getLastLoadBalancerUpdate().isPresent() && getLoadBalancerRequestId(pendingDeploy).getId().equals(pendingDeploy.getLastLoadBalancerUpdate().get().getLoadBalancerRequestId().getId()) && (pendingDeploy.getLastLoadBalancerUpdate().get().getLoadBalancerState() != BaragonRequestState.UNKNOWN); }
private boolean isWaitingForCurrentLbRequest(SingularityPendingDeploy pendingDeploy) { return pendingDeploy.getLastLoadBalancerUpdate().isPresent() && getLoadBalancerRequestId(pendingDeploy).getId().equals(pendingDeploy.getLastLoadBalancerUpdate().get().getLoadBalancerRequestId().getId()) && pendingDeploy.getLastLoadBalancerUpdate().get().getLoadBalancerState() == BaragonRequestState.WAITING; }
cleanup.setLoadBalancerUpdate(Optional.of(lbDeleteUpdate)); requestManager.saveLbCleanupRequest(cleanup); } else if (maybeDeleteUpdate.get().getLoadBalancerState() == BaragonRequestState.WAITING || maybeDeleteUpdate.get().getLoadBalancerState() == BaragonRequestState.CANCELING) { lbDeleteUpdate = lbClient.getState(loadBalancerRequestId); cleanup.setLoadBalancerUpdate(Optional.of(lbDeleteUpdate)); switch (lbDeleteUpdate.getLoadBalancerState()) { case SUCCESS: return CheckLBState.DONE; case FAILED: case CANCELED: LOG.error("LB delete request {} ({}) got unexpected response {}", lbDeleteUpdate, loadBalancerRequestId, lbDeleteUpdate.getLoadBalancerState()); exceptionNotifier.notify(String.format("LB delete failed for %s", lbDeleteUpdate.getLoadBalancerRequestId().toString()), ImmutableMap.of("state", lbDeleteUpdate.getLoadBalancerState().name(), "loadBalancerRequestId", loadBalancerRequestId.toString(), "addUpdate", lbDeleteUpdate.toString())); return CheckLBState.RETRY; case UNKNOWN: case CANCELING: case WAITING: LOG.trace("Waiting on LB delete request {} in state {}", loadBalancerRequestId, lbDeleteUpdate.getLoadBalancerState()); break; case INVALID_REQUEST_NOOP: exceptionNotifier.notify(String.format("LB delete failed for %s", lbDeleteUpdate.getLoadBalancerRequestId().toString()), ImmutableMap.of("state", lbDeleteUpdate.getLoadBalancerState().name(), "loadBalancerRequestId", loadBalancerRequestId.toString(), "addUpdate", lbDeleteUpdate.toString())); return CheckLBState.LOAD_BALANCE_FAILED;
private SingularityDeployResult enqueueAndProcessLbRequest(SingularityRequest request, Optional<SingularityDeploy> deploy, SingularityPendingDeploy pendingDeploy, Optional<SingularityUpdatePendingDeployRequest> updatePendingDeployRequest, Collection<SingularityTaskId> deployActiveTasks, Collection<SingularityTaskId> otherActiveTasks) { Collection<SingularityTaskId> toShutDown = tasksToShutDown(pendingDeploy.getDeployProgress().get(), otherActiveTasks, request); final Map<SingularityTaskId, SingularityTask> tasks = taskManager.getTasks(Iterables.concat(deployActiveTasks, toShutDown)); final LoadBalancerRequestId lbRequestId = getLoadBalancerRequestId(pendingDeploy); List<SingularityTaskId> toRemoveFromLb = new ArrayList<>(); for (SingularityTaskId taskId : toShutDown) { Optional<SingularityLoadBalancerUpdate> maybeAddUpdate = taskManager.getLoadBalancerState(taskId, LoadBalancerRequestType.ADD); if (maybeAddUpdate.isPresent() && maybeAddUpdate.get().getLoadBalancerState() == BaragonRequestState.SUCCESS) { toRemoveFromLb.add(taskId); } } updateLoadBalancerStateForTasks(deployActiveTasks, LoadBalancerRequestType.ADD, SingularityLoadBalancerUpdate.preEnqueue(lbRequestId)); updateLoadBalancerStateForTasks(toRemoveFromLb, LoadBalancerRequestType.REMOVE, SingularityLoadBalancerUpdate.preEnqueue(lbRequestId)); SingularityLoadBalancerUpdate enqueueResult = lbClient.enqueue(lbRequestId, request, deploy.get(), getTasks(deployActiveTasks, tasks), getTasks(toShutDown, tasks)); return processLbState(request, deploy, pendingDeploy, updatePendingDeployRequest, deployActiveTasks, otherActiveTasks, toShutDown, enqueueResult); }
} else if (maybeLbRemoveUpdate.get().getLoadBalancerState() == BaragonRequestState.WAITING || maybeLbRemoveUpdate.get().getLoadBalancerState() == BaragonRequestState.CANCELING) { lbRemoveUpdate = lbClient.getState(loadBalancerRequestId); switch (lbRemoveUpdate.getLoadBalancerState()) { case SUCCESS: if (configuration.getLoadBalancerRemovalGracePeriodMillis() > 0) { case FAILED: case CANCELED: LOG.error("LB removal request {} ({}) got unexpected response {}", lbRemoveUpdate, loadBalancerRequestId, lbRemoveUpdate.getLoadBalancerState()); exceptionNotifier.notify("LB removal failed", ImmutableMap.of("state", lbRemoveUpdate.getLoadBalancerState().name(), "loadBalancerRequestId", loadBalancerRequestId.toString(), "addUpdate", lbRemoveUpdate.toString())); return CheckLBState.RETRY; case UNKNOWN: case CANCELING: case WAITING: LOG.trace("Waiting on LB cleanup request {} in state {}", loadBalancerRequestId, lbRemoveUpdate.getLoadBalancerState()); break; case INVALID_REQUEST_NOOP: exceptionNotifier.notify("LB removal failed", ImmutableMap.of("state", lbRemoveUpdate.getLoadBalancerState().name(), "loadBalancerRequestId", loadBalancerRequestId.toString(), "addUpdate", lbRemoveUpdate.toString())); return CheckLBState.LOAD_BALANCE_FAILED;
Optional<CheckTaskState> maybeCheckTaskState = checkLbState(lbUpdate.get().getLoadBalancerState()); Optional<CheckTaskState> maybeCheckTaskState = checkLbState(newLbUpdate.getLoadBalancerState());
@Test public void testPauseLbCleanup() { initLoadBalancedRequest(); initFirstDeploy(); requestManager.saveLbCleanupRequest(new SingularityRequestLbCleanup(requestId, Sets.newHashSet("test"), "/basepath", Collections.<String>emptyList(), Optional.<SingularityLoadBalancerUpdate>absent())); requestManager.pause(request, System.currentTimeMillis(), Optional.<String>absent(), Optional.<String>absent()); testingLbClient.setNextBaragonRequestState(BaragonRequestState.WAITING); cleaner.drainCleanupQueue(); Assert.assertTrue(!requestManager.getLbCleanupRequestIds().isEmpty()); Optional<SingularityLoadBalancerUpdate> lbUpdate = requestManager.getLbCleanupRequest(requestId).get().getLoadBalancerUpdate(); Assert.assertTrue(lbUpdate.isPresent()); Assert.assertTrue(lbUpdate.get().getLoadBalancerState() == BaragonRequestState.WAITING); testingLbClient.setNextBaragonRequestState(BaragonRequestState.FAILED); cleaner.drainCleanupQueue(); Assert.assertTrue(!requestManager.getLbCleanupRequestIds().isEmpty()); lbUpdate = requestManager.getLbCleanupRequest(requestId).get().getLoadBalancerUpdate(); Assert.assertTrue(lbUpdate.isPresent()); Assert.assertTrue(lbUpdate.get().getLoadBalancerState() == BaragonRequestState.FAILED); testingLbClient.setNextBaragonRequestState(BaragonRequestState.SUCCESS); cleaner.drainCleanupQueue(); Assert.assertTrue(requestManager.getLbCleanupRequestIds().isEmpty()); }
Assert.assertTrue(lbUpdate.get().getLoadBalancerState() == BaragonRequestState.WAITING); Assert.assertTrue(lbUpdate.get().getLoadBalancerState() == BaragonRequestState.FAILED); Assert.assertTrue(lbUpdate.get().getLoadBalancerState() == BaragonRequestState.SUCCESS); Assert.assertTrue(lbUpdate.get().getLoadBalancerRequestId().getAttemptNumber() == 2);
private LoadBalancerRequestId getLoadBalancerRequestId(String requestId, Optional<SingularityLoadBalancerUpdate> lbDeleteUpdate) { if (!lbDeleteUpdate.isPresent()) { return new LoadBalancerRequestId(String.format("%s-%s", requestId, System.currentTimeMillis()), LoadBalancerRequestType.DELETE, Optional.absent()); } switch (lbDeleteUpdate.get().getLoadBalancerState()) { case FAILED: case CANCELED: return new LoadBalancerRequestId(String.format("%s-%s", requestId, System.currentTimeMillis()), LoadBalancerRequestType.DELETE, Optional.of(lbDeleteUpdate.get().getLoadBalancerRequestId().getAttemptNumber() + 1)); default: return lbDeleteUpdate.get().getLoadBalancerRequestId(); } }
private boolean unknownNotRemoving(SingularityLoadBalancerUpdate update) { return update.getLoadBalancerState() == BaragonRequestState.UNKNOWN && update.getLoadBalancerRequestId().getRequestType() != LoadBalancerRequestType.REMOVE; } }
private boolean isWaitingForCurrentLbRequest(SingularityPendingDeploy pendingDeploy) { return pendingDeploy.getLastLoadBalancerUpdate().isPresent() && getLoadBalancerRequestId(pendingDeploy).getId().equals(pendingDeploy.getLastLoadBalancerUpdate().get().getLoadBalancerRequestId().getId()) && pendingDeploy.getLastLoadBalancerUpdate().get().getLoadBalancerState() == BaragonRequestState.WAITING; }
private boolean shouldCheckLbState(final SingularityPendingDeploy pendingDeploy) { return pendingDeploy.getLastLoadBalancerUpdate().isPresent() && getLoadBalancerRequestId(pendingDeploy).getId().equals(pendingDeploy.getLastLoadBalancerUpdate().get().getLoadBalancerRequestId().getId()) && (pendingDeploy.getLastLoadBalancerUpdate().get().getLoadBalancerState() != BaragonRequestState.UNKNOWN); }