@Override public String apply(@Nonnull SingularityPendingDeploy input) { return input.getDeployMarker().getRequestId(); } };
@Override public SingularityDeployKey apply(@Nonnull SingularityPendingDeploy input) { return SingularityDeployKey.fromDeployMarker(input.getDeployMarker()); } };
private Optional<SingularityUpdatePendingDeployRequest> findUpdateRequest(List<SingularityUpdatePendingDeployRequest> updateRequests, SingularityPendingDeploy pendingDeploy) { for (SingularityUpdatePendingDeployRequest updateRequest : updateRequests) { if (updateRequest.getRequestId().equals(pendingDeploy.getDeployMarker().getRequestId()) && updateRequest.getDeployId().equals(pendingDeploy.getDeployMarker().getDeployId())) { return Optional.of(updateRequest); } } return Optional.absent(); }
private boolean isPartOfPendingDeploy(List<SingularityPendingDeploy> pendingDeploys, SingularityTaskId taskId) { for (SingularityPendingDeploy pendingDeploy : pendingDeploys) { if (pendingDeploy.getDeployMarker().getDeployId().equals(taskId.getDeployId()) && pendingDeploy.getDeployMarker().getRequestId().equals(taskId.getRequestId())) { return true; } } return false; }
private void failPendingDeployDueToState(SingularityPendingDeploy pendingDeploy, Optional<SingularityRequestWithState> maybeRequestWithState, Optional<SingularityDeploy> deploy) { SingularityDeployResult deployResult = new SingularityDeployResult(DeployState.FAILED, Optional.of(String.format("Request in state %s is not deployable", SingularityRequestWithState.getRequestState(maybeRequestWithState))), Optional.<SingularityLoadBalancerUpdate>absent()); if (!maybeRequestWithState.isPresent()) { deployManager.saveDeployResult(pendingDeploy.getDeployMarker(), deploy, deployResult); removePendingDeploy(pendingDeploy); return; } saveNewDeployState(pendingDeploy.getDeployMarker(), Optional.<SingularityDeployMarker> absent()); finishDeploy(maybeRequestWithState.get(), deploy, pendingDeploy, Collections.<SingularityTaskId>emptyList(), deployResult); }
private LoadBalancerRequestId getLoadBalancerRequestId(SingularityPendingDeploy pendingDeploy) { return new LoadBalancerRequestId( String.format("%s-%s-%s", pendingDeploy.getDeployMarker().getRequestId(), pendingDeploy.getDeployMarker().getDeployId(), pendingDeploy.getDeployProgress().get().getTargetActiveInstances()), LoadBalancerRequestType.DEPLOY, Optional.<Integer> absent()); }
private void deleteObsoletePendingTasks(SingularityPendingDeploy pendingDeploy) { List<SingularityPendingTaskId> obsoletePendingTasks = taskManager.getPendingTaskIds() .stream() .filter(taskId -> taskId.getRequestId().equals(pendingDeploy.getDeployMarker().getRequestId())) .filter(taskId -> !taskId.getDeployId().equals(pendingDeploy.getDeployMarker().getDeployId())) .collect(Collectors.toList()); for (SingularityPendingTaskId pendingTaskId : obsoletePendingTasks) { LOG.debug("Deleting obsolete pending task {}", pendingTaskId.getId()); taskManager.deletePendingTask(pendingTaskId); } }
private SingularityRequest updatedRequest(Optional<SingularityPendingDeploy> maybePendingDeploy, SingularityPendingRequest pendingRequest, SingularityRequestWithState currentRequest) { if (maybePendingDeploy.isPresent() && pendingRequest.getDeployId().equals(maybePendingDeploy.get().getDeployMarker().getDeployId())) { return maybePendingDeploy.get().getUpdatedRequest().or(currentRequest.getRequest()); } else { return currentRequest.getRequest(); } } }
private void maybeUpdatePendingRequest(SingularityPendingDeploy pendingDeploy, Optional<SingularityDeploy> deploy, SingularityRequest request, Optional<SingularityUpdatePendingDeployRequest> updatePendingDeployRequest, Optional<SingularityLoadBalancerUpdate> lbUpdate) { if (updatePendingDeployRequest.isPresent() && pendingDeploy.getDeployProgress().isPresent()) { SingularityDeployProgress newProgress = pendingDeploy.getDeployProgress().get().withNewTargetInstances(Math.min(updatePendingDeployRequest.get().getTargetActiveInstances(), request.getInstancesSafe())); updatePendingDeploy(pendingDeploy, lbUpdate.or(pendingDeploy.getLastLoadBalancerUpdate()), DeployState.WAITING, Optional.of(newProgress)); requestManager .addToPendingQueue(new SingularityPendingRequest(request.getId(), pendingDeploy.getDeployMarker().getDeployId(), System.currentTimeMillis(), pendingDeploy.getDeployMarker().getUser(), PendingType.NEXT_DEPLOY_STEP, deploy.isPresent() ? deploy.get().getSkipHealthchecksOnDeploy() : Optional.<Boolean> absent(), pendingDeploy.getDeployMarker().getMessage())); } }
private void removePendingDeploy(SingularityPendingDeploy pendingDeploy) { deployManager.deletePendingDeploy(pendingDeploy.getDeployMarker().getRequestId()); }
private void cleanupTasks(SingularityPendingDeploy pendingDeploy, SingularityRequest request, SingularityDeployResult deployResult, Iterable<SingularityTaskId> tasksToKill) { for (SingularityTaskId matchingTask : tasksToKill) { taskManager.saveTaskCleanup(new SingularityTaskCleanup(pendingDeploy.getDeployMarker().getUser(), getCleanupType(pendingDeploy, request, deployResult), deployResult.getTimestamp(), matchingTask, Optional.of(String.format("Deploy %s - %s", pendingDeploy.getDeployMarker().getDeployId(), deployResult.getDeployState().name())), Optional.<String> absent(), Optional.<SingularityTaskShellCommandRequestId>absent())); } }
private long getStartTime(SingularityPendingDeploy pendingDeploy) { if (pendingDeploy.getDeployProgress().isPresent()) { return pendingDeploy.getDeployProgress().get().getTimestamp(); } else { return pendingDeploy.getDeployMarker().getTimestamp(); } }
private SingularityDeployResult checkCanMoveToNextDeployStep(SingularityRequest request, Optional<SingularityDeploy> deploy, SingularityPendingDeploy pendingDeploy, Optional<SingularityUpdatePendingDeployRequest> updatePendingDeployRequest) { SingularityDeployProgress deployProgress = pendingDeploy.getDeployProgress().get(); if (canMoveToNextStep(deployProgress) || updatePendingDeployRequest.isPresent()) { SingularityDeployProgress newProgress = deployProgress.withNewTargetInstances(getNewTargetInstances(deployProgress, request, updatePendingDeployRequest)); updatePendingDeploy(pendingDeploy, pendingDeploy.getLastLoadBalancerUpdate(), DeployState.WAITING, Optional.of(newProgress)); requestManager.addToPendingQueue( new SingularityPendingRequest(request.getId(), pendingDeploy.getDeployMarker().getDeployId(), System.currentTimeMillis(), pendingDeploy.getDeployMarker().getUser(), PendingType.NEXT_DEPLOY_STEP, deploy.isPresent() ? deploy.get().getSkipHealthchecksOnDeploy() : Optional.<Boolean> absent(), pendingDeploy.getDeployMarker().getMessage())); } return new SingularityDeployResult(DeployState.WAITING); }
public int checkDeploys() { final List<SingularityPendingDeploy> pendingDeploys = deployManager.getPendingDeploys(); final List<SingularityDeployMarker> cancelDeploys = deployManager.getCancelDeploys(); final List<SingularityUpdatePendingDeployRequest> updateRequests = deployManager.getPendingDeployUpdates(); if (pendingDeploys.isEmpty() && cancelDeploys.isEmpty()) { return 0; } final Map<SingularityPendingDeploy, SingularityDeployKey> pendingDeployToKey = SingularityDeployKey.fromPendingDeploys(pendingDeploys); final Map<SingularityDeployKey, SingularityDeploy> deployKeyToDeploy = deployManager.getDeploysForKeys(pendingDeployToKey.values()); pendingDeploys.parallelStream().forEach((pendingDeploy) -> { lock.runWithRequestLock(() -> { checkDeploy(pendingDeploy, cancelDeploys, pendingDeployToKey, deployKeyToDeploy, updateRequests); }, pendingDeploy.getDeployMarker().getRequestId(), getClass().getSimpleName()); }); cancelDeploys.forEach(deployManager::deleteCancelDeployRequest); updateRequests.forEach(deployManager::deleteUpdatePendingDeployRequest); return pendingDeploys.size(); }
private Optional<SingularityTaskIdsByStatus> getTaskIdsByStatusForRequest(SingularityRequestWithState requestWithState) { String requestId = requestWithState.getRequest().getId(); Optional<SingularityPendingDeploy> pendingDeploy = deployManager.getPendingDeploy(requestId); List<SingularityTaskId> cleaningTaskIds = taskManager.getCleanupTaskIds().stream().filter((t) -> t.getRequestId().equals(requestId)).collect(Collectors.toList()); List<SingularityPendingTaskId> pendingTaskIds = taskManager.getPendingTaskIdsForRequest(requestId); List<SingularityTaskId> activeTaskIds = taskManager.getActiveTaskIdsForRequest(requestId); activeTaskIds.removeAll(cleaningTaskIds); List<SingularityTaskId> healthyTaskIds = new ArrayList<>(); List<SingularityTaskId> notYetHealthyTaskIds = new ArrayList<>(); Map<String, List<SingularityTaskId>> taskIdsByDeployId = activeTaskIds.stream().collect(Collectors.groupingBy(SingularityTaskId::getDeployId)); for (Map.Entry<String, List<SingularityTaskId>> entry : taskIdsByDeployId.entrySet()) { Optional<SingularityDeploy> deploy = deployManager.getDeploy(requestId, entry.getKey()); List<SingularityTaskId> healthyTasksIdsForDeploy = deployHealthHelper.getHealthyTasks( requestWithState.getRequest(), deploy, entry.getValue(), pendingDeploy.isPresent() && pendingDeploy.get().getDeployMarker().getDeployId().equals(entry.getKey())); for (SingularityTaskId taskId : entry.getValue()) { if (healthyTasksIdsForDeploy.contains(taskId)) { healthyTaskIds.add(taskId); } else { notYetHealthyTaskIds.add(taskId); } } } return Optional.of(new SingularityTaskIdsByStatus(healthyTaskIds, notYetHealthyTaskIds, pendingTaskIds, cleaningTaskIds)); }
private void updatePendingDeploy(SingularityPendingDeploy pendingDeploy, Optional<SingularityLoadBalancerUpdate> lbUpdate, DeployState deployState, Optional<SingularityDeployProgress> deployProgress) { SingularityPendingDeploy copy = new SingularityPendingDeploy(pendingDeploy.getDeployMarker(), lbUpdate, deployState, deployProgress, pendingDeploy.getUpdatedRequest()); deployManager.savePendingDeploy(copy); }
public SingularityCreateResult savePendingDeploy(SingularityPendingDeploy pendingDeploy) { return save(getPendingDeployPath(pendingDeploy.getDeployMarker().getRequestId()), pendingDeploy, pendingDeployTranscoder); }
public SingularityCreateResult createPendingDeploy(SingularityPendingDeploy pendingDeploy) { return create(getPendingDeployPath(pendingDeploy.getDeployMarker().getRequestId()), pendingDeploy, pendingDeployTranscoder); }
private int numInstancesExpected(SingularityRequest request, SingularityPendingRequest pendingRequest, Optional<SingularityPendingDeploy> maybePendingDeploy) { if (!maybePendingDeploy.isPresent() || (maybePendingDeploy.get().getCurrentDeployState() == DeployState.CANCELED) || !maybePendingDeploy.get().getDeployProgress().isPresent()) { return request.getInstancesSafe(); } SingularityDeployProgress deployProgress = maybePendingDeploy.get().getDeployProgress().get(); if (maybePendingDeploy.get().getDeployMarker().getDeployId().equals(pendingRequest.getDeployId())) { return deployProgress.getTargetActiveInstances(); } else { if (deployProgress.isStepComplete()) { return Math.max(request.getInstancesSafe() - deployProgress.getTargetActiveInstances(), 0); } else { return request.getInstancesSafe() - (Math.max(deployProgress.getTargetActiveInstances() - deployProgress.getDeployInstanceCountPerStep(), 0)); } } }
private boolean shouldHealthcheck(final SingularityTask task, final Optional<SingularityRequestWithState> request, Optional<SingularityPendingDeploy> pendingDeploy) { if (disasterManager.isDisabled(SingularityAction.RUN_HEALTH_CHECKS)) { return false; } if (!task.getTaskRequest().getRequest().isLongRunning() || !task.getTaskRequest().getDeploy().getHealthcheck().isPresent() || task.getTaskRequest().getDeploy().getHealthcheck().get().getHealthcheckResultFilePath().isPresent()) { return false; } if (task.getTaskRequest().getPendingTask().getSkipHealthchecks().or(false)) { return false; } if (pendingDeploy.isPresent() && pendingDeploy.get().getDeployMarker().getDeployId().equals(task.getTaskId().getDeployId()) && task.getTaskRequest().getDeploy().getSkipHealthchecksOnDeploy().or(false)) { return false; } if (request.isPresent() && request.get().getRequest().getSkipHealthchecks().or(false)) { return false; } Optional<SingularityTaskHealthcheckResult> lastHealthcheck = taskManager.getLastHealthcheck(task.getTaskId()); if (lastHealthcheck.isPresent() && !lastHealthcheck.get().isFailed()) { LOG.debug("Not submitting a new healthcheck for {} because it already passed a healthcheck", task.getTaskId()); return false; } return true; }