private boolean saveNewDeployState(SingularityDeployMarker pendingDeployMarker, Optional<SingularityDeployMarker> newActiveDeploy) { Optional<SingularityRequestDeployState> deployState = deployManager.getRequestDeployState(pendingDeployMarker.getRequestId()); if (!deployState.isPresent()) { LOG.error("Expected deploy state for deploy marker: {} but didn't find it", pendingDeployMarker); return false; } deployManager.saveNewRequestDeployState(new SingularityRequestDeployState(deployState.get().getRequestId(), newActiveDeploy.or(deployState.get().getActiveDeploy()), Optional.absent())); return true; }
public Optional<String> getInUseDeployId(String requestId) { Optional<SingularityRequestDeployState> deployState = getRequestDeployState(requestId); if (!deployState.isPresent() || !deployState.get().getActiveDeploy().isPresent() && !deployState.get().getPendingDeploy().isPresent()) { return Optional.absent(); } return Optional.of(deployState.get().getActiveDeploy().or(deployState.get().getPendingDeploy()).get().getDeployId()); }
private List<String> getUnderProvisionedRequestIds(Set<String> possiblyUnderProvisionedRequestIds) { final List<String> underProvisionedRequestIds = new ArrayList<>(possiblyUnderProvisionedRequestIds.size()); if (!possiblyUnderProvisionedRequestIds.isEmpty()) { Map<String, SingularityRequestDeployState> deployStates = deployManager.getRequestDeployStatesByRequestIds(possiblyUnderProvisionedRequestIds); for (SingularityRequestDeployState deployState : deployStates.values()) { if (deployState.getActiveDeploy().isPresent() || deployState.getPendingDeploy().isPresent()) { underProvisionedRequestIds.add(deployState.getRequestId()); } } } return underProvisionedRequestIds; }
private boolean shouldExitCooldown(SingularityRequestWithState cooldownRequest) { Optional<SingularityRequestDeployState> maybeDeployState = deployManager.getRequestDeployState(cooldownRequest.getRequest().getId()); if (!maybeDeployState.isPresent() || !maybeDeployState.get().getActiveDeploy().isPresent()) { LOG.trace("{} had no deployState / activeDeploy {}, exiting cooldown", cooldownRequest.getRequest().getId(), maybeDeployState); return true; } Optional<SingularityDeployStatistics> maybeDeployStatistics = deployManager.getDeployStatistics(cooldownRequest.getRequest().getId(), maybeDeployState.get().getActiveDeploy().get().getDeployId()); if (!maybeDeployStatistics.isPresent()) { LOG.trace("{} had no deploy statistics, exiting cooldown", new SingularityDeployKey(cooldownRequest.getRequest().getId(), maybeDeployState.get().getActiveDeploy().get().getDeployId())); return true; } Optional<Long> lastFinishAt = maybeDeployStatistics.get().getLastFinishAt(); if (!lastFinishAt.isPresent()) { LOG.trace("{} had no last finish, exiting cooldown", new SingularityDeployKey(cooldownRequest.getRequest().getId(), maybeDeployState.get().getActiveDeploy().get().getDeployId())); return true; } if (cooldown.hasCooldownExpired(cooldownRequest.getRequest(), maybeDeployStatistics.get(), Optional.<Integer> absent(), Optional.<Long> absent())) { return true; } return false; }
public SingularityCreateResult saveDeploy(SingularityRequest request, SingularityDeployMarker deployMarker, SingularityDeploy deploy) { final SingularityCreateResult deploySaveResult = create(getDeployDataPath(deploy.getRequestId(), deploy.getId()), deploy, deployTranscoder); if (deploySaveResult == SingularityCreateResult.EXISTED) { LOG.info("Deploy object for {} already existed (new marker: {})", deploy, deployMarker); } singularityEventListener.deployHistoryEvent(new SingularityDeployUpdate(deployMarker, Optional.of(deploy), DeployEventType.STARTING, Optional.<SingularityDeployResult>absent())); create(getDeployMarkerPath(deploy.getRequestId(), deploy.getId()), deployMarker, deployMarkerTranscoder); final Optional<SingularityRequestDeployState> currentState = getRequestDeployState(deploy.getRequestId()); Optional<SingularityDeployMarker> activeDeploy = Optional.absent(); Optional<SingularityDeployMarker> pendingDeploy = Optional.absent(); if (request.isDeployable()) { if (currentState.isPresent()) { activeDeploy = currentState.get().getActiveDeploy(); } pendingDeploy = Optional.of(deployMarker); } else { activeDeploy = Optional.of(deployMarker); } final SingularityRequestDeployState newState = new SingularityRequestDeployState(deploy.getRequestId(), activeDeploy, pendingDeploy); return saveNewRequestDeployState(newState); }
@DELETE @Path("/deploy/{deployId}/request/{requestId}") @Operation( summary = "Cancel a pending deployment (best effort - the deploy may still succeed or fail)", responses = { @ApiResponse(responseCode = "400", description = "Deploy is not in the pending state pending or is not not present"), } ) public SingularityRequestParent cancelDeploy( @Parameter(hidden = true) @Auth SingularityUser user, @Parameter(required = true, description = "The Singularity Request Id from which the deployment is removed.") @PathParam("requestId") String requestId, @Parameter(required = true, description = "The Singularity Deploy Id that should be removed.") @PathParam("deployId") String deployId) { SingularityRequestWithState requestWithState = fetchRequestWithState(requestId, user); authorizationHelper.checkForAuthorization(requestWithState.getRequest(), user, SingularityAuthorizationScope.WRITE); validator.checkActionEnabled(SingularityAction.CANCEL_DEPLOY); Optional<SingularityRequestDeployState> deployState = deployManager.getRequestDeployState(requestWithState.getRequest().getId()); checkBadRequest(deployState.isPresent() && deployState.get().getPendingDeploy().isPresent() && deployState.get().getPendingDeploy().get().getDeployId().equals(deployId), "Request %s does not have a pending deploy %s", requestId, deployId); deployManager.createCancelDeployRequest(new SingularityDeployMarker(requestId, deployId, System.currentTimeMillis(), user.getEmail(), Optional.<String> absent())); return fillEntireRequest(requestWithState); }
protected void finishDeploy(SingularityDeployMarker marker, SingularityDeploy deploy) { deployManager.deletePendingDeploy(marker.getRequestId()); deployManager.saveDeployResult(marker, Optional.of(deploy), new SingularityDeployResult(DeployState.SUCCEEDED)); deployManager.saveNewRequestDeployState(new SingularityRequestDeployState(marker.getRequestId(), Optional.of(marker), Optional.<SingularityDeployMarker> absent())); }
@Override public String apply(SingularityRequestDeployState input) { return input.getRequestId(); }
private List<String> getUnderProvisionedRequestIds(Set<String> possiblyUnderProvisionedRequestIds) { final List<String> underProvisionedRequestIds = new ArrayList<>(possiblyUnderProvisionedRequestIds.size()); if (!possiblyUnderProvisionedRequestIds.isEmpty()) { Map<String, SingularityRequestDeployState> deployStates = deployManager.getRequestDeployStatesByRequestIds(possiblyUnderProvisionedRequestIds); for (SingularityRequestDeployState deployState : deployStates.values()) { if (deployState.getActiveDeploy().isPresent() || deployState.getPendingDeploy().isPresent()) { underProvisionedRequestIds.add(deployState.getRequestId()); } } } return underProvisionedRequestIds; }
private void checkActiveRequest(SingularityRequestWithState requestWithState, Map<SingularityDeployKey, SingularityPendingTaskId> deployKeyToPendingTaskId, final long timestamp) { final SingularityRequest request = requestWithState.getRequest(); if (request.getRequestType() == RequestType.ON_DEMAND || request.getRequestType() == RequestType.RUN_ONCE) { return; // There's no situation where we'd want to schedule an On Demand or Run Once request at startup, so don't even bother with them. } Optional<SingularityRequestDeployState> requestDeployState = deployManager.getRequestDeployState(request.getId()); if (!requestDeployState.isPresent() || !requestDeployState.get().getActiveDeploy().isPresent()) { LOG.debug("No active deploy for {} - not scheduling on startup", request.getId()); return; } final String activeDeployId = requestDeployState.get().getActiveDeploy().get().getDeployId(); if (request.isScheduled()) { SingularityDeployKey deployKey = new SingularityDeployKey(request.getId(), activeDeployId); SingularityPendingTaskId pendingTaskId = deployKeyToPendingTaskId.get(deployKey); if (pendingTaskId != null && pendingTaskId.getCreatedAt() >= requestWithState.getTimestamp()) { LOG.info("Not rescheduling {} because {} is newer than {}", request.getId(), pendingTaskId, requestWithState.getTimestamp()); return; } } requestManager.addToPendingQueue(new SingularityPendingRequest(request.getId(), activeDeployId, timestamp, Optional.<String> absent(), PendingType.STARTUP, Optional.<Boolean> absent(), Optional.<String> absent())); }
public SingularityCreateResult saveDeploy(SingularityRequest request, SingularityDeployMarker deployMarker, SingularityDeploy deploy) { final SingularityCreateResult deploySaveResult = create(getDeployDataPath(deploy.getRequestId(), deploy.getId()), deploy, deployTranscoder); if (deploySaveResult == SingularityCreateResult.EXISTED) { LOG.info("Deploy object for {} already existed (new marker: {})", deploy, deployMarker); } singularityEventListener.deployHistoryEvent(new SingularityDeployUpdate(deployMarker, Optional.of(deploy), DeployEventType.STARTING, Optional.<SingularityDeployResult>absent())); create(getDeployMarkerPath(deploy.getRequestId(), deploy.getId()), deployMarker, deployMarkerTranscoder); final Optional<SingularityRequestDeployState> currentState = getRequestDeployState(deploy.getRequestId()); Optional<SingularityDeployMarker> activeDeploy = Optional.absent(); Optional<SingularityDeployMarker> pendingDeploy = Optional.absent(); if (request.isDeployable()) { if (currentState.isPresent()) { activeDeploy = currentState.get().getActiveDeploy(); } pendingDeploy = Optional.of(deployMarker); } else { activeDeploy = Optional.of(deployMarker); } final SingularityRequestDeployState newState = new SingularityRequestDeployState(deploy.getRequestId(), activeDeploy, pendingDeploy); return saveNewRequestDeployState(newState); }
@POST @Path("/update") @Operation( summary = "Update the target active instance count for a pending deploy", responses = { @ApiResponse(responseCode = "400", description = "Deploy is not in the pending state pending or is not not present") } ) public SingularityRequestParent updatePendingDeploy( @Parameter(hidden = true) @Auth SingularityUser user, @RequestBody(required = true) SingularityUpdatePendingDeployRequest updateRequest) { SingularityRequestWithState requestWithState = fetchRequestWithState(updateRequest.getRequestId(), user); authorizationHelper.checkForAuthorization(requestWithState.getRequest(), user, SingularityAuthorizationScope.WRITE); Optional<SingularityRequestDeployState> deployState = deployManager.getRequestDeployState(requestWithState.getRequest().getId()); checkBadRequest(deployState.isPresent() && deployState.get().getPendingDeploy().isPresent() && deployState.get().getPendingDeploy().get().getDeployId().equals(updateRequest.getDeployId()), "Request %s does not have a pending deploy %s", updateRequest.getRequestId(), updateRequest.getDeployId()); checkBadRequest(updateRequest.getTargetActiveInstances() > 0 && updateRequest.getTargetActiveInstances() <= requestWithState.getRequest().getInstancesSafe(), "Cannot update pending deploy to have more instances (%s) than instances set for request (%s), or less than 1 instance", updateRequest.getTargetActiveInstances(), requestWithState.getRequest().getInstancesSafe()); deployManager.createUpdatePendingDeployRequest(updateRequest); return fillEntireRequest(requestWithState); } }
protected void finishDeploy(SingularityDeployMarker marker, SingularityDeploy deploy) { deployManager.deletePendingDeploy(marker.getRequestId()); deployManager.saveDeployResult(marker, Optional.of(deploy), new SingularityDeployResult(DeployState.SUCCEEDED)); deployManager.saveNewRequestDeployState(new SingularityRequestDeployState(marker.getRequestId(), Optional.of(marker), Optional.<SingularityDeployMarker> absent())); }
public void putRequestDeployState(SingularityRequestDeployState requestDeployState) { if (!active) { LOG.warn("putRequestDeployState {}, but not active", requestDeployState.getRequestId()); return; } requestIdToDeployState.put(requestDeployState.getRequestId(), requestDeployState); }
private boolean shouldTransferDeploy(SingularityRequestDeployState deployState, SingularityDeployKey deployKey) { if (deployState == null) { LOG.warn("Missing request deploy state for deployKey {}", deployKey); return true; } if (deployState.getActiveDeploy().isPresent() && deployState.getActiveDeploy().get().getDeployId().equals(deployKey.getDeployId())) { return false; } if (deployState.getPendingDeploy().isPresent() && deployState.getPendingDeploy().get().getDeployId().equals(deployKey.getDeployId())) { return false; } return true; }
private boolean saveNewDeployState(SingularityDeployMarker pendingDeployMarker, Optional<SingularityDeployMarker> newActiveDeploy) { Optional<SingularityRequestDeployState> deployState = deployManager.getRequestDeployState(pendingDeployMarker.getRequestId()); if (!deployState.isPresent()) { LOG.error("Expected deploy state for deploy marker: {} but didn't find it", pendingDeployMarker); return false; } deployManager.saveNewRequestDeployState(new SingularityRequestDeployState(deployState.get().getRequestId(), newActiveDeploy.or(deployState.get().getActiveDeploy()), Optional.absent())); return true; }
SingularityPendingRequest pendingRequest = new SingularityPendingRequest(request.getId(), requestDeployState.get().getActiveDeploy().get().getDeployId(), System.currentTimeMillis(), Optional.absent(), pendingType, cmdLineArgsList, Optional.absent(), Optional.absent(), Optional.absent(), Optional.absent(), resources, Collections.emptyList(), Optional.absent(), Collections.emptyMap(), Collections.emptyList(), Optional.absent());
@DELETE @Path("/deploy/{deployId}/request/{requestId}") @Operation( summary = "Cancel a pending deployment (best effort - the deploy may still succeed or fail)", responses = { @ApiResponse(responseCode = "400", description = "Deploy is not in the pending state pending or is not not present"), } ) public SingularityRequestParent cancelDeploy( @Parameter(hidden = true) @Auth SingularityUser user, @Parameter(required = true, description = "The Singularity Request Id from which the deployment is removed.") @PathParam("requestId") String requestId, @Parameter(required = true, description = "The Singularity Deploy Id that should be removed.") @PathParam("deployId") String deployId) { SingularityRequestWithState requestWithState = fetchRequestWithState(requestId, user); authorizationHelper.checkForAuthorization(requestWithState.getRequest(), user, SingularityAuthorizationScope.WRITE); validator.checkActionEnabled(SingularityAction.CANCEL_DEPLOY); Optional<SingularityRequestDeployState> deployState = deployManager.getRequestDeployState(requestWithState.getRequest().getId()); checkBadRequest(deployState.isPresent() && deployState.get().getPendingDeploy().isPresent() && deployState.get().getPendingDeploy().get().getDeployId().equals(deployId), "Request %s does not have a pending deploy %s", requestId, deployId); deployManager.createCancelDeployRequest(new SingularityDeployMarker(requestId, deployId, System.currentTimeMillis(), user.getEmail(), Optional.<String> absent())); return fillEntireRequest(requestWithState); }
public SingularityCreateResult saveNewRequestDeployState(SingularityRequestDeployState newDeployState) { if (leaderCache.active()) { leaderCache.putRequestDeployState(newDeployState); } return save(getRequestDeployStatePath(newDeployState.getRequestId()), newDeployState, requestDeployStateTranscoder); }
private boolean isDeployInUse(Optional<SingularityRequestDeployState> requestDeployState, String deployId, boolean mustMatchActiveDeploy) { if (!requestDeployState.isPresent()) { return false; } if (matchesDeployMarker(requestDeployState.get().getActiveDeploy(), deployId)) { return true; } if (mustMatchActiveDeploy) { return false; } return matchesDeployMarker(requestDeployState.get().getPendingDeploy(), deployId); }