public static SingularityDeployKey valueOf(String string) { final String[] splits = JavaUtils.reverseSplit(string, 2, "-"); final String requestId = splits[0]; final String deployId = splits[1]; return new SingularityDeployKey(requestId, deployId); }
private Map<SingularityDeployKey, SingularityPendingTaskId> getDeployKeyToPendingTaskId() { final List<SingularityPendingTaskId> pendingTaskIds = taskManager.getPendingTaskIds(); final Map<SingularityDeployKey, SingularityPendingTaskId> deployKeyToPendingTaskId = Maps.newHashMapWithExpectedSize(pendingTaskIds.size()); for (SingularityPendingTaskId taskId : pendingTaskIds) { SingularityDeployKey deployKey = new SingularityDeployKey(taskId.getRequestId(), taskId.getDeployId()); deployKeyToPendingTaskId.put(deployKey, taskId); } return deployKeyToPendingTaskId; }
public SingularityRequestParent cancelPendingDeployForSingularityRequest(String requestId, String deployId) { final Function<String, String> requestUri = (host) -> String.format(DELETE_DEPLOY_FORMAT, getApiBase(host), deployId, requestId); SingularityRequestParent singularityRequestParent = delete(requestUri, "pending deploy", new SingularityDeployKey(requestId, deployId).getId(), Optional.absent(), Optional.of(SingularityRequestParent.class)).get(); return getAndLogRequestAndDeployStatus(singularityRequestParent); }
private String getPendingPath(String requestId, String deployId) { return ZKPaths.makePath(PENDING_PATH_ROOT, new SingularityDeployKey(requestId, deployId).getId()); }
public static SingularityDeployKey fromDeployMarker(SingularityDeployMarker deployMarker) { return new SingularityDeployKey(deployMarker.getRequestId(), deployMarker.getDeployId()); }
public static SingularityDeployKey fromTaskId(SingularityTaskId taskId) { return new SingularityDeployKey(taskId.getRequestId(), taskId.getDeployId()); }
public static SingularityDeployKey fromDeploy(SingularityDeploy deploy) { return new SingularityDeployKey(deploy.getRequestId(), deploy.getId()); }
@Timed public void drainPendingQueue() { final long start = System.currentTimeMillis(); final ImmutableList<SingularityPendingRequest> pendingRequests = ImmutableList.copyOf(requestManager.getPendingRequests()); if (pendingRequests.isEmpty()) { LOG.trace("Pending queue was empty"); return; } LOG.info("Pending queue had {} requests", pendingRequests.size()); Map<SingularityDeployKey, List<SingularityPendingRequest>> deployKeyToPendingRequests = pendingRequests.stream() .collect(Collectors.groupingBy((request) -> new SingularityDeployKey(request.getRequestId(), request.getDeployId()))); AtomicInteger totalNewScheduledTasks = new AtomicInteger(0); AtomicInteger heldForScheduledActiveTask = new AtomicInteger(0); AtomicInteger obsoleteRequests = new AtomicInteger(0); deployKeyToPendingRequests.forEach((deployKey, pendingRequestsForDeployKey) -> { lock.runWithRequestLock( () -> handlePendingRequestsForDeployKey(obsoleteRequests, heldForScheduledActiveTask, totalNewScheduledTasks, deployKey, pendingRequestsForDeployKey), deployKey.getRequestId(), String.format("%s#%s", getClass().getSimpleName(), "drainPendingQueue")); }); LOG.info("Scheduled {} new tasks ({} obsolete requests, {} held) in {}", totalNewScheduledTasks.get(), obsoleteRequests.get(), heldForScheduledActiveTask.get(), JavaUtils.duration(start)); }
public Optional<SingularityDeployHistory> getHistoryForRequestDeploy(String requestId, String deployId) { final Function<String, String> requestUri = (host) -> String.format(REQUEST_DEPLOY_HISTORY_FORMAT, getApiBase(host), requestId, deployId); return getSingle(requestUri, "deploy history", new SingularityDeployKey(requestId, deployId).getId(), SingularityDeployHistory.class); }
public SingularityRequestParent updateIncrementalDeployInstanceCount(SingularityUpdatePendingDeployRequest updateRequest) { final Function<String, String> requestUri = (host) -> String.format(UPDATE_DEPLOY_FORMAT, getApiBase(host)); HttpResponse response = post(requestUri, String.format("update deploy %s", new SingularityDeployKey(updateRequest.getRequestId(), updateRequest.getDeployId())), Optional.of(updateRequest)); return getAndLogRequestAndDeployStatus(response.getAs(SingularityRequestParent.class)); }
public SingularityRequestParent createDeployForSingularityRequest(String requestId, SingularityDeploy pendingDeploy, Optional<Boolean> deployUnpause, Optional<String> message, Optional<SingularityRequest> updatedRequest) { final Function<String, String> requestUri = (String host) -> String.format(DEPLOYS_FORMAT, getApiBase(host)); HttpResponse response = post(requestUri, String.format("new deploy %s", new SingularityDeployKey(requestId, pendingDeploy.getId())), Optional.of(new SingularityDeployRequest(pendingDeploy, deployUnpause, message, updatedRequest))); return getAndLogRequestAndDeployStatus(response.getAs(SingularityRequestParent.class)); }
private String getDeployParentPath(String requestId, String deployId) { return ZKPaths.makePath(getDeployIdPath(requestId), new SingularityDeployKey(requestId, deployId).getId()); }
SingularityDeployKey deployKey = new SingularityDeployKey(taskId.getRequestId(), taskId.getDeployId()); Optional<SingularityPendingDeploy> pendingDeploy = Optional.fromNullable(pendingDeploys.get(deployKey)); Optional<SingularityRequestWithState> request = Optional.fromNullable(idToRequest.get(taskId.getRequestId()));
private boolean shouldExitCooldown(SingularityRequestWithState cooldownRequest) { Optional<SingularityRequestDeployState> maybeDeployState = deployManager.getRequestDeployState(cooldownRequest.getRequest().getId()); if (!maybeDeployState.isPresent() || !maybeDeployState.get().getActiveDeploy().isPresent()) { LOG.trace("{} had no deployState / activeDeploy {}, exiting cooldown", cooldownRequest.getRequest().getId(), maybeDeployState); return true; } Optional<SingularityDeployStatistics> maybeDeployStatistics = deployManager.getDeployStatistics(cooldownRequest.getRequest().getId(), maybeDeployState.get().getActiveDeploy().get().getDeployId()); if (!maybeDeployStatistics.isPresent()) { LOG.trace("{} had no deploy statistics, exiting cooldown", new SingularityDeployKey(cooldownRequest.getRequest().getId(), maybeDeployState.get().getActiveDeploy().get().getDeployId())); return true; } Optional<Long> lastFinishAt = maybeDeployStatistics.get().getLastFinishAt(); if (!lastFinishAt.isPresent()) { LOG.trace("{} had no last finish, exiting cooldown", new SingularityDeployKey(cooldownRequest.getRequest().getId(), maybeDeployState.get().getActiveDeploy().get().getDeployId())); return true; } if (cooldown.hasCooldownExpired(cooldownRequest.getRequest(), maybeDeployStatistics.get(), Optional.<Integer> absent(), Optional.<Long> absent())) { return true; } return false; }
@GET @Path("/request/{requestId}/deploy/{deployId}/tasks/inactive") @Operation(summary = "Retrieve the task history for a specific deploy") public List<SingularityTaskIdHistory> getInactiveDeployTasks( @Parameter(hidden = true) @Auth SingularityUser user, @Parameter(required = true, description = "Request ID for deploy") @PathParam("requestId") String requestId, @Parameter(required = true, description = "Deploy ID") @PathParam("deployId") String deployId, @Parameter(description = "Maximum number of items to return") @QueryParam("count") Integer count, @Parameter(description = "Which page of items to view") @QueryParam("page") Integer page) { authorizationHelper.checkForAuthorizationByRequestId(requestId, user, SingularityAuthorizationScope.READ); final Integer limitCount = getLimitCount(count); final Integer limitStart = getLimitStart(limitCount, page); SingularityDeployKey key = new SingularityDeployKey(requestId, deployId); return deployTaskHistoryHelper.getBlendedHistory(key, limitStart, limitCount); }
private String pendingQueueKey(SingularityPendingRequest pendingRequest) { SingularityDeployKey deployKey = new SingularityDeployKey(pendingRequest.getRequestId(), pendingRequest.getDeployId()); if (pendingRequest.getPendingType() == PendingType.ONEOFF || pendingRequest.getPendingType() == PendingType.IMMEDIATE) { return String.format("%s%s%s", deployKey.toString(), pendingRequest.getTimestamp(), pendingRequest.getRunId().or("")); } else { return deployKey.toString(); } }
public static SingularityDeployKey fromPendingTask(SingularityPendingTask pendingTask) { return new SingularityDeployKey(pendingTask.getPendingTaskId().getRequestId(), pendingTask.getPendingTaskId().getDeployId()); }
@GET @Path("/request/{requestId}/deploy/{deployId}/tasks/inactive/withmetadata") @Operation(summary = "Retrieve the task history for a specific deploy") public SingularityPaginatedResponse<SingularityTaskIdHistory> getInactiveDeployTasksWithMetadata( @Parameter(hidden = true) @Auth SingularityUser user, @Parameter(required = true, description = "Request ID for deploy") @PathParam("requestId") String requestId, @Parameter(required = true, description = "Deploy ID") @PathParam("deployId") String deployId, @Parameter(description = "Maximum number of items to return") @QueryParam("count") Integer count, @Parameter(description = "Which page of items to view") @QueryParam("page") Integer page) { authorizationHelper.checkForAuthorizationByRequestId(requestId, user, SingularityAuthorizationScope.READ); SingularityDeployKey key = new SingularityDeployKey(requestId, deployId); Optional<Integer> dataCount = deployTaskHistoryHelper.getBlendedHistoryCount(key); final Integer limitCount = getLimitCount(count); final Integer limitStart = getLimitStart(limitCount, page); final List<SingularityTaskIdHistory> data = deployTaskHistoryHelper.getBlendedHistory(key, limitStart, limitCount); Optional<Integer> pageCount = getPageCount(dataCount, limitCount); return new SingularityPaginatedResponse<>(dataCount, pageCount, Optional.fromNullable(page), data); }
private void checkActiveRequest(SingularityRequestWithState requestWithState, Map<SingularityDeployKey, SingularityPendingTaskId> deployKeyToPendingTaskId, final long timestamp) { final SingularityRequest request = requestWithState.getRequest(); if (request.getRequestType() == RequestType.ON_DEMAND || request.getRequestType() == RequestType.RUN_ONCE) { return; // There's no situation where we'd want to schedule an On Demand or Run Once request at startup, so don't even bother with them. } Optional<SingularityRequestDeployState> requestDeployState = deployManager.getRequestDeployState(request.getId()); if (!requestDeployState.isPresent() || !requestDeployState.get().getActiveDeploy().isPresent()) { LOG.debug("No active deploy for {} - not scheduling on startup", request.getId()); return; } final String activeDeployId = requestDeployState.get().getActiveDeploy().get().getDeployId(); if (request.isScheduled()) { SingularityDeployKey deployKey = new SingularityDeployKey(request.getId(), activeDeployId); SingularityPendingTaskId pendingTaskId = deployKeyToPendingTaskId.get(deployKey); if (pendingTaskId != null && pendingTaskId.getCreatedAt() >= requestWithState.getTimestamp()) { LOG.info("Not rescheduling {} because {} is newer than {}", request.getId(), pendingTaskId, requestWithState.getTimestamp()); return; } } requestManager.addToPendingQueue(new SingularityPendingRequest(request.getId(), activeDeployId, timestamp, Optional.<String> absent(), PendingType.STARTUP, Optional.<Boolean> absent(), Optional.<String> absent())); }
private String getPendingPath(String requestId, String deployId) { return ZKPaths.makePath(SingularityCmdLineArgsMigration.REQUEST_PENDING_PATH, new SingularityDeployKey(requestId, deployId).getId()); }