private SingularityRequest updatedRequest(Optional<SingularityPendingDeploy> maybePendingDeploy, SingularityPendingRequest pendingRequest, SingularityRequestWithState currentRequest) { if (maybePendingDeploy.isPresent() && pendingRequest.getDeployId().equals(maybePendingDeploy.get().getDeployMarker().getDeployId())) { return maybePendingDeploy.get().getUpdatedRequest().or(currentRequest.getRequest()); } else { return currentRequest.getRequest(); } } }
private void schedule(int numMissingInstances, List<SingularityTaskId> matchingTaskIds, SingularityRequest request, RequestState state, SingularityDeployStatistics deployStatistics, SingularityPendingRequest pendingRequest, Optional<SingularityPendingDeploy> maybePendingDeploy) { final List<SingularityPendingTask> scheduledTasks = getScheduledTaskIds(numMissingInstances, matchingTaskIds, request, state, deployStatistics, pendingRequest.getDeployId(), pendingRequest, maybePendingDeploy); if (!scheduledTasks.isEmpty()) { LOG.trace("Scheduling tasks: {}", scheduledTasks); for (SingularityPendingTask scheduledTask : scheduledTasks) { taskManager.savePendingTask(scheduledTask); } } else { LOG.info("No new scheduled tasks found for {}, setting state to {}", request.getId(), RequestState.FINISHED); requestManager.finish(request, System.currentTimeMillis()); } }
@Timed public void drainPendingQueue() { final long start = System.currentTimeMillis(); final ImmutableList<SingularityPendingRequest> pendingRequests = ImmutableList.copyOf(requestManager.getPendingRequests()); if (pendingRequests.isEmpty()) { LOG.trace("Pending queue was empty"); return; } LOG.info("Pending queue had {} requests", pendingRequests.size()); Map<SingularityDeployKey, List<SingularityPendingRequest>> deployKeyToPendingRequests = pendingRequests.stream() .collect(Collectors.groupingBy((request) -> new SingularityDeployKey(request.getRequestId(), request.getDeployId()))); AtomicInteger totalNewScheduledTasks = new AtomicInteger(0); AtomicInteger heldForScheduledActiveTask = new AtomicInteger(0); AtomicInteger obsoleteRequests = new AtomicInteger(0); deployKeyToPendingRequests.forEach((deployKey, pendingRequestsForDeployKey) -> { lock.runWithRequestLock( () -> handlePendingRequestsForDeployKey(obsoleteRequests, heldForScheduledActiveTask, totalNewScheduledTasks, deployKey, pendingRequestsForDeployKey), deployKey.getRequestId(), String.format("%s#%s", getClass().getSimpleName(), "drainPendingQueue")); }); LOG.info("Scheduled {} new tasks ({} obsolete requests, {} held) in {}", totalNewScheduledTasks.get(), obsoleteRequests.get(), heldForScheduledActiveTask.get(), JavaUtils.duration(start)); }
private void deleteScheduledTasks(final Collection<SingularityPendingTask> scheduledTasks, SingularityPendingRequest pendingRequest) { List<SingularityPendingTask> tasksForDeploy = scheduledTasks .stream() .filter(task -> pendingRequest.getRequestId().equals(task.getPendingTaskId().getRequestId())) .filter(task -> pendingRequest.getDeployId().equals(task.getPendingTaskId().getDeployId())) .collect(Collectors.toList()); for (SingularityPendingTask task : tasksForDeploy) { LOG.debug("Deleting pending task {} in order to reschedule {}", task.getPendingTaskId().getId(), pendingRequest); taskManager.deletePendingTask(task.getPendingTaskId()); } }
public SingularityCreateResult addToPendingQueue(SingularityPendingRequest pendingRequest) { SingularityCreateResult result = create(getPendingPath(pendingRequest), pendingRequest, pendingRequestTranscoder); if (result == SingularityCreateResult.EXISTED) { Optional<SingularityPendingRequest> existingPendingRequest = getPendingRequest(pendingRequest.getRequestId(), pendingRequest.getDeployId()); if (!existingPendingRequest.isPresent() || (existingPendingRequest.get().getPendingType() == PendingType.STARTUP)) { // Fresh pending requests take priority over STARTUP-type pending requests (or if the pending request we originally found is gone, we'll go ahead and do the write now) set(getPendingPath(pendingRequest), pendingRequest, pendingRequestTranscoder); LOG.info("{} added to pending queue, overwriting an existing SingularityPendingRequest of type STARTUP. Previous pending request was {}", existingPendingRequest, result); return result; } } LOG.info("{} added to pending queue with result: {}", pendingRequest, result); return result; }
.forPath(String.format("%s/%s", basePath, childPath))); if (pendingRequest.getPendingType() == PendingType.IMMEDIATE) { String rewrittenPath = new SingularityDeployKey(pendingRequest.getRequestId(), pendingRequest.getDeployId()) .getId(); LOG.warn("Rewriting path {} to {}", childPath, String.format("%s%s", rewrittenPath, pendingRequest.getTimestamp()));
private int numInstancesExpected(SingularityRequest request, SingularityPendingRequest pendingRequest, Optional<SingularityPendingDeploy> maybePendingDeploy) { if (!maybePendingDeploy.isPresent() || (maybePendingDeploy.get().getCurrentDeployState() == DeployState.CANCELED) || !maybePendingDeploy.get().getDeployProgress().isPresent()) { return request.getInstancesSafe(); } SingularityDeployProgress deployProgress = maybePendingDeploy.get().getDeployProgress().get(); if (maybePendingDeploy.get().getDeployMarker().getDeployId().equals(pendingRequest.getDeployId())) { return deployProgress.getTargetActiveInstances(); } else { if (deployProgress.isStepComplete()) { return Math.max(request.getInstancesSafe() - deployProgress.getTargetActiveInstances(), 0); } else { return request.getInstancesSafe() - (Math.max(deployProgress.getTargetActiveInstances() - deployProgress.getDeployInstanceCountPerStep(), 0)); } } }
.forPath(String.format("%s/%s", basePath, originalBasename))); if (pendingRequest.getPendingType() == PendingType.IMMEDIATE || pendingRequest.getPendingType() == PendingType.ONEOFF) { String deployKey = new SingularityDeployKey(pendingRequest.getRequestId(), pendingRequest.getDeployId()).getId(); String rewrittenBasename = String.format("%s%s%s", deployKey, pendingRequest.getTimestamp(), pendingRequest.getRunId().or("")); if (originalBasename.equals(rewrittenBasename)) {
private boolean shouldScheduleTasks(SingularityRequest request, SingularityPendingRequest pendingRequest, Optional<SingularityPendingDeploy> maybePendingDeploy, Optional<SingularityRequestDeployState> maybeRequestDeployState) { if (request.isDeployable() && pendingRequest.getPendingType() == PendingType.NEW_DEPLOY && !maybePendingDeploy.isPresent()) { return false; } if (request.getRequestType() == RequestType.RUN_ONCE && pendingRequest.getPendingType() == PendingType.NEW_DEPLOY) { return true; } return isDeployInUse(maybeRequestDeployState, pendingRequest.getDeployId(), false); }
private String pendingQueueKey(SingularityPendingRequest pendingRequest) { SingularityDeployKey deployKey = new SingularityDeployKey(pendingRequest.getRequestId(), pendingRequest.getDeployId()); if (pendingRequest.getPendingType() == PendingType.ONEOFF || pendingRequest.getPendingType() == PendingType.IMMEDIATE) { return String.format("%s%s%s", deployKey.toString(), pendingRequest.getTimestamp(), pendingRequest.getRunId().or("")); } else { return deployKey.toString(); } }
Assert.assertEquals(r.getDeployId(), p1.getDeployId()); Assert.assertEquals(r.getTimestamp(), p1.getTimestamp()); Assert.assertEquals(r.getPendingType(), p1.getPendingType()); Assert.assertEquals(r.getUser(), p1.getUser()); } else { Assert.assertEquals(r.getDeployId(), p2.getDeployId()); Assert.assertEquals(r.getTimestamp(), p2.getTimestamp()); Assert.assertEquals(r.getPendingType(), p2.getPendingType());
private void schedule(int numMissingInstances, List<SingularityTaskId> matchingTaskIds, SingularityRequest request, RequestState state, SingularityDeployStatistics deployStatistics, SingularityPendingRequest pendingRequest, Optional<SingularityPendingDeploy> maybePendingDeploy) { final List<SingularityPendingTask> scheduledTasks = getScheduledTaskIds(numMissingInstances, matchingTaskIds, request, state, deployStatistics, pendingRequest.getDeployId(), pendingRequest, maybePendingDeploy); if (!scheduledTasks.isEmpty()) { LOG.trace("Scheduling tasks: {}", scheduledTasks); for (SingularityPendingTask scheduledTask : scheduledTasks) { taskManager.savePendingTask(scheduledTask); } } else { LOG.info("No new scheduled tasks found for {}, setting state to {}", request.getId(), RequestState.FINISHED); requestManager.finish(request, System.currentTimeMillis()); } }
private SingularityRequest updatedRequest(Optional<SingularityPendingDeploy> maybePendingDeploy, SingularityPendingRequest pendingRequest, SingularityRequestWithState currentRequest) { if (maybePendingDeploy.isPresent() && pendingRequest.getDeployId().equals(maybePendingDeploy.get().getDeployMarker().getDeployId())) { return maybePendingDeploy.get().getUpdatedRequest().or(currentRequest.getRequest()); } else { return currentRequest.getRequest(); } } }
@Timed public void drainPendingQueue() { final long start = System.currentTimeMillis(); final ImmutableList<SingularityPendingRequest> pendingRequests = ImmutableList.copyOf(requestManager.getPendingRequests()); if (pendingRequests.isEmpty()) { LOG.trace("Pending queue was empty"); return; } LOG.info("Pending queue had {} requests", pendingRequests.size()); Map<SingularityDeployKey, List<SingularityPendingRequest>> deployKeyToPendingRequests = pendingRequests.stream() .collect(Collectors.groupingBy((request) -> new SingularityDeployKey(request.getRequestId(), request.getDeployId()))); AtomicInteger totalNewScheduledTasks = new AtomicInteger(0); AtomicInteger heldForScheduledActiveTask = new AtomicInteger(0); AtomicInteger obsoleteRequests = new AtomicInteger(0); deployKeyToPendingRequests.forEach((deployKey, pendingRequestsForDeployKey) -> { lock.runWithRequestLock( () -> handlePendingRequestsForDeployKey(obsoleteRequests, heldForScheduledActiveTask, totalNewScheduledTasks, deployKey, pendingRequestsForDeployKey), deployKey.getRequestId(), String.format("%s#%s", getClass().getSimpleName(), "drainPendingQueue")); }); LOG.info("Scheduled {} new tasks ({} obsolete requests, {} held) in {}", totalNewScheduledTasks.get(), obsoleteRequests.get(), heldForScheduledActiveTask.get(), JavaUtils.duration(start)); }
private void deleteScheduledTasks(final Collection<SingularityPendingTask> scheduledTasks, SingularityPendingRequest pendingRequest) { List<SingularityPendingTask> tasksForDeploy = scheduledTasks .stream() .filter(task -> pendingRequest.getRequestId().equals(task.getPendingTaskId().getRequestId())) .filter(task -> pendingRequest.getDeployId().equals(task.getPendingTaskId().getDeployId())) .collect(Collectors.toList()); for (SingularityPendingTask task : tasksForDeploy) { LOG.debug("Deleting pending task {} in order to reschedule {}", task.getPendingTaskId().getId(), pendingRequest); taskManager.deletePendingTask(task.getPendingTaskId()); } }
public SingularityCreateResult addToPendingQueue(SingularityPendingRequest pendingRequest) { SingularityCreateResult result = create(getPendingPath(pendingRequest), pendingRequest, pendingRequestTranscoder); if (result == SingularityCreateResult.EXISTED) { Optional<SingularityPendingRequest> existingPendingRequest = getPendingRequest(pendingRequest.getRequestId(), pendingRequest.getDeployId()); if (!existingPendingRequest.isPresent() || (existingPendingRequest.get().getPendingType() == PendingType.STARTUP)) { // Fresh pending requests take priority over STARTUP-type pending requests (or if the pending request we originally found is gone, we'll go ahead and do the write now) set(getPendingPath(pendingRequest), pendingRequest, pendingRequestTranscoder); LOG.info("{} added to pending queue, overwriting an existing SingularityPendingRequest of type STARTUP. Previous pending request was {}", existingPendingRequest, result); return result; } } LOG.info("{} added to pending queue with result: {}", pendingRequest, result); return result; }
private int numInstancesExpected(SingularityRequest request, SingularityPendingRequest pendingRequest, Optional<SingularityPendingDeploy> maybePendingDeploy) { if (!maybePendingDeploy.isPresent() || (maybePendingDeploy.get().getCurrentDeployState() == DeployState.CANCELED) || !maybePendingDeploy.get().getDeployProgress().isPresent()) { return request.getInstancesSafe(); } SingularityDeployProgress deployProgress = maybePendingDeploy.get().getDeployProgress().get(); if (maybePendingDeploy.get().getDeployMarker().getDeployId().equals(pendingRequest.getDeployId())) { return deployProgress.getTargetActiveInstances(); } else { if (deployProgress.isStepComplete()) { return Math.max(request.getInstancesSafe() - deployProgress.getTargetActiveInstances(), 0); } else { return request.getInstancesSafe() - (Math.max(deployProgress.getTargetActiveInstances() - deployProgress.getDeployInstanceCountPerStep(), 0)); } } }
private boolean shouldScheduleTasks(SingularityRequest request, SingularityPendingRequest pendingRequest, Optional<SingularityPendingDeploy> maybePendingDeploy, Optional<SingularityRequestDeployState> maybeRequestDeployState) { if (request.isDeployable() && pendingRequest.getPendingType() == PendingType.NEW_DEPLOY && !maybePendingDeploy.isPresent()) { return false; } if (request.getRequestType() == RequestType.RUN_ONCE && pendingRequest.getPendingType() == PendingType.NEW_DEPLOY) { return true; } return isDeployInUse(maybeRequestDeployState, pendingRequest.getDeployId(), false); }
private String pendingQueueKey(SingularityPendingRequest pendingRequest) { SingularityDeployKey deployKey = new SingularityDeployKey(pendingRequest.getRequestId(), pendingRequest.getDeployId()); if (pendingRequest.getPendingType() == PendingType.ONEOFF || pendingRequest.getPendingType() == PendingType.IMMEDIATE) { return String.format("%s%s%s", deployKey.toString(), pendingRequest.getTimestamp(), pendingRequest.getRunId().or("")); } else { return deployKey.toString(); } }
Assert.assertEquals(r.getDeployId(), p1.getDeployId()); Assert.assertEquals(r.getTimestamp(), p1.getTimestamp()); Assert.assertEquals(r.getPendingType(), p1.getPendingType()); Assert.assertEquals(r.getUser(), p1.getUser()); } else { Assert.assertEquals(r.getDeployId(), p2.getDeployId()); Assert.assertEquals(r.getTimestamp(), p2.getTimestamp()); Assert.assertEquals(r.getPendingType(), p2.getPendingType());