@Override public String apply(@Nonnull SingularityPendingRequest input) { return input.getRequestId(); } };
private void filterForPendingRequests(Set<String> possiblyUnderProvisionedRequestIds) { if (possiblyUnderProvisionedRequestIds.size() == 0) { return; } final Set<String> pendingRequestIds = requestManager.getPendingRequests().stream().map((r) -> r.getRequestId()).collect(Collectors.toCollection(HashSet::new)); possiblyUnderProvisionedRequestIds.removeAll(pendingRequestIds); }
@Timed public void drainPendingQueue() { final long start = System.currentTimeMillis(); final ImmutableList<SingularityPendingRequest> pendingRequests = ImmutableList.copyOf(requestManager.getPendingRequests()); if (pendingRequests.isEmpty()) { LOG.trace("Pending queue was empty"); return; } LOG.info("Pending queue had {} requests", pendingRequests.size()); Map<SingularityDeployKey, List<SingularityPendingRequest>> deployKeyToPendingRequests = pendingRequests.stream() .collect(Collectors.groupingBy((request) -> new SingularityDeployKey(request.getRequestId(), request.getDeployId()))); AtomicInteger totalNewScheduledTasks = new AtomicInteger(0); AtomicInteger heldForScheduledActiveTask = new AtomicInteger(0); AtomicInteger obsoleteRequests = new AtomicInteger(0); deployKeyToPendingRequests.forEach((deployKey, pendingRequestsForDeployKey) -> { lock.runWithRequestLock( () -> handlePendingRequestsForDeployKey(obsoleteRequests, heldForScheduledActiveTask, totalNewScheduledTasks, deployKey, pendingRequestsForDeployKey), deployKey.getRequestId(), String.format("%s#%s", getClass().getSimpleName(), "drainPendingQueue")); }); LOG.info("Scheduled {} new tasks ({} obsolete requests, {} held) in {}", totalNewScheduledTasks.get(), obsoleteRequests.get(), heldForScheduledActiveTask.get(), JavaUtils.duration(start)); }
private void deleteScheduledTasks(final Collection<SingularityPendingTask> scheduledTasks, SingularityPendingRequest pendingRequest) { List<SingularityPendingTask> tasksForDeploy = scheduledTasks .stream() .filter(task -> pendingRequest.getRequestId().equals(task.getPendingTaskId().getRequestId())) .filter(task -> pendingRequest.getDeployId().equals(task.getPendingTaskId().getDeployId())) .collect(Collectors.toList()); for (SingularityPendingTask task : tasksForDeploy) { LOG.debug("Deleting pending task {} in order to reschedule {}", task.getPendingTaskId().getId(), pendingRequest); taskManager.deletePendingTask(task.getPendingTaskId()); } }
.forPath(String.format("%s/%s", basePath, childPath))); if (pendingRequest.getPendingType() == PendingType.IMMEDIATE) { String rewrittenPath = new SingularityDeployKey(pendingRequest.getRequestId(), pendingRequest.getDeployId()) .getId(); LOG.warn("Rewriting path {} to {}", childPath, String.format("%s%s", rewrittenPath, pendingRequest.getTimestamp()));
public SingularityCreateResult addToPendingQueue(SingularityPendingRequest pendingRequest) { SingularityCreateResult result = create(getPendingPath(pendingRequest), pendingRequest, pendingRequestTranscoder); if (result == SingularityCreateResult.EXISTED) { Optional<SingularityPendingRequest> existingPendingRequest = getPendingRequest(pendingRequest.getRequestId(), pendingRequest.getDeployId()); if (!existingPendingRequest.isPresent() || (existingPendingRequest.get().getPendingType() == PendingType.STARTUP)) { // Fresh pending requests take priority over STARTUP-type pending requests (or if the pending request we originally found is gone, we'll go ahead and do the write now) set(getPendingPath(pendingRequest), pendingRequest, pendingRequestTranscoder); LOG.info("{} added to pending queue, overwriting an existing SingularityPendingRequest of type STARTUP. Previous pending request was {}", existingPendingRequest, result); return result; } } LOG.info("{} added to pending queue with result: {}", pendingRequest, result); return result; }
.forPath(String.format("%s/%s", basePath, originalBasename))); if (pendingRequest.getPendingType() == PendingType.IMMEDIATE || pendingRequest.getPendingType() == PendingType.ONEOFF) { String deployKey = new SingularityDeployKey(pendingRequest.getRequestId(), pendingRequest.getDeployId()).getId(); String rewrittenBasename = String.format("%s%s%s", deployKey, pendingRequest.getTimestamp(), pendingRequest.getRunId().or("")); if (originalBasename.equals(rewrittenBasename)) {
private String pendingQueueKey(SingularityPendingRequest pendingRequest) { SingularityDeployKey deployKey = new SingularityDeployKey(pendingRequest.getRequestId(), pendingRequest.getDeployId()); if (pendingRequest.getPendingType() == PendingType.ONEOFF || pendingRequest.getPendingType() == PendingType.IMMEDIATE) { return String.format("%s%s%s", deployKey.toString(), pendingRequest.getTimestamp(), pendingRequest.getRunId().or("")); } else { return deployKey.toString(); } }
if (pendingRequest.getRequestId().equals(requestId) && pendingRequest.getRunId().isPresent() && pendingRequest.getRunId().get().equals(runId)) { return Optional.of(new SingularityTaskState( Optional.absent(),
if (r.getRequestId().equals("r1")) { Assert.assertEquals(r.getDeployId(), p1.getDeployId()); Assert.assertEquals(r.getTimestamp(), p1.getTimestamp());
private void filterForPendingRequests(Set<String> possiblyUnderProvisionedRequestIds) { if (possiblyUnderProvisionedRequestIds.size() == 0) { return; } final Set<String> pendingRequestIds = requestManager.getPendingRequests().stream().map((r) -> r.getRequestId()).collect(Collectors.toCollection(HashSet::new)); possiblyUnderProvisionedRequestIds.removeAll(pendingRequestIds); }
@Timed public void drainPendingQueue() { final long start = System.currentTimeMillis(); final ImmutableList<SingularityPendingRequest> pendingRequests = ImmutableList.copyOf(requestManager.getPendingRequests()); if (pendingRequests.isEmpty()) { LOG.trace("Pending queue was empty"); return; } LOG.info("Pending queue had {} requests", pendingRequests.size()); Map<SingularityDeployKey, List<SingularityPendingRequest>> deployKeyToPendingRequests = pendingRequests.stream() .collect(Collectors.groupingBy((request) -> new SingularityDeployKey(request.getRequestId(), request.getDeployId()))); AtomicInteger totalNewScheduledTasks = new AtomicInteger(0); AtomicInteger heldForScheduledActiveTask = new AtomicInteger(0); AtomicInteger obsoleteRequests = new AtomicInteger(0); deployKeyToPendingRequests.forEach((deployKey, pendingRequestsForDeployKey) -> { lock.runWithRequestLock( () -> handlePendingRequestsForDeployKey(obsoleteRequests, heldForScheduledActiveTask, totalNewScheduledTasks, deployKey, pendingRequestsForDeployKey), deployKey.getRequestId(), String.format("%s#%s", getClass().getSimpleName(), "drainPendingQueue")); }); LOG.info("Scheduled {} new tasks ({} obsolete requests, {} held) in {}", totalNewScheduledTasks.get(), obsoleteRequests.get(), heldForScheduledActiveTask.get(), JavaUtils.duration(start)); }
private void deleteScheduledTasks(final Collection<SingularityPendingTask> scheduledTasks, SingularityPendingRequest pendingRequest) { List<SingularityPendingTask> tasksForDeploy = scheduledTasks .stream() .filter(task -> pendingRequest.getRequestId().equals(task.getPendingTaskId().getRequestId())) .filter(task -> pendingRequest.getDeployId().equals(task.getPendingTaskId().getDeployId())) .collect(Collectors.toList()); for (SingularityPendingTask task : tasksForDeploy) { LOG.debug("Deleting pending task {} in order to reschedule {}", task.getPendingTaskId().getId(), pendingRequest); taskManager.deletePendingTask(task.getPendingTaskId()); } }
.forPath(String.format("%s/%s", basePath, childPath))); if (pendingRequest.getPendingType() == PendingType.IMMEDIATE) { String rewrittenPath = new SingularityDeployKey(pendingRequest.getRequestId(), pendingRequest.getDeployId()) .getId(); LOG.warn("Rewriting path {} to {}", childPath, String.format("%s%s", rewrittenPath, pendingRequest.getTimestamp()));
public SingularityCreateResult addToPendingQueue(SingularityPendingRequest pendingRequest) { SingularityCreateResult result = create(getPendingPath(pendingRequest), pendingRequest, pendingRequestTranscoder); if (result == SingularityCreateResult.EXISTED) { Optional<SingularityPendingRequest> existingPendingRequest = getPendingRequest(pendingRequest.getRequestId(), pendingRequest.getDeployId()); if (!existingPendingRequest.isPresent() || (existingPendingRequest.get().getPendingType() == PendingType.STARTUP)) { // Fresh pending requests take priority over STARTUP-type pending requests (or if the pending request we originally found is gone, we'll go ahead and do the write now) set(getPendingPath(pendingRequest), pendingRequest, pendingRequestTranscoder); LOG.info("{} added to pending queue, overwriting an existing SingularityPendingRequest of type STARTUP. Previous pending request was {}", existingPendingRequest, result); return result; } } LOG.info("{} added to pending queue with result: {}", pendingRequest, result); return result; }
.forPath(String.format("%s/%s", basePath, originalBasename))); if (pendingRequest.getPendingType() == PendingType.IMMEDIATE || pendingRequest.getPendingType() == PendingType.ONEOFF) { String deployKey = new SingularityDeployKey(pendingRequest.getRequestId(), pendingRequest.getDeployId()).getId(); String rewrittenBasename = String.format("%s%s%s", deployKey, pendingRequest.getTimestamp(), pendingRequest.getRunId().or("")); if (originalBasename.equals(rewrittenBasename)) {
private String pendingQueueKey(SingularityPendingRequest pendingRequest) { SingularityDeployKey deployKey = new SingularityDeployKey(pendingRequest.getRequestId(), pendingRequest.getDeployId()); if (pendingRequest.getPendingType() == PendingType.ONEOFF || pendingRequest.getPendingType() == PendingType.IMMEDIATE) { return String.format("%s%s%s", deployKey.toString(), pendingRequest.getTimestamp(), pendingRequest.getRunId().or("")); } else { return deployKey.toString(); } }
if (pendingRequest.getRequestId().equals(requestId) && pendingRequest.getRunId().isPresent() && pendingRequest.getRunId().get().equals(runId)) { return Optional.of(new SingularityTaskState( Optional.absent(),
if (r.getRequestId().equals("r1")) { Assert.assertEquals(r.getDeployId(), p1.getDeployId()); Assert.assertEquals(r.getTimestamp(), p1.getTimestamp());