public SingularityPendingRequest build() { return new SingularityPendingRequest( requestId, deployId, timestamp, user, pendingType, cmdLineArgsList, runId, skipHealthchecks, message, actionId, resources, s3UploaderAdditionalFiles, runAsUserOverride, envOverrides, extraArtifacts, runAt ); }
private String pendingQueueKey(SingularityPendingRequest pendingRequest) { SingularityDeployKey deployKey = new SingularityDeployKey(pendingRequest.getRequestId(), pendingRequest.getDeployId()); if (pendingRequest.getPendingType() == PendingType.ONEOFF || pendingRequest.getPendingType() == PendingType.IMMEDIATE) { return String.format("%s%s%s", deployKey.toString(), pendingRequest.getTimestamp(), pendingRequest.getRunId().or("")); } else { return deployKey.toString(); } }
new SingularityPendingTaskId( request.getId(), deployId, nextRunAt.get(), nextInstanceNumber, pendingRequest.getPendingType(), pendingRequest.getTimestamp()), pendingRequest.getCmdLineArgsList(), pendingRequest.getUser(), pendingRequest.getRunId(), pendingRequest.getSkipHealthchecks(), pendingRequest.getMessage(), pendingRequest.getResources(), pendingRequest.getS3UploaderAdditionalFiles(), pendingRequest.getRunAsUserOverride(), pendingRequest.getEnvOverrides(), pendingRequest.getExtraArtifacts(), pendingRequest.getActionId()));
private boolean shouldScheduleTasks(SingularityRequest request, SingularityPendingRequest pendingRequest, Optional<SingularityPendingDeploy> maybePendingDeploy, Optional<SingularityRequestDeployState> maybeRequestDeployState) { if (request.isDeployable() && pendingRequest.getPendingType() == PendingType.NEW_DEPLOY && !maybePendingDeploy.isPresent()) { return false; } if (request.getRequestType() == RequestType.RUN_ONCE && pendingRequest.getPendingType() == PendingType.NEW_DEPLOY) { return true; } return isDeployInUse(maybeRequestDeployState, pendingRequest.getDeployId(), false); }
if (r.getRequestId().equals("r1")) { Assert.assertEquals(r.getDeployId(), p1.getDeployId()); Assert.assertEquals(r.getTimestamp(), p1.getTimestamp()); Assert.assertEquals(r.getPendingType(), p1.getPendingType()); Assert.assertTrue(!r.getCmdLineArgsList().isPresent()); Assert.assertEquals(r.getUser(), p1.getUser()); } else { Assert.assertEquals(r.getDeployId(), p2.getDeployId()); Assert.assertEquals(r.getTimestamp(), p2.getTimestamp()); Assert.assertEquals(r.getPendingType(), p2.getPendingType()); Assert.assertTrue(r.getCmdLineArgsList().get().size() == 1); Assert.assertTrue(r.getCmdLineArgsList().get().get(0).equals("cmd line args")); Assert.assertEquals(r.getUser(), p2.getUser());
@Test public void testRequestsInPendingQueueAreOrderedByTimestamp() { long now = System.currentTimeMillis(); initRequestWithType(RequestType.SCHEDULED, false); startFirstDeploy(); SingularityPendingRequest pendingDeployRequest = new SingularityPendingRequest(requestId, firstDeploy.getId(), now, Optional.absent(), PendingType.NEW_DEPLOY, firstDeploy.getSkipHealthchecksOnDeploy(), Optional.absent()); SingularityPendingRequest pendingRunNowRequest = new SingularityPendingRequest(requestId, firstDeploy.getId(), now + 200, Optional.absent(), PendingType.IMMEDIATE, firstDeploy.getSkipHealthchecksOnDeploy(), Optional.absent()); requestManager.addToPendingQueue(pendingDeployRequest); requestManager.addToPendingQueue(pendingRunNowRequest); Assert.assertEquals(2, requestManager.getPendingRequests().size()); // Was added first Assert.assertEquals(PendingType.NEW_DEPLOY, requestManager.getPendingRequests().get(0).getPendingType()); // Was added second Assert.assertEquals(PendingType.IMMEDIATE, requestManager.getPendingRequests().get(1).getPendingType()); resourceOffers(); }
private void deleteScheduledTasks(final Collection<SingularityPendingTask> scheduledTasks, SingularityPendingRequest pendingRequest) { List<SingularityPendingTask> tasksForDeploy = scheduledTasks .stream() .filter(task -> pendingRequest.getRequestId().equals(task.getPendingTaskId().getRequestId())) .filter(task -> pendingRequest.getDeployId().equals(task.getPendingTaskId().getDeployId())) .collect(Collectors.toList()); for (SingularityPendingTask task : tasksForDeploy) { LOG.debug("Deleting pending task {} in order to reschedule {}", task.getPendingTaskId().getId(), pendingRequest); taskManager.deletePendingTask(task.getPendingTaskId()); } }
public SingularityCreateResult addToPendingQueue(SingularityPendingRequest pendingRequest) { SingularityCreateResult result = create(getPendingPath(pendingRequest), pendingRequest, pendingRequestTranscoder); if (result == SingularityCreateResult.EXISTED) { Optional<SingularityPendingRequest> existingPendingRequest = getPendingRequest(pendingRequest.getRequestId(), pendingRequest.getDeployId()); if (!existingPendingRequest.isPresent() || (existingPendingRequest.get().getPendingType() == PendingType.STARTUP)) { // Fresh pending requests take priority over STARTUP-type pending requests (or if the pending request we originally found is gone, we'll go ahead and do the write now) set(getPendingPath(pendingRequest), pendingRequest, pendingRequestTranscoder); LOG.info("{} added to pending queue, overwriting an existing SingularityPendingRequest of type STARTUP. Previous pending request was {}", existingPendingRequest, result); return result; } } LOG.info("{} added to pending queue with result: {}", pendingRequest, result); return result; }
if (pendingRequest.getRequestId().equals(requestId) && pendingRequest.getRunId().isPresent() && pendingRequest.getRunId().get().equals(runId)) { return Optional.of(new SingularityTaskState( Optional.absent(), Optional.absent(), pendingRequest.getRunId(), Optional.absent(), Collections.emptyList(),
private int getNumMissingInstances(List<SingularityTaskId> matchingTaskIds, SingularityRequest request, SingularityPendingRequest pendingRequest, Optional<SingularityPendingDeploy> maybePendingDeploy) { PendingType pendingType = pendingRequest.getPendingType(); if (request.isOneOff()) { if (pendingType == PendingType.ONEOFF || pendingType == PendingType.RETRY) { return 1; } else { return 0; } } else if (request.getRequestType() == RequestType.RUN_ONCE && pendingType == PendingType.NEW_DEPLOY) { return 1; } return numInstancesExpected(request, pendingRequest, maybePendingDeploy) - matchingTaskIds.size(); }
@Override protected void handleExpiringObject(SingularityExpiringBounce expiringObject, SingularityRequestWithState requestWithState, String message) { for (SingularityTaskCleanup taskCleanup : taskManager.getCleanupTasks()) { if (taskCleanup.getTaskId().getRequestId().equals(expiringObject.getRequestId()) && taskCleanup.getActionId().isPresent() && expiringObject.getActionId().equals(taskCleanup.getActionId().get())) { LOG.info("Discarding cleanup for {} ({}) because of {}", taskCleanup.getTaskId(), taskCleanup, expiringObject); taskManager.deleteCleanupTask(taskCleanup.getTaskId().getId()); if (!taskManager.getTaskCleanup(taskCleanup.getTaskId().getId()).isPresent()) { LOG.info("No other task cleanups found, removing task cleanup update for {}", taskCleanup.getTaskId()); List<SingularityTaskHistoryUpdate> historyUpdates = taskManager.getTaskHistoryUpdates(taskCleanup.getTaskId()); Collections.sort(historyUpdates); if (Iterables.getLast(historyUpdates).getTaskState() == ExtendedTaskState.TASK_CLEANING) { Optional<SingularityTaskHistoryUpdate> maybePreviousHistoryUpdate = historyUpdates.size() > 1 ? Optional.of(historyUpdates.get(historyUpdates.size() - 2)) : Optional.<SingularityTaskHistoryUpdate>absent(); taskManager.deleteTaskHistoryUpdate(taskCleanup.getTaskId(), ExtendedTaskState.TASK_CLEANING, maybePreviousHistoryUpdate); } } } } Optional<SingularityPendingRequest> pendingRequest = requestManager.getPendingRequest(expiringObject.getRequestId(), expiringObject.getDeployId()); if (pendingRequest.isPresent() && pendingRequest.get().getActionId().isPresent() && pendingRequest.get().getActionId().get().equals(expiringObject.getActionId())) { LOG.info("Discarding pending request for {} ({}) because of {}", expiringObject.getRequestId(), pendingRequest.get(), expiringObject); requestManager.deletePendingRequest(pendingRequest.get()); } requestManager.addToPendingQueue(new SingularityPendingRequest(expiringObject.getRequestId(), expiringObject.getDeployId(), System.currentTimeMillis(), expiringObject.getUser(), PendingType.CANCEL_BOUNCE, Optional.absent(), Optional.absent(), Optional.absent(), Optional.of(message), Optional.of(expiringObject.getActionId()))); }
long now = System.currentTimeMillis(); SingularityPendingRequest immediateRequest = new SingularityPendingRequest("immediateRequest", "immediateDeploy", now, Optional.absent(), PendingType.IMMEDIATE, Optional.absent(), Optional.of("run1"), Optional.absent(), Optional.absent(), Optional.absent()); SingularityPendingRequest newDeploy = new SingularityPendingRequest("newDeployRequest", "newDeploy", now, Optional.absent(), PendingType.NEW_DEPLOY, Optional.absent(), Optional.absent()); SingularityPendingRequest oneOffRequest = new SingularityPendingRequest("oneOffRequest", "oneOffDeploy", now, Optional.absent(), PendingType.ONEOFF, Optional.absent(), Optional.absent(), Optional.absent(), Optional.absent(), Optional.absent()); curator.create().creatingParentsIfNeeded().forPath(String.format("%s%s", "/requests/pending/immediateRequest-immediateDeploy", now), objectMapper.writeValueAsBytes(immediateRequest)); curator.create().creatingParentsIfNeeded().forPath("/requests/pending/newDeployRequest-newDeploy", objectMapper.writeValueAsBytes(newDeploy)); .doesNotContain(newDeploy.toString()) .contains(oneOffRequest.toString(), immediateRequest.toString()); .hasSize(1) .extracting(SingularityPendingRequest::toString) .doesNotContain(oneOffRequest.toString()) .contains(immediateRequest.toString());
@Override public String apply(@Nonnull SingularityPendingRequest input) { return input.getRequestId(); } };
@Test public void whenRunNowIfNoRunIdSetItWillGenerateAnId() { String deployID = "deploy"; Optional<String> userEmail = Optional.absent(); SingularityRequest request = new SingularityRequestBuilder("request2", RequestType.ON_DEMAND) .build(); Optional<SingularityRunNowRequest> runNowRequest = Optional.of(runNowRequest()); List<SingularityTaskId> activeTasks = Collections.emptyList(); List<SingularityPendingTaskId> pendingTasks = Collections.emptyList(); SingularityPendingRequest pendingRequest = validator.checkRunNowRequest(deployID, userEmail, request, runNowRequest, activeTasks, pendingTasks); Assert.assertTrue(pendingRequest.getRunId().isPresent()); }
private void rebalanceAttributeDistribution( SingularityRequest request, RequestState state, SingularityDeployStatistics deployStatistics, SingularityPendingRequest pendingRequest, Optional<SingularityPendingDeploy> maybePendingDeploy, List<SingularityTaskId> remainingActiveTasks) { Set<SingularityTaskId> extraTasksToClean = rebalancingHelper.rebalanceAttributeDistribution(request, pendingRequest.getUser(), remainingActiveTasks); remainingActiveTasks.removeAll(extraTasksToClean); schedule(extraTasksToClean.size(), remainingActiveTasks, request, state, deployStatistics, pendingRequest, maybePendingDeploy); }
private void schedule(int numMissingInstances, List<SingularityTaskId> matchingTaskIds, SingularityRequest request, RequestState state, SingularityDeployStatistics deployStatistics, SingularityPendingRequest pendingRequest, Optional<SingularityPendingDeploy> maybePendingDeploy) { final List<SingularityPendingTask> scheduledTasks = getScheduledTaskIds(numMissingInstances, matchingTaskIds, request, state, deployStatistics, pendingRequest.getDeployId(), pendingRequest, maybePendingDeploy); if (!scheduledTasks.isEmpty()) { LOG.trace("Scheduling tasks: {}", scheduledTasks); for (SingularityPendingTask scheduledTask : scheduledTasks) { taskManager.savePendingTask(scheduledTask); } } else { LOG.info("No new scheduled tasks found for {}, setting state to {}", request.getId(), RequestState.FINISHED); requestManager.finish(request, System.currentTimeMillis()); } }
private Optional<Long> getNextRunAt(SingularityRequest request, RequestState state, SingularityDeployStatistics deployStatistics, SingularityPendingRequest pendingRequest, Optional<SingularityPendingDeploy> maybePendingDeploy) { PendingType pendingType = pendingRequest.getPendingType(); final long now = System.currentTimeMillis(); if (!request.isLongRunning() && pendingRequest.getRunAt().isPresent()) { nextRunAt = Math.max(nextRunAt, pendingRequest.getRunAt().get());
if (r.getRequestId().equals("r1")) { Assert.assertEquals(r.getDeployId(), p1.getDeployId()); Assert.assertEquals(r.getTimestamp(), p1.getTimestamp()); Assert.assertEquals(r.getPendingType(), p1.getPendingType()); Assert.assertTrue(!r.getCmdLineArgsList().isPresent()); Assert.assertEquals(r.getUser(), p1.getUser()); } else { Assert.assertEquals(r.getDeployId(), p2.getDeployId()); Assert.assertEquals(r.getTimestamp(), p2.getTimestamp()); Assert.assertEquals(r.getPendingType(), p2.getPendingType()); Assert.assertTrue(r.getCmdLineArgsList().get().size() == 1); Assert.assertTrue(r.getCmdLineArgsList().get().get(0).equals("cmd line args")); Assert.assertEquals(r.getUser(), p2.getUser());
@Test public void testOnDemandDoesntGetRescheduled() { saveRequest(new SingularityRequestBuilder(requestId, RequestType.ON_DEMAND).build()); deploy(firstDeployId); deployChecker.checkDeploys(); Assert.assertTrue(requestManager.getPendingRequests().isEmpty()); Assert.assertTrue(taskManager.getPendingTaskIds().isEmpty()); startup.checkSchedulerForInconsistentState(); Assert.assertTrue(requestManager.getPendingRequests().isEmpty()); Assert.assertTrue(taskManager.getPendingTaskIds().isEmpty()); requestManager.addToPendingQueue(new SingularityPendingRequest(requestId, firstDeployId, System.currentTimeMillis(), Optional.<String> absent(), PendingType.ONEOFF, Optional.<Boolean> absent(), Optional.<String> absent())); startup.checkSchedulerForInconsistentState(); Assert.assertTrue(requestManager.getPendingRequests().get(0).getPendingType() == PendingType.ONEOFF); }
@Timed public void drainPendingQueue() { final long start = System.currentTimeMillis(); final ImmutableList<SingularityPendingRequest> pendingRequests = ImmutableList.copyOf(requestManager.getPendingRequests()); if (pendingRequests.isEmpty()) { LOG.trace("Pending queue was empty"); return; } LOG.info("Pending queue had {} requests", pendingRequests.size()); Map<SingularityDeployKey, List<SingularityPendingRequest>> deployKeyToPendingRequests = pendingRequests.stream() .collect(Collectors.groupingBy((request) -> new SingularityDeployKey(request.getRequestId(), request.getDeployId()))); AtomicInteger totalNewScheduledTasks = new AtomicInteger(0); AtomicInteger heldForScheduledActiveTask = new AtomicInteger(0); AtomicInteger obsoleteRequests = new AtomicInteger(0); deployKeyToPendingRequests.forEach((deployKey, pendingRequestsForDeployKey) -> { lock.runWithRequestLock( () -> handlePendingRequestsForDeployKey(obsoleteRequests, heldForScheduledActiveTask, totalNewScheduledTasks, deployKey, pendingRequestsForDeployKey), deployKey.getRequestId(), String.format("%s#%s", getClass().getSimpleName(), "drainPendingQueue")); }); LOG.info("Scheduled {} new tasks ({} obsolete requests, {} held) in {}", totalNewScheduledTasks.get(), obsoleteRequests.get(), heldForScheduledActiveTask.get(), JavaUtils.duration(start)); }