private void filterForPendingRequests(Set<String> possiblyUnderProvisionedRequestIds) { if (possiblyUnderProvisionedRequestIds.size() == 0) { return; } final Set<String> pendingRequestIds = requestManager.getPendingRequests().stream().map((r) -> r.getRequestId()).collect(Collectors.toCollection(HashSet::new)); possiblyUnderProvisionedRequestIds.removeAll(pendingRequestIds); }
@Timed public void drainPendingQueue() { final long start = System.currentTimeMillis(); final ImmutableList<SingularityPendingRequest> pendingRequests = ImmutableList.copyOf(requestManager.getPendingRequests()); if (pendingRequests.isEmpty()) { LOG.trace("Pending queue was empty"); return; } LOG.info("Pending queue had {} requests", pendingRequests.size()); Map<SingularityDeployKey, List<SingularityPendingRequest>> deployKeyToPendingRequests = pendingRequests.stream() .collect(Collectors.groupingBy((request) -> new SingularityDeployKey(request.getRequestId(), request.getDeployId()))); AtomicInteger totalNewScheduledTasks = new AtomicInteger(0); AtomicInteger heldForScheduledActiveTask = new AtomicInteger(0); AtomicInteger obsoleteRequests = new AtomicInteger(0); deployKeyToPendingRequests.forEach((deployKey, pendingRequestsForDeployKey) -> { lock.runWithRequestLock( () -> handlePendingRequestsForDeployKey(obsoleteRequests, heldForScheduledActiveTask, totalNewScheduledTasks, deployKey, pendingRequestsForDeployKey), deployKey.getRequestId(), String.format("%s#%s", getClass().getSimpleName(), "drainPendingQueue")); }); LOG.info("Scheduled {} new tasks ({} obsolete requests, {} held) in {}", totalNewScheduledTasks.get(), obsoleteRequests.get(), heldForScheduledActiveTask.get(), JavaUtils.duration(start)); }
@GET @PropertyFiltering @Path("/queued/pending") @Operation(summary = "Retrieve the list of pending requests") public List<SingularityPendingRequest> getPendingRequests(@Parameter(hidden = true) @Auth SingularityUser user) { return authorizationHelper.filterByAuthorizedRequests(user, requestManager.getPendingRequests(), SingularityTransformHelpers.PENDING_REQUEST_TO_REQUEST_ID, SingularityAuthorizationScope.READ); }
@Test public void testObsoletePendingRequestsRemoved() { initRequest(); initFirstDeploy(); SingularityTask taskOne = startTask(firstDeploy); requestResource.pause(requestId, Optional.absent(), singularityUser); requestManager.addToPendingQueue(new SingularityPendingRequest(requestId, firstDeployId, System.currentTimeMillis(), Optional.<String>absent(), PendingType.NEW_DEPLOY, Optional.<Boolean>absent(), Optional.<String>absent())); Assert.assertEquals(requestManager.getPendingRequests().size(), 1); scheduler.drainPendingQueue(); Assert.assertEquals(requestManager.getPendingRequests().size(), 0); }
@Test public void testOnDemandDoesntGetRescheduled() { saveRequest(new SingularityRequestBuilder(requestId, RequestType.ON_DEMAND).build()); deploy(firstDeployId); deployChecker.checkDeploys(); Assert.assertTrue(requestManager.getPendingRequests().isEmpty()); Assert.assertTrue(taskManager.getPendingTaskIds().isEmpty()); startup.checkSchedulerForInconsistentState(); Assert.assertTrue(requestManager.getPendingRequests().isEmpty()); Assert.assertTrue(taskManager.getPendingTaskIds().isEmpty()); requestManager.addToPendingQueue(new SingularityPendingRequest(requestId, firstDeployId, System.currentTimeMillis(), Optional.<String> absent(), PendingType.ONEOFF, Optional.<Boolean> absent(), Optional.<String> absent())); startup.checkSchedulerForInconsistentState(); Assert.assertTrue(requestManager.getPendingRequests().get(0).getPendingType() == PendingType.ONEOFF); }
curator.create().creatingParentsIfNeeded().forPath(String.format("%s%s", "/requests/pending/oneOffRequest-oneOffDeploy", now), objectMapper.writeValueAsBytes(oneOffRequest)); Assert.assertEquals("3 existing requests under old paths", 3, requestManager.getPendingRequests().size()); System.out.println(curator.getChildren().forPath("/requests/pending")); Assert.assertEquals("3 existing requests under new paths", 3, requestManager.getPendingRequests().size()); System.out.println(curator.getChildren().forPath("/requests/pending")); Assertions.assertThat(requestManager.getPendingRequests()) .as("Non-renamed, non-timestamped nodes can be deleted") .hasSize(2) Assertions.assertThat(requestManager.getPendingRequests()) .as("Non-renamed immediate (i.e., run-now scheduled without a runId) nodes can be deleted") .hasSize(1) Assertions.assertThat(requestManager.getPendingRequests()) .as("Renamed one-off (i.e., run-now on-demand) nodes can be deleted") .hasSize(0);
@Test public void testRequestsInPendingQueueAreOrderedByTimestamp() { long now = System.currentTimeMillis(); initRequestWithType(RequestType.SCHEDULED, false); startFirstDeploy(); SingularityPendingRequest pendingDeployRequest = new SingularityPendingRequest(requestId, firstDeploy.getId(), now, Optional.absent(), PendingType.NEW_DEPLOY, firstDeploy.getSkipHealthchecksOnDeploy(), Optional.absent()); SingularityPendingRequest pendingRunNowRequest = new SingularityPendingRequest(requestId, firstDeploy.getId(), now + 200, Optional.absent(), PendingType.IMMEDIATE, firstDeploy.getSkipHealthchecksOnDeploy(), Optional.absent()); requestManager.addToPendingQueue(pendingDeployRequest); requestManager.addToPendingQueue(pendingRunNowRequest); Assert.assertEquals(2, requestManager.getPendingRequests().size()); // Was added first Assert.assertEquals(PendingType.NEW_DEPLOY, requestManager.getPendingRequests().get(0).getPendingType()); // Was added second Assert.assertEquals(PendingType.IMMEDIATE, requestManager.getPendingRequests().get(1).getPendingType()); resourceOffers(); }
curator.create().creatingParentsIfNeeded().forPath(String.format("%s%s", "/requests/pending/oneOffRequest-oneOffDeploy", now), objectMapper.writeValueAsBytes(oneOffRequest)); Assert.assertEquals("3 existing requests under old paths", 3, requestManager.getPendingRequests().size()); System.out.println(curator.getChildren().forPath("/requests/pending")); Assert.assertEquals("3 existing requests under new paths", 3, requestManager.getPendingRequests().size()); System.out.println(curator.getChildren().forPath("/requests/pending")); Assertions.assertThat(requestManager.getPendingRequests()) .as("Non-renamed, non-timestamped nodes can be deleted") .hasSize(2) Assertions.assertThat(requestManager.getPendingRequests()) .as("Non-renamed timestamped nodes can be deleted") .hasSize(1) Assertions.assertThat(requestManager.getPendingRequests()) .as("Renamed nodes can be deleted") .hasSize(0);
@Test public void testImmediateRequestsAreConsistentlyDeleted() { long now = System.currentTimeMillis(); initRequestWithType(RequestType.SCHEDULED, false); startFirstDeploy(); SingularityPendingRequest pendingDeployRequest = new SingularityPendingRequest(requestId, firstDeploy.getId(), now, Optional.absent(), PendingType.NEW_DEPLOY, firstDeploy.getSkipHealthchecksOnDeploy(), Optional.absent()); SingularityPendingRequest pendingRunNowRequest = new SingularityPendingRequest(requestId, firstDeploy.getId(), now + 200, Optional.absent(), PendingType.IMMEDIATE, firstDeploy.getSkipHealthchecksOnDeploy(), Optional.absent()); requestManager.addToPendingQueue(pendingDeployRequest); requestManager.addToPendingQueue(pendingRunNowRequest); // Pending queue has two requests: NEW_DEPLOY & IMMEDIATE Assert.assertEquals(2, requestManager.getPendingRequests().size()); finishNewTaskChecks(); requestManager.deletePendingRequest(pendingDeployRequest); // Just the immediate run Assert.assertEquals(1, requestManager.getPendingRequests().size()); requestManager.deletePendingRequest(pendingRunNowRequest); // Immediate run was successfully deleted Assert.assertEquals(0, requestManager.getPendingRequests().size()); }
@Test public void testSchedulerDropsMultipleScheduledTaskInstances() { initScheduledRequest(); SingularityDeploy deploy = SingularityDeploy.newBuilder(requestId, firstDeployId) .setCommand(Optional.of("sleep 100")) .build(); SingularityDeployRequest singularityDeployRequest = new SingularityDeployRequest(deploy, Optional.absent(), Optional.absent(), Optional.absent()); deployResource.deploy(singularityDeployRequest, singularityUser); scheduler.drainPendingQueue(); requestManager.addToPendingQueue(new SingularityPendingRequest(requestId, firstDeployId, Instant.now().plus(3, ChronoUnit.DAYS).toEpochMilli(), Optional.absent(), PendingType.NEW_DEPLOY, Optional.absent(), Optional.absent())); SingularityRunNowRequest runNowRequest = new SingularityRunNowRequestBuilder().build(); requestResource.scheduleImmediately(singularityUser, requestId, runNowRequest); Assert.assertEquals("Both requests make it into the pending queue", 2, requestManager.getPendingRequests().size()); Assert.assertEquals(PendingType.IMMEDIATE, requestManager.getPendingRequests().get(0).getPendingType()); Assert.assertEquals(PendingType.NEW_DEPLOY, requestManager.getPendingRequests().get(1).getPendingType()); scheduler.drainPendingQueue(); Assertions.assertThat(taskManager.getPendingTaskIds()) .describedAs("Only the immediate request gets run") .hasSize(1) .extracting(SingularityPendingTaskId::getPendingType) .containsExactly(PendingType.IMMEDIATE); Assertions.assertThat(requestManager.getPendingRequests()) .describedAs("The scheduled request is dropped from the pending queue") .hasSize(0); }
@Test public void testRunOnceDoesntGetRescheduled() { saveRequest(new SingularityRequestBuilder(requestId, RequestType.RUN_ONCE).build()); deploy(firstDeployId); deployChecker.checkDeploys(); scheduler.drainPendingQueue(); resourceOffers(); Assert.assertTrue(requestManager.getPendingRequests().isEmpty()); Assert.assertTrue(taskManager.getPendingTaskIds().isEmpty()); startup.checkSchedulerForInconsistentState(); // assert that SingularityStartup does not enqueue a SingularityPendingRequest (pendingType=NOT_STARTED) for the RUN_ONCE request Assert.assertTrue(requestManager.getPendingRequests().isEmpty()); } }
private void runTest(RequestType requestType, Reason reason, boolean shouldRetry) { initRequestWithType(requestType, false); initFirstDeploy(); SingularityTask task = startTask(firstDeploy); Assert.assertEquals(taskManager.getPendingTaskIds().size(), 0); Assert.assertEquals(requestManager.getPendingRequests().size(), 0); try { updateHandler.processStatusUpdateAsync(TaskStatus.newBuilder() .setState(TaskState.TASK_LOST) .setReason(reason) .setTaskId(TaskID.newBuilder().setValue(task.getTaskId().getId())) .build()).get(); } catch (InterruptedException | ExecutionException e) { Assert.assertTrue(false); } if (shouldRetry) { Assert.assertEquals(requestManager.getPendingRequests().size(), 1); Assert.assertEquals(requestManager.getPendingRequests().get(0).getPendingType(), PendingType.RETRY); } else { if (requestManager.getPendingRequests().size() > 0) { Assert.assertEquals(requestManager.getPendingRequests().get(0).getPendingType(), PendingType.TASK_DONE); } } scheduler.drainPendingQueue(); } }
@Test public void testScheduledTasksDontGetRescheduled() { initScheduledRequest(); deploy(firstDeployId); deployChecker.checkDeploys(); Assert.assertEquals("NEW_DEPLOY request added", 1, requestManager.getPendingRequests().size()); Assert.assertTrue("No tasks started yet", taskManager.getPendingTaskIds().isEmpty()); startup.checkSchedulerForInconsistentState(); Assert.assertEquals("NEW_DEPLOY request added", 1, requestManager.getPendingRequests().size()); Assert.assertEquals("NEW_DEPLOY is first", PendingType.NEW_DEPLOY, requestManager.getPendingRequests().get(0).getPendingType()); Assert.assertTrue("No tasks started yet", taskManager.getPendingTaskIds().isEmpty()); scheduler.drainPendingQueue(); Assert.assertTrue("Pending queue is cleared", requestManager.getPendingRequests().isEmpty()); List<SingularityPendingTask> pending = taskManager.getPendingTasks(); Assert.assertEquals("One task is started", 1, taskManager.getPendingTaskIds().size()); Assert.assertEquals("First request takes precedence", PendingType.NEW_DEPLOY, taskManager.getPendingTaskIds().get(0).getPendingType()); startup.checkSchedulerForInconsistentState(); scheduler.drainPendingQueue(); Assert.assertTrue(requestManager.getPendingRequests().isEmpty()); Assert.assertTrue(taskManager.getPendingTaskIds().size() == 1); Assert.assertTrue(taskManager.getActiveTaskIds().isEmpty()); Assert.assertTrue(pending.equals(taskManager.getPendingTasks())); taskManager.deletePendingTask(pending.get(0).getPendingTaskId()); startup.checkSchedulerForInconsistentState(); Assert.assertTrue(requestManager.getPendingRequests().size() == 1); Assert.assertTrue(taskManager.getPendingTaskIds().isEmpty()); }
@Test public void testSchedulerCanBatchOnOffers() { initRequest(); initFirstDeploy(); requestResource.postRequest(request.toBuilder().setInstances(Optional.of(3)).build(), singularityUser); scheduler.drainPendingQueue(); List<Offer> oneOffer = Arrays.asList(createOffer(12, 1024, 5000)); sms.resourceOffers(oneOffer); Assert.assertTrue(taskManager.getActiveTasks().size() == 3); Assert.assertTrue(taskManager.getPendingTaskIds().isEmpty()); Assert.assertTrue(requestManager.getPendingRequests().isEmpty()); }
@Test public void testScheduledTasksDontGetRescheduledDuringRun() { initScheduledRequest(); initFirstDeploy(); startTask(firstDeploy); startup.checkSchedulerForInconsistentState(); scheduler.drainPendingQueue(); Assert.assertTrue(taskManager.getPendingTaskIds().isEmpty()); Assert.assertTrue(requestManager.getPendingRequests().isEmpty()); boolean caughtException = false; try { requestResource.scheduleImmediately(singularityUser, requestId, ((SingularityRunNowRequest) null)); } catch (Exception e) { caughtException = true; } Assert.assertTrue(caughtException); Assert.assertTrue(taskManager.getPendingTaskIds().isEmpty()); Assert.assertTrue(requestManager.getPendingRequests().isEmpty()); }
Assert.assertEquals(1, requestManager.getPendingRequests().size()); Assert.assertEquals(0, requestManager.getCleanupRequests().size()); Assert.assertEquals(3, taskManager.getCleanupTaskIds().size()); Assert.assertEquals(5, taskManager.getActiveTaskIds().size()); Assert.assertEquals(3, taskManager.getCleanupTaskIds().size()); Assert.assertEquals(0, requestManager.getPendingRequests().size()); Assert.assertEquals(0, requestManager.getCleanupRequests().size()); Assert.assertEquals(0, taskManager.getPendingTasks().size()); Assert.assertEquals(0, taskManager.getCleanupTaskIds().size()); Assert.assertEquals(0, requestManager.getPendingRequests().size()); Assert.assertEquals(0, requestManager.getCleanupRequests().size());
@Test public void testOneOffsDontRunByThemselves() { SingularityRequestBuilder bldr = new SingularityRequestBuilder(requestId, RequestType.ON_DEMAND); requestResource.postRequest(bldr.build(), singularityUser); Assert.assertTrue(requestManager.getPendingRequests().isEmpty()); deploy("d2"); Assert.assertTrue(requestManager.getPendingRequests().isEmpty()); deployChecker.checkDeploys(); Assert.assertTrue(requestManager.getPendingRequests().isEmpty()); requestResource.scheduleImmediately(singularityUser, requestId, ((SingularityRunNowRequest) null)); resourceOffers(); Assert.assertEquals(1, taskManager.getActiveTaskIds().size()); statusUpdate(taskManager.getActiveTasks().get(0), TaskState.TASK_FINISHED); resourceOffers(); Assert.assertEquals(0, taskManager.getActiveTaskIds().size()); Assert.assertEquals(0, taskManager.getPendingTaskIds().size()); requestResource.scheduleImmediately(singularityUser, requestId, ((SingularityRunNowRequest) null)); resourceOffers(); Assert.assertEquals(1, taskManager.getActiveTaskIds().size()); statusUpdate(taskManager.getActiveTasks().get(0), TaskState.TASK_LOST); resourceOffers(); Assert.assertEquals(0, taskManager.getActiveTaskIds().size()); Assert.assertEquals(0, taskManager.getPendingTaskIds().size()); }
@Test public void testFinishedRequestCanBeDeployed() { initScheduledRequest(); initFirstDeploy(); schedule = "*/1 * * * * ? 1995"; // cause it to be pending requestResource.postRequest(request.toBuilder().setQuartzSchedule(Optional.of(schedule)).build(), singularityUser); scheduler.drainPendingQueue(); Assert.assertTrue(requestResource.getActiveRequests(singularityUser, false, false, false, 10, Collections.emptyList()).isEmpty()); Assert.assertTrue(requestManager.getRequest(requestId).get().getState() == RequestState.FINISHED); SingularityDeployBuilder db = new SingularityDeployBuilder(requestId, secondDeployId); initDeploy(db, System.currentTimeMillis()); deployChecker.checkDeploys(); Assert.assertEquals(RequestState.ACTIVE, requestManager.getRequest(requestId).get().getState()); Assert.assertEquals(1, requestManager.getPendingRequests().size()); }
Assert.assertTrue(taskManager.getPendingTaskIds().contains(pt2)); Assert.assertTrue(requestManager.getPendingRequests().size() == 2); for (SingularityPendingRequest r : requestManager.getPendingRequests()) { if (r.getRequestId().equals("r1")) { Assert.assertEquals(r.getDeployId(), p1.getDeployId());
Assert.assertTrue(requestManager.getPendingRequests().isEmpty());