protected void saveAndSchedule(SingularityRequestBuilder bldr) { SingularityRequest build = bldr.build(); requestManager.activate(build, RequestHistoryType.UPDATED, System.currentTimeMillis(), Optional.<String> absent(), Optional.<String> absent()); requestManager.addToPendingQueue(new SingularityPendingRequest(build.getId(), firstDeployId, System.currentTimeMillis(), Optional.<String> absent(), PendingType.UPDATED_REQUEST, Optional.<Boolean> absent(), Optional.<String> absent())); scheduler.drainPendingQueue(); }
@Test public void testQueueMultipleOneOffs() { SingularityRequestBuilder bldr = new SingularityRequestBuilder(requestId, RequestType.ON_DEMAND); requestResource.postRequest(bldr.build(), singularityUser); deploy("on_demand_deploy"); deployChecker.checkDeploys(); requestManager.addToPendingQueue(new SingularityPendingRequest(requestId, "on_demand_deploy", System.currentTimeMillis(), Optional.<String>absent(), PendingType.ONEOFF, Optional.<List<String>>absent(), Optional.<String>absent(), Optional.<Boolean>absent(), Optional.<String>absent(), Optional.<String>absent())); requestManager.addToPendingQueue(new SingularityPendingRequest(requestId, "on_demand_deploy", System.currentTimeMillis(), Optional.<String>absent(), PendingType.ONEOFF, Optional.<List<String>>absent(), Optional.<String>absent(), Optional.<Boolean>absent(), Optional.<String>absent(), Optional.<String>absent())); scheduler.drainPendingQueue(); Assert.assertEquals(2, taskManager.getPendingTaskIds().size()); }
@Test public void testMaxOnDemandTasks() { SingularityRequestBuilder bldr = new SingularityRequestBuilder(requestId, RequestType.ON_DEMAND); bldr.setInstances(Optional.of(1)); requestResource.postRequest(bldr.build(), singularityUser); deploy("on_demand_deploy"); deployChecker.checkDeploys(); requestManager.addToPendingQueue(new SingularityPendingRequest(requestId, "on_demand_deploy", System.currentTimeMillis(), Optional.<String>absent(), PendingType.ONEOFF, Optional.<List<String>>absent(), Optional.<String>absent(), Optional.<Boolean>absent(), Optional.<String>absent(), Optional.<String>absent())); requestManager.addToPendingQueue(new SingularityPendingRequest(requestId, "on_demand_deploy", System.currentTimeMillis(), Optional.<String>absent(), PendingType.ONEOFF, Optional.<List<String>>absent(), Optional.<String>absent(), Optional.<Boolean>absent(), Optional.<String>absent(), Optional.<String>absent())); scheduler.drainPendingQueue(); resourceOffers(); Assert.assertEquals(1, taskManager.getActiveTaskIds().size()); }
protected void killKilledTasks() { for (SingularityKilledTaskIdRecord killed : taskManager.getKilledTaskIdRecords()) { statusUpdate(taskManager.getTask(killed.getTaskId()).get(), TaskState.TASK_KILLED); } scheduler.drainPendingQueue(); }
@Test public void testScaleWithBounceDoesNotLaunchExtraInstances() { initRequest(); initFirstDeploy(); launchTask(request, firstDeploy, 1, TaskState.TASK_RUNNING); requestResource.scale(requestId, new SingularityScaleRequest(Optional.of(5), Optional.of(1L), Optional.absent(), Optional.absent(), Optional.absent(), Optional.of(true), Optional.absent(), Optional.absent()), singularityUser); Assert.assertEquals(1, requestManager.getCleanupRequests().size()); cleaner.drainCleanupQueue(); Assert.assertEquals(1, taskManager.getNumCleanupTasks()); scheduler.drainPendingQueue(); Assert.assertEquals(5, taskManager.getPendingTaskIds().size()); }
@Test public void testObsoletePendingRequestsRemoved() { initRequest(); initFirstDeploy(); SingularityTask taskOne = startTask(firstDeploy); requestResource.pause(requestId, Optional.absent(), singularityUser); requestManager.addToPendingQueue(new SingularityPendingRequest(requestId, firstDeployId, System.currentTimeMillis(), Optional.<String>absent(), PendingType.NEW_DEPLOY, Optional.<Boolean>absent(), Optional.<String>absent())); Assert.assertEquals(requestManager.getPendingRequests().size(), 1); scheduler.drainPendingQueue(); Assert.assertEquals(requestManager.getPendingRequests().size(), 0); }
@Test public void testEmptyDecommissioning() { sms.resourceOffers(Arrays.asList(createOffer(1, 129, 1025, "slave1", "host1", Optional.of("rack1")))); Assert.assertEquals(StateChangeResult.SUCCESS, slaveManager.changeState("slave1", MachineState.STARTING_DECOMMISSION, Optional.absent(), Optional.of("user1"))); scheduler.drainPendingQueue(); sms.resourceOffers(Arrays.asList(createOffer(1, 129, 1025, "slave1", "host1", Optional.of("rack1")))); Assert.assertEquals(MachineState.DECOMMISSIONED, slaveManager.getObject("slave1").get().getCurrentState().getState()); }
@Test public void testScheduledJobLivesThroughDeploy() { initScheduledRequest(); initFirstDeploy(); createAndSchedulePendingTask(firstDeployId); Assert.assertTrue(!taskManager.getPendingTaskIds().isEmpty()); deploy("d2"); scheduler.drainPendingQueue(); deployChecker.checkDeploys(); scheduler.drainPendingQueue(); Assert.assertTrue(!taskManager.getPendingTaskIds().isEmpty()); }
@Test public void testSchedulerRandomizesOffers() { initRequest(); initFirstDeploy(); requestResource.postRequest(request.toBuilder().setInstances(Optional.of(15)).build(), singularityUser); scheduler.drainPendingQueue(); sms.resourceOffers(Arrays.asList(createOffer(20, 1024, 20000), createOffer(20, 1024, 20000))); Assert.assertEquals(15, taskManager.getActiveTaskIds().size()); Set<String> offerIds = Sets.newHashSet(); for (SingularityTask activeTask : taskManager.getActiveTasks()) { offerIds.addAll(activeTask.getOffers().stream().map((o) -> o.getId().getValue()).collect(Collectors.toList())); } Assert.assertEquals(2, offerIds.size()); }
@Test public void testScaleDownTakesHighestInstances() { initRequest(); initFirstDeploy(); saveAndSchedule(request.toBuilder().setInstances(Optional.of(5))); resourceOffers(); Assert.assertEquals(5, taskManager.getActiveTaskIds().size()); requestResource.scale(requestId, new SingularityScaleRequest(Optional.of(2), Optional.absent(), Optional.absent(), Optional.absent(), Optional.absent(), Optional.absent(), Optional.absent(), Optional.absent()), singularityUser); resourceOffers(); cleaner.drainCleanupQueue(); Assert.assertEquals(3, taskManager.getKilledTaskIdRecords().size()); for (SingularityKilledTaskIdRecord taskId : taskManager.getKilledTaskIdRecords()) { Assert.assertTrue(taskId.getTaskId().getInstanceNo() > 2); scheduler.drainPendingQueue(); } }
@Test public void testSchedulerCanBatchOnOffers() { initRequest(); initFirstDeploy(); requestResource.postRequest(request.toBuilder().setInstances(Optional.of(3)).build(), singularityUser); scheduler.drainPendingQueue(); List<Offer> oneOffer = Arrays.asList(createOffer(12, 1024, 5000)); sms.resourceOffers(oneOffer); Assert.assertTrue(taskManager.getActiveTasks().size() == 3); Assert.assertTrue(taskManager.getPendingTaskIds().isEmpty()); Assert.assertTrue(requestManager.getPendingRequests().isEmpty()); }
@Test public void testSchedulerExhaustsOffers() { initRequest(); initFirstDeploy(); requestResource.postRequest(request.toBuilder().setInstances(Optional.of(10)).build(), singularityUser); scheduler.drainPendingQueue(); sms.resourceOffers(Arrays.asList(createOffer(2, 1024, 2048), createOffer(1, 1024, 2048))); Assert.assertEquals(3, taskManager.getActiveTaskIds().size()); Assert.assertEquals(7, taskManager.getPendingTaskIds().size()); }
@Test public void testRunOnceDontMoveDuringDecomission() { SingularityRequestBuilder bldr = new SingularityRequestBuilder(requestId, RequestType.RUN_ONCE); request = bldr.build(); saveRequest(request); deployResource.deploy(new SingularityDeployRequest(new SingularityDeployBuilder(requestId, "d1").setCommand(Optional.of("cmd")).build(), Optional.absent(), Optional.absent()), singularityUser); scheduler.drainPendingQueue(); deployChecker.checkDeploys(); validateTaskDoesntMoveDuringDecommission(); }
@Test public void testJobRescheduledWhenItFinishesDuringDecommission() { initScheduledRequest(); initFirstDeploy(); resourceOffers(); SingularityTask task = launchTask(request, firstDeploy, 1, TaskState.TASK_RUNNING); slaveManager.changeState("slave1", MachineState.STARTING_DECOMMISSION, Optional.<String> absent(), Optional.of("user1")); cleaner.drainCleanupQueue(); resourceOffers(); cleaner.drainCleanupQueue(); statusUpdate(task, TaskState.TASK_FINISHED); scheduler.drainPendingQueue(); Assert.assertTrue(!taskManager.getPendingTaskIds().isEmpty()); }
@Test public void testFinishedRequestCanBeDeployed() { initScheduledRequest(); initFirstDeploy(); schedule = "*/1 * * * * ? 1995"; // cause it to be pending requestResource.postRequest(request.toBuilder().setQuartzSchedule(Optional.of(schedule)).build(), singularityUser); scheduler.drainPendingQueue(); Assert.assertTrue(requestResource.getActiveRequests(singularityUser, false, false, false, 10, Collections.emptyList()).isEmpty()); Assert.assertTrue(requestManager.getRequest(requestId).get().getState() == RequestState.FINISHED); SingularityDeployBuilder db = new SingularityDeployBuilder(requestId, secondDeployId); initDeploy(db, System.currentTimeMillis()); deployChecker.checkDeploys(); Assert.assertEquals(RequestState.ACTIVE, requestManager.getRequest(requestId).get().getState()); Assert.assertEquals(1, requestManager.getPendingRequests().size()); }
@Test public void testRunOnceDoesntGetRescheduled() { saveRequest(new SingularityRequestBuilder(requestId, RequestType.RUN_ONCE).build()); deploy(firstDeployId); deployChecker.checkDeploys(); scheduler.drainPendingQueue(); resourceOffers(); Assert.assertTrue(requestManager.getPendingRequests().isEmpty()); Assert.assertTrue(taskManager.getPendingTaskIds().isEmpty()); startup.checkSchedulerForInconsistentState(); // assert that SingularityStartup does not enqueue a SingularityPendingRequest (pendingType=NOT_STARTED) for the RUN_ONCE request Assert.assertTrue(requestManager.getPendingRequests().isEmpty()); } }
@Test public void testNotAcceptOfferWithRoleForRequestWithoutRole() { SingularityRequestBuilder bldr = new SingularityRequestBuilder(requestId, RequestType.ON_DEMAND); requestResource.postRequest(bldr.build(), singularityUser); deploy("d2"); SingularityRunNowRequest runNowRequest = new SingularityRunNowRequestBuilder() .setResources(new Resources(2, 2, 0)) .build(); requestResource.scheduleImmediately(singularityUser, requestId, runNowRequest); scheduler.drainPendingQueue(); SingularityPendingTask pendingTaskWithResources = taskManager.getPendingTasks().get(0); Assert.assertTrue(pendingTaskWithResources.getResources().isPresent()); Assert.assertEquals(pendingTaskWithResources.getResources().get().getCpus(), 2, 0.0); sms.resourceOffers(Arrays.asList(createOffer(5, 5, 5, Optional.of("test-role")))); pendingTaskWithResources = taskManager.getPendingTasks().get(0); Assert.assertTrue(pendingTaskWithResources.getResources().isPresent()); Assert.assertEquals(pendingTaskWithResources.getResources().get().getCpus(), 2, 0.0); }
@Test public void testMaxTasksPerOffer() { configuration.setMaxTasksPerOffer(3); initRequest(); initFirstDeploy(); requestResource.postRequest(request.toBuilder().setInstances(Optional.of(20)).build(), singularityUser); scheduler.drainPendingQueue(); sms.resourceOffers(Arrays.asList(createOffer(36, 12024, 50000))); Assert.assertTrue(taskManager.getActiveTasks().size() == 3); sms.resourceOffers(Arrays.asList(createOffer(20, 20000, 50000, "slave1", "host1"), createOffer(20, 20000, 50000, "slave2", "host2"))); Assert.assertTrue(taskManager.getActiveTasks().size() == 9); configuration.setMaxTasksPerOffer(0); resourceOffers(); Assert.assertTrue(taskManager.getActiveTasks().size() == 20); }
@Test public void testMultipleRunOnceTasks() { SingularityRequestBuilder bldr = new SingularityRequestBuilder(requestId, RequestType.RUN_ONCE); request = bldr.build(); saveRequest(request); deployResource.deploy(new SingularityDeployRequest(new SingularityDeployBuilder(requestId, "d1").setCommand(Optional.of("cmd")).build(), Optional.absent(), Optional.absent()), singularityUser); deployChecker.checkDeploys(); Assert.assertEquals(1, requestManager.getSizeOfPendingQueue()); deployResource.deploy(new SingularityDeployRequest(new SingularityDeployBuilder(requestId, "d2").setCommand(Optional.of("cmd")).build(), Optional.absent(), Optional.absent()), singularityUser); deployChecker.checkDeploys(); Assert.assertEquals(2, requestManager.getSizeOfPendingQueue()); scheduler.drainPendingQueue(); resourceOffers(); Assert.assertEquals(2, taskManager.getActiveTaskIds().size()); }
@Test public void testCustomResourcesWithRunNowRequest() { SingularityRequestBuilder bldr = new SingularityRequestBuilder(requestId, RequestType.ON_DEMAND); requestResource.postRequest(bldr.build(), singularityUser); deploy("d2"); SingularityRunNowRequest runNowRequest = new SingularityRunNowRequestBuilder() .setResources(new Resources(2, 2, 0)) .build(); requestResource.scheduleImmediately(singularityUser, requestId, runNowRequest); scheduler.drainPendingQueue(); SingularityPendingTask pendingTaskWithResourcs = taskManager.getPendingTasks().get(0); Assert.assertTrue(pendingTaskWithResourcs.getResources().isPresent()); Assert.assertEquals(pendingTaskWithResourcs.getResources().get().getCpus(), 2, 0.0); sms.resourceOffers(Arrays.asList(createOffer(5, 5, 5, "slave1", "host1", Optional.of("rack1")))); SingularityTask task = taskManager.getActiveTasks().get(0); Assert.assertEquals(MesosUtils.getNumCpus(mesosProtosUtils.toResourceList(task.getMesosTask().getResources()), Optional.<String>absent()), 2.0, 0.0); }