public SingularityRequestBuilder toBuilder() { return new SingularityRequestBuilder(id, requestType) .setLoadBalanced(loadBalanced) .setInstances(instances) .setNumRetriesOnFailure(numRetriesOnFailure) .setOwners(copyOfList(owners)) .setRackSensitive(rackSensitive) .setSchedule(schedule) .setKillOldNonLongRunningTasksAfterMillis(killOldNonLongRunningTasksAfterMillis) .setTaskExecutionTimeLimitMillis(taskExecutionTimeLimitMillis) .setScheduleType(scheduleType) .setQuartzSchedule(quartzSchedule) .setScheduleTimeZone(scheduleTimeZone) .setRackAffinity(copyOfList(rackAffinity)) .setWaitAtLeastMillisAfterTaskFinishesForReschedule(waitAtLeastMillisAfterTaskFinishesForReschedule) .setSlavePlacement(slavePlacement) .setRequiredSlaveAttributes(requiredSlaveAttributes) .setAllowedSlaveAttributes(allowedSlaveAttributes) .setSlaveAttributeMinimums(slaveAttributeMinimums) .setScheduledExpectedRuntimeMillis(scheduledExpectedRuntimeMillis) .setRequiredRole(requiredRole) .setGroup(group) .setReadWriteGroups(readWriteGroups) .setReadOnlyGroups(readOnlyGroups) .setBounceAfterScale(bounceAfterScale) .setEmailConfigurationOverrides(emailConfigurationOverrides) .setSkipHealthchecks(skipHealthchecks) .setHideEvenNumberAcrossRacksHint(hideEvenNumberAcrossRacksHint) .setTaskLogErrorRegex(taskLogErrorRegex) .setTaskLogErrorRegexCaseSensitive(taskLogErrorRegexCaseSensitive)
protected SingularityRequest buildRequest(String requestId) { SingularityRequest request = new SingularityRequestBuilder(requestId, RequestType.WORKER).build(); saveRequest(request); return request; }
private void submitScaleRequest(SingularityRequestWithState oldRequestWithState, Integer newRequestedInstances) { SingularityRequest oldRequest = oldRequestWithState.getRequest(); SingularityRequest newRequest = oldRequest.toBuilder().setInstances(Optional.of((newRequestedInstances))).build(); Optional<SingularityRequestHistory.RequestHistoryType> historyType = Optional.of(SingularityRequestHistory.RequestHistoryType.SCALED); Optional<String> message = Optional.of(String.format("Auto scale number of instances to spread to all %d available slaves", newRequestedInstances)); requestHelper.updateRequest(newRequest, Optional.of(oldRequest), oldRequestWithState.getState(), historyType, Optional.<String>absent(), oldRequest.getSkipHealthchecks(), message, Optional.<SingularityBounceRequest>absent()); }
protected SingularityRequest createRequest(String requestId) { SingularityRequestBuilder bldr = new SingularityRequestBuilder(requestId, RequestType.SERVICE); bldr.setInstances(Optional.of(5)); bldr.setSlavePlacement(Optional.of(SlavePlacement.SEPARATE)); SingularityRequest request = bldr.build(); saveRequest(bldr.build()); return request; }
@Test(expected = WebApplicationException.class) public void itRestrictsAUserFromUpdatingGroupsIfTheyWillNotHaveAccess() { final SingularityAuthorizationHelper authorizationHelper = buildAuthorizationHelper(buildAuthEnabledConfig()); Set<String> readWriteGroupsOld = new HashSet<>(); readWriteGroupsOld.add("a"); final SingularityRequest oldRequest = new SingularityRequestBuilder("test_c", RequestType.SERVICE) .setGroup(Optional.of("c")) .setReadWriteGroups(Optional.of(readWriteGroupsOld)) .build(); Set<String> readWriteGroupsNew = new HashSet<>(); readWriteGroupsNew.add("b"); final SingularityRequest newRequest = new SingularityRequestBuilder("test_c", RequestType.SERVICE) .setGroup(Optional.of("c")) .setReadWriteGroups(Optional.of(readWriteGroupsNew)) .build(); authorizationHelper.checkForAuthorizedChanges(newRequest, oldRequest, USER_GROUP_A); } }
private SingularityRequestParent updateAuthorizedGroups(SingularityUser user, String requestId, SingularityUpdateGroupsRequest updateGroupsRequest) { SingularityRequestWithState oldRequestWithState = fetchRequestWithState(requestId, user); authorizationHelper.checkForAuthorization(oldRequestWithState.getRequest(), user, SingularityAuthorizationScope.WRITE); SingularityRequest newRequest = oldRequestWithState.getRequest().toBuilder() .setGroup(updateGroupsRequest.getGroup()) .setReadWriteGroups(Optional.of(updateGroupsRequest.getReadWriteGroups())) .setReadOnlyGroups(Optional.of(updateGroupsRequest.getReadOnlyGroups())) .build(); submitRequest(newRequest, Optional.of(oldRequestWithState), Optional.of(RequestHistoryType.UPDATED), Optional.absent(), updateGroupsRequest.getMessage(), Optional.absent(), user); return fillEntireRequest(fetchRequestWithState(requestId, user)); }
@Test(expected = WebApplicationException.class) public void testInvalidQuartzTimeZoneErrors() { SingularityRequest req = new SingularityRequestBuilder(requestId, RequestType.SCHEDULED) .setQuartzSchedule(Optional.of("*/1 * * * * ? 2020")) .setScheduleType(Optional.of(ScheduleType.QUARTZ)) .setScheduleTimeZone(Optional.of("invalid_timezone")) .build(); requestResource.postRequest(req, singularityUser); }
initFirstDeploy(); SingularityRequest newRequest = request.toBuilder() .setInstances(Optional.of(2)) .setRackSensitive(Optional.of(true)) .setSlavePlacement(Optional.of(SlavePlacement.SEPARATE)) .setAllowBounceToSameHost(Optional.of(true)) .build(); saveAndSchedule(newRequest.toBuilder()); scheduler.drainPendingQueue(); saveAndSchedule(newRequest.toBuilder().setInstances(Optional.of(2))); sms.resourceOffers(Arrays.asList(createOffer(1, 128, 1024, "slave1", "host1", Optional.of("rack1")))); Assert.assertEquals(3, taskManager.getActiveTaskIds().size());
@Test(expected = WebApplicationException.class) public void itForbidsRunNowOfScheduledWhenAlreadyRunning() { String deployID = "deploy"; Optional<String> userEmail = Optional.absent(); SingularityRequest request = new SingularityRequestBuilder("request2", RequestType.SCHEDULED) .setInstances(Optional.of(1)) .build(); Optional<SingularityRunNowRequest> runNowRequest = Optional.absent(); List<SingularityTaskId> activeTasks = Collections.singletonList(activeTask()); List<SingularityPendingTaskId> pendingTasks = Collections.emptyList(); validator.checkRunNowRequest(deployID, userEmail, request, runNowRequest, activeTasks, pendingTasks); }
SingularityRequest newRequest = requestWithState.getRequest().toBuilder().setQuartzSchedule(Optional.of(newQuartzSchedule)).build();
@Test public void testSlaveAttributeMinimumsCanBeExceeded() { Map<String, List<String>> reservedAttributes = new HashMap<>(); reservedAttributes.put("instance_lifecycle_type", Arrays.asList("spot")); configuration.setReserveSlavesWithAttributes(reservedAttributes); Map<String, String> allowedAttributes = new HashMap<>(); allowedAttributes.put("instance_lifecycle_type", "spot"); Map<String, Map<String, Integer>> attributeMinimums = new HashMap<>(); attributeMinimums.put("instance_lifecycle_type", ImmutableMap.of("non_spot", 70)); initRequest(); initFirstDeploy(); saveAndSchedule(request.toBuilder() .setInstances(Optional.of(10)) .setAllowedSlaveAttributes(Optional.of(allowedAttributes)) .setSlaveAttributeMinimums(Optional.of(attributeMinimums))); // Ensure we can go over the minimum if there are enough resources available sms.resourceOffers(Arrays.asList(createOffer(20, 20000, 50000, "slave1", "host1", Optional.<String>absent(), ImmutableMap.of("instance_lifecycle_type", "non_spot")))); Assert.assertTrue(taskManager.getActiveTaskIds().size() == 10); Assert.assertEquals(10, taskManager.getTasksOnSlave(taskManager.getActiveTaskIds(), slaveManager.getObject("slave1").get()).size()); }
protected void initRequestWithType(RequestType requestType, boolean isLoadBalanced) { SingularityRequestBuilder bldr = new SingularityRequestBuilder(requestId, requestType); bldr.setLoadBalanced(Optional.of(isLoadBalanced)); if (requestType == RequestType.SCHEDULED) { bldr.setQuartzSchedule(Optional.of(schedule)); } request = bldr.build(); saveRequest(request); }
@Test public void testCronScheduleChanges() throws Exception { final String requestId = "test-change-cron"; final String oldSchedule = "*/5 * * * *"; final String oldScheduleQuartz = "0 */5 * * * ?"; final String newSchedule = "*/30 * * * *"; final String newScheduleQuartz = "0 */30 * * * ?"; SingularityRequest request = new SingularityRequestBuilder(requestId, RequestType.SCHEDULED) .setSchedule(Optional.of(oldSchedule)) .build(); request = validator.checkSingularityRequest(request, Optional.<SingularityRequest>absent(), Optional.<SingularityDeploy>absent(), Optional.<SingularityDeploy>absent()); saveRequest(request); Assert.assertEquals(oldScheduleQuartz, requestManager.getRequest(requestId).get().getRequest().getQuartzScheduleSafe()); initAndFinishDeploy(request, "1"); scheduler.drainPendingQueue(); final SingularityRequest newRequest = request.toBuilder() .setSchedule(Optional.of(newSchedule)) .setQuartzSchedule(Optional.<String>absent()) .build(); final SingularityDeploy newDeploy = new SingularityDeployBuilder(request.getId(), "2").setCommand(Optional.of("sleep 100")).build(); deployResource.deploy(new SingularityDeployRequest(newDeploy, Optional.absent(), Optional.absent(), Optional.of(newRequest)), singularityUser); deployChecker.checkDeploys(); scheduler.drainPendingQueue(); Assert.assertEquals(newScheduleQuartz, requestManager.getRequest(requestId).get().getRequest().getQuartzScheduleSafe()); }
.setSkipHealthchecks(Optional.of(true)) .setInstances(Optional.of(2)) .build(), Optional.<String>absent())); .setSkipHealthchecks(Optional.<Boolean>absent()) .setInstances(Optional.of(2)) .build();
@Test public void testOfferCache() { configuration.setCacheOffers(true); configuration.setOfferCacheSize(2); List<Offer> offers2 = resourceOffers(); sms.rescind(offers2.get(0).getId()); initRequest(); initFirstDeploy(); requestResource.postRequest(request.toBuilder().setSlavePlacement(Optional.of(SlavePlacement.SEPARATE)).setInstances(Optional.of(2)).build(), singularityUser); schedulerPoller.runActionOnPoll(); Assert.assertEquals(1, taskManager.getActiveTasks().size()); resourceOffers(); Assert.assertEquals(2, taskManager.getActiveTasks().size()); }
@Override protected void handleExpiringObject(SingularityExpiringSkipHealthchecks expiringObject, SingularityRequestWithState requestWithState, String message) { final SingularityRequest oldRequest = requestWithState.getRequest(); final SingularityRequest newRequest = oldRequest.toBuilder().setSkipHealthchecks(expiringObject.getRevertToSkipHealthchecks()).build(); try { requestHelper.updateRequest(newRequest, Optional.of(oldRequest), requestWithState.getState(), Optional.<RequestHistoryType>absent(), expiringObject.getUser(), Optional.<Boolean>absent(), Optional.of(message), Optional.<SingularityBounceRequest>absent()); } catch (WebApplicationException wae) { LOG.error("While trying to apply {} for {}", expiringObject, expiringObject.getRequestId(), wae); } } }
@Test public void testSlavePlacementGreedy() { initRequest(); initFirstDeploy(); saveAndSchedule(request.toBuilder().setInstances(Optional.of(3)).setSlavePlacement(Optional.of(SlavePlacement.GREEDY))); sms.resourceOffers(Arrays.asList(createOffer(20, 20000, 50000, "slave1", "host1"))); Assert.assertTrue(taskManager.getActiveTaskIds().size() == 3); }
@Test public void testRequiredSlaveAttributesForRequest() { Map<String, String> requiredAttributes = new HashMap<>(); requiredAttributes.put("requiredKey", "requiredValue1"); initRequest(); initFirstDeploy(); saveAndSchedule(request.toBuilder().setInstances(Optional.of(1)).setRequiredSlaveAttributes(Optional.of(requiredAttributes))); sms.resourceOffers(Arrays.asList(createOffer(20, 20000, 50000, "slave1", "host1", Optional.<String>absent(), ImmutableMap.of("requiredKey", "notTheRightValue")))); sms.resourceOffers(Arrays.asList(createOffer(20, 20000, 50000, "slave2", "host2", Optional.<String>absent(), ImmutableMap.of("notTheRightKey", "requiredValue1")))); Assert.assertTrue(taskManager.getActiveTaskIds().size() == 0); sms.resourceOffers(Arrays.asList(createOffer(20, 20000, 50000, "slave2", "host2", Optional.<String>absent(), requiredAttributes))); Assert.assertTrue(taskManager.getActiveTaskIds().size() == 1); }
@Test(expected = WebApplicationException.class) public void itDoesNotWorkerToServiceTransitionIfLoadBalanced() { SingularityRequest request = new SingularityRequestBuilder("test", RequestType.WORKER) .build(); SingularityRequest newRequest = new SingularityRequestBuilder("test", RequestType.SERVICE) .setLoadBalanced(Optional.of(true)) .build(); validator.checkSingularityRequest(newRequest, Optional.of(request), Optional.absent(), Optional.absent()); }
@Test public void testAllowedSlaveAttributes() { Map<String, List<String>> reservedAttributes = new HashMap<>(); reservedAttributes.put("reservedKey", Arrays.asList("reservedValue1")); configuration.setReserveSlavesWithAttributes(reservedAttributes); Map<String, String> allowedAttributes = new HashMap<>(); allowedAttributes.put("reservedKey", "reservedValue1"); initRequest(); initFirstDeploy(); saveAndSchedule(request.toBuilder().setInstances(Optional.of(1))); sms.resourceOffers(Arrays.asList(createOffer(20, 20000, 50000, "slave1", "host1", Optional.<String>absent(), ImmutableMap.of("reservedKey", "reservedValue1")))); Assert.assertTrue(taskManager.getActiveTaskIds().size() == 0); saveAndSchedule(request.toBuilder().setInstances(Optional.of(1)).setAllowedSlaveAttributes(Optional.of(allowedAttributes))); sms.resourceOffers(Arrays.asList(createOffer(20, 20000, 50000, "slave1", "host1", Optional.<String>absent(), ImmutableMap.of("reservedKey", "reservedValue1")))); Assert.assertTrue(taskManager.getActiveTaskIds().size() == 1); }