private void checkForIllegalChanges(SingularityRequest request, SingularityRequest existingRequest) { if (request.getRequestType() != existingRequest.getRequestType()) { boolean validWorkerServiceTransition = (existingRequest.getRequestType() == RequestType.SERVICE && !existingRequest.isLoadBalanced() && request.getRequestType() == RequestType.WORKER) || (request.getRequestType() == RequestType.SERVICE && !request.isLoadBalanced() && existingRequest.getRequestType() == RequestType.WORKER); checkBadRequest(validWorkerServiceTransition, String.format("Request can not change requestType from %s to %s", existingRequest.getRequestType(), request.getRequestType())); } checkBadRequest(request.isLoadBalanced() == existingRequest.isLoadBalanced(), "Request can not change whether it is load balanced"); }
public double getTaskPriorityLevelForRequest(SingularityRequest request) { return request.getTaskPriorityLevel().or(Optional.fromNullable(defaultTaskPriorityLevelForRequestType.get(request.getRequestType()))).or(defaultTaskPriorityLevel); } }
private void setRequestType(RequestType type) { Mockito.when(request.getRequestType()).thenReturn(type); } }
private void populateRequestEmailProperties(Map<String, Object> templateProperties, SingularityRequest request, SingularityEmailType emailType) { templateProperties.put("requestId", request.getId()); templateProperties.put("singularityRequestLink", mailTemplateHelpers.getSingularityRequestLink(request.getId())); templateProperties.put("requestAlwaysRunning", request.isAlwaysRunning()); templateProperties.put("requestRunOnce", request.getRequestType() == RequestType.RUN_ONCE); templateProperties.put("requestScheduled", request.isScheduled()); templateProperties.put("requestOneOff", request.isOneOff()); templateProperties.put("taskWillRetry", request.getNumRetriesOnFailure().or(0) > 0); templateProperties.put("numRetries", request.getNumRetriesOnFailure().or(0)); templateProperties.put("color", emailType.getColor()); }
private long getLastActionTimeForRequest(SingularityRequest request, Optional<SingularityRequestHistory> lastHistory, Optional<SingularityRequestDeployState> deployState, Optional<SingularityTaskIdHistory> mostRecentTask) { long lastUpdate = 0; if (lastHistory.isPresent()) { lastUpdate = lastHistory.get().getCreatedAt(); } if (deployState.isPresent()) { if (deployState.get().getActiveDeploy().isPresent()) { lastUpdate = Math.max(lastUpdate, deployState.get().getActiveDeploy().get().getTimestamp()); } if (deployState.get().getPendingDeploy().isPresent()) { lastUpdate = Math.max(lastUpdate, deployState.get().getPendingDeploy().get().getTimestamp()); } } // Only consider most recent task time for non-long-running if (mostRecentTask.isPresent() && !request.getRequestType().isLongRunning()) { lastUpdate = Math.max(lastUpdate, mostRecentTask.get().getUpdatedAt()); } return lastUpdate; } }
private int getNumMissingInstances(List<SingularityTaskId> matchingTaskIds, SingularityRequest request, SingularityPendingRequest pendingRequest, Optional<SingularityPendingDeploy> maybePendingDeploy) { PendingType pendingType = pendingRequest.getPendingType(); if (request.isOneOff()) { if (pendingType == PendingType.ONEOFF || pendingType == PendingType.RETRY) { return 1; } else { return 0; } } else if (request.getRequestType() == RequestType.RUN_ONCE && pendingType == PendingType.NEW_DEPLOY) { return 1; } return numInstancesExpected(request, pendingRequest, maybePendingDeploy) - matchingTaskIds.size(); }
private boolean isLongRunning(SingularityTaskId task) { Optional<SingularityRequestWithState> request = requestManager.getRequest(task.getRequestId()); if (request.isPresent()) { return request.get().getRequest().getRequestType().isLongRunning(); } LOG.warn("Couldn't find request id {} for task {}", task.getRequestId(), task.getId()); return false; }
private boolean isTooManyInstancesForRequest(SingularityTaskRequest taskRequest, List<SingularityTaskId> activeTaskIdsForRequest) { if (taskRequest.getRequest().getRequestType() == RequestType.ON_DEMAND) { int maxActiveOnDemandTasks = taskRequest.getRequest().getInstances().or(configuration.getMaxActiveOnDemandTasksPerRequest()); if (maxActiveOnDemandTasks > 0) { int activeTasksForRequest = activeTaskIdsForRequest.size(); LOG.debug("Running {} instances for request {}. Max is {}", activeTasksForRequest, taskRequest.getRequest().getId(), maxActiveOnDemandTasks); if (activeTasksForRequest >= maxActiveOnDemandTasks) { return true; } } } return false; } }
private boolean isPreemptibleTask(SingularityTaskRequest taskRequest) { // A long running task can be replaced + killed easily if (taskRequest.getRequest().getRequestType().isLongRunning()) { return true; } // A short, non-long-running task Optional<SingularityDeployStatistics> deployStatistics = deployManager.getDeployStatistics(taskRequest.getRequest().getId(), taskRequest.getDeploy().getId()); return deployStatistics.isPresent() && deployStatistics.get().getAverageRuntimeMillis().isPresent() && deployStatistics.get().getAverageRuntimeMillis().get() < configuration.getPreemptibleTaskMaxExpectedRuntimeMs(); }
private boolean requestTypeIsOnDemand(SingularityPendingTaskId taskId) { if (requestManager.getRequest(taskId.getRequestId()).isPresent()) { return requestManager.getRequest(taskId.getRequestId()).get().getRequest().getRequestType().equals(RequestType.ON_DEMAND); } return false; }
private boolean shouldScheduleTasks(SingularityRequest request, SingularityPendingRequest pendingRequest, Optional<SingularityPendingDeploy> maybePendingDeploy, Optional<SingularityRequestDeployState> maybeRequestDeployState) { if (request.isDeployable() && pendingRequest.getPendingType() == PendingType.NEW_DEPLOY && !maybePendingDeploy.isPresent()) { return false; } if (request.getRequestType() == RequestType.RUN_ONCE && pendingRequest.getPendingType() == PendingType.NEW_DEPLOY) { return true; } return isDeployInUse(maybeRequestDeployState, pendingRequest.getDeployId(), false); }
private void checkActiveRequest(SingularityRequestWithState requestWithState, Map<SingularityDeployKey, SingularityPendingTaskId> deployKeyToPendingTaskId, final long timestamp) { final SingularityRequest request = requestWithState.getRequest(); if (request.getRequestType() == RequestType.ON_DEMAND || request.getRequestType() == RequestType.RUN_ONCE) { return; // There's no situation where we'd want to schedule an On Demand or Run Once request at startup, so don't even bother with them. } Optional<SingularityRequestDeployState> requestDeployState = deployManager.getRequestDeployState(request.getId()); if (!requestDeployState.isPresent() || !requestDeployState.get().getActiveDeploy().isPresent()) { LOG.debug("No active deploy for {} - not scheduling on startup", request.getId()); return; } final String activeDeployId = requestDeployState.get().getActiveDeploy().get().getDeployId(); if (request.isScheduled()) { SingularityDeployKey deployKey = new SingularityDeployKey(request.getId(), activeDeployId); SingularityPendingTaskId pendingTaskId = deployKeyToPendingTaskId.get(deployKey); if (pendingTaskId != null && pendingTaskId.getCreatedAt() >= requestWithState.getTimestamp()) { LOG.info("Not rescheduling {} because {} is newer than {}", request.getId(), pendingTaskId, requestWithState.getTimestamp()); return; } } requestManager.addToPendingQueue(new SingularityPendingRequest(request.getId(), activeDeployId, timestamp, Optional.<String> absent(), PendingType.STARTUP, Optional.<Boolean> absent(), Optional.<String> absent())); }
private boolean shouldRetryImmediately(SingularityRequest request, SingularityDeployStatistics deployStatistics, Optional<SingularityTask> task) { if (!request.getNumRetriesOnFailure().isPresent()) { return false; } if (task.isPresent() && task.get().getTaskRequest().getPendingTask().getPendingTaskId().getPendingType() == PendingType.IMMEDIATE && request.getRequestType() == RequestType.SCHEDULED) { return false; // don't retry UI triggered scheduled jobs (UI triggered on-demand jobs are okay to retry though) } final int numRetriesInARow = deployStatistics.getNumSequentialRetries(); if (numRetriesInARow >= request.getNumRetriesOnFailure().get()) { LOG.debug("Request {} had {} retries in a row, not retrying again (num retries on failure: {})", request.getId(), numRetriesInARow, request.getNumRetriesOnFailure()); return false; } LOG.debug("Request {} had {} retries in a row - retrying again (num retries on failure: {})", request.getId(), numRetriesInARow, request.getNumRetriesOnFailure()); return true; }
public Optional<SingularityPendingTask> deleteScheduledTask(String taskId, SingularityUser user) { Optional<SingularityPendingTask> maybePendingTask = taskManager.getPendingTask(getPendingTaskIdFromStr(taskId)); if (maybePendingTask.isPresent()) { SingularityPendingTaskId pendingTaskId = maybePendingTask.get().getPendingTaskId(); Optional<SingularityRequestWithState> maybeRequest = requestManager.getRequest(pendingTaskId.getRequestId()); checkNotFound(maybeRequest.isPresent(), "Couldn't find: " + taskId); SingularityRequest request = maybeRequest.get().getRequest(); authorizationHelper.checkForAuthorizationByRequestId(request.getId(), user, SingularityAuthorizationScope.WRITE); checkBadRequest(request.getRequestType() == RequestType.ON_DEMAND, "Only ON_DEMAND tasks may be deleted."); taskManager.markPendingTaskForDeletion(pendingTaskId); } return maybePendingTask; }
private int scheduleTasks(SingularityRequest request, RequestState state, SingularityDeployStatistics deployStatistics, SingularityPendingRequest pendingRequest, List<SingularityTaskId> matchingTaskIds, Optional<SingularityPendingDeploy> maybePendingDeploy) { if (request.getRequestType() != RequestType.ON_DEMAND) { deleteScheduledTasks(leaderCache.getPendingTasks(), pendingRequest);
private List<SingularityTaskRequest> checkForStaleScheduledTasks(List<SingularityPendingTask> pendingTasks, List<SingularityTaskRequest> taskRequests) { final Set<String> foundPendingTaskId = Sets.newHashSetWithExpectedSize(taskRequests.size()); final Set<String> requestIds = Sets.newHashSetWithExpectedSize(taskRequests.size()); for (SingularityTaskRequest taskRequest : taskRequests) { foundPendingTaskId.add(taskRequest.getPendingTask().getPendingTaskId().getId()); requestIds.add(taskRequest.getRequest().getId()); } for (SingularityPendingTask pendingTask : pendingTasks) { if (!foundPendingTaskId.contains(pendingTask.getPendingTaskId().getId())) { LOG.info("Removing stale pending task {}", pendingTask.getPendingTaskId()); taskManager.deletePendingTask(pendingTask.getPendingTaskId()); } } // TODO this check isn't necessary if we keep track better during deploys final Map<String, SingularityRequestDeployState> deployStates = deployManager.getRequestDeployStatesByRequestIds(requestIds); final List<SingularityTaskRequest> taskRequestsWithValidDeploys = Lists.newArrayListWithCapacity(taskRequests.size()); for (SingularityTaskRequest taskRequest : taskRequests) { SingularityRequestDeployState requestDeployState = deployStates.get(taskRequest.getRequest().getId()); if (!matchesDeploy(requestDeployState, taskRequest) && !(taskRequest.getRequest().getRequestType() == RequestType.RUN_ONCE)) { LOG.info("Removing stale pending task {} because the deployId did not match active/pending deploys {}", taskRequest.getPendingTask().getPendingTaskId(), requestDeployState); taskManager.deletePendingTask(taskRequest.getPendingTask().getPendingTaskId()); } else { taskRequestsWithValidDeploys.add(taskRequest); } } return taskRequestsWithValidDeploys; }
validator.checkActionEnabled(SingularityAction.BOUNCE_REQUEST); checkBadRequest(requestWithState.getRequest().isLongRunning(), "Can not bounce a %s request (%s)", requestWithState.getRequest().getRequestType(), requestWithState);
@Test public void itAllowsWorkerToServiceTransitionIfNotLoadBalanced() { SingularityRequest request = new SingularityRequestBuilder("test", RequestType.WORKER) .build(); SingularityRequest newRequest = new SingularityRequestBuilder("test", RequestType.SERVICE) .build(); SingularityRequest result = validator.checkSingularityRequest(newRequest, Optional.of(request), Optional.absent(), Optional.absent()); Assert.assertEquals(newRequest.getRequestType(), result.getRequestType()); }
validator.checkActionEnabled(SingularityAction.BOUNCE_REQUEST); checkBadRequest(newRequest.isLongRunning(), "Can not bounce a %s request (%s)", newRequest.getRequestType(), newRequest); checkConflict(oldRequestWithState.getState() != RequestState.PAUSED, "Request %s is paused. Unable to bounce (it must be manually unpaused first)", newRequest.getId()); checkConflict(!requestManager.cleanupRequestExists(newRequest.getId(), RequestCleanupType.BOUNCE), "Request %s is already bouncing cannot bounce again", newRequest.getId());
@Test public void testSingularityRequestTypeMigration() throws Exception { metadataManager.setZkDataVersion("8"); final List<String> owners = ImmutableList.of("foo1@bar.com", "foo2@bar.com"); final SingularityRequestTypeMigration.OldSingularityRequest oldOnDemandRequest = new SingularityRequestTypeMigration.OldSingularityRequest("old-on-demand", null, Optional.<String>absent(), Optional.of(false), Optional.<Boolean>absent()); final SingularityRequestTypeMigration.OldSingularityRequest oldWorkerRequest = new SingularityRequestTypeMigration.OldSingularityRequest("old-worker", null, Optional.<String>absent(), Optional.of(true), Optional.<Boolean>absent()); final SingularityRequestTypeMigration.OldSingularityRequest oldScheduledRequest = new SingularityRequestTypeMigration.OldSingularityRequest("old-scheduled", null, Optional.of("0 0 0 0 0"), Optional.<Boolean>absent(), Optional.<Boolean>absent()); final SingularityRequestTypeMigration.OldSingularityRequest oldServiceRequest = new SingularityRequestTypeMigration.OldSingularityRequest("old-service", null, Optional.<String>absent(), Optional.of(true), Optional.of(true)); oldOnDemandRequest.setUnknownField("owners", owners); // save old-style requests to ZK curator.create().creatingParentsIfNeeded().forPath("/requests/all/" + oldOnDemandRequest.getId(), objectMapper.writeValueAsBytes(new SingularityRequestTypeMigration.OldSingularityRequestWithState(oldOnDemandRequest, RequestState.ACTIVE, System.currentTimeMillis()))); curator.create().creatingParentsIfNeeded().forPath("/requests/all/" + oldWorkerRequest.getId(), objectMapper.writeValueAsBytes(new SingularityRequestTypeMigration.OldSingularityRequestWithState(oldWorkerRequest, RequestState.ACTIVE, System.currentTimeMillis()))); curator.create().creatingParentsIfNeeded().forPath("/requests/all/" + oldScheduledRequest.getId(), objectMapper.writeValueAsBytes(new SingularityRequestTypeMigration.OldSingularityRequestWithState(oldScheduledRequest, RequestState.ACTIVE, System.currentTimeMillis()))); curator.create().creatingParentsIfNeeded().forPath("/requests/all/" + oldServiceRequest.getId(), objectMapper.writeValueAsBytes(new SingularityRequestTypeMigration.OldSingularityRequestWithState(oldServiceRequest, RequestState.ACTIVE, System.currentTimeMillis()))); // run ZK migration migrationRunner.checkMigrations(); // assert that the migration properly set the requestType field Assert.assertEquals(RequestType.ON_DEMAND, requestManager.getRequest(oldOnDemandRequest.getId()).get().getRequest().getRequestType()); Assert.assertEquals(RequestType.WORKER, requestManager.getRequest(oldWorkerRequest.getId()).get().getRequest().getRequestType()); Assert.assertEquals(RequestType.SCHEDULED, requestManager.getRequest(oldScheduledRequest.getId()).get().getRequest().getRequestType()); Assert.assertEquals(RequestType.SERVICE, requestManager.getRequest(oldServiceRequest.getId()).get().getRequest().getRequestType()); // assert that the migration properly carried over any additional fields on the request Assert.assertEquals(Optional.of(owners), requestManager.getRequest(oldOnDemandRequest.getId()).get().getRequest().getOwners()); }