checkBadRequest(request.getId() != null && ! REQUEST_ID_ILLEGAL_PATTERN.matcher(request.getId()).find(), "Id cannot be null or contain characters other than [a-zA-Z0-9_]"); checkBadRequest(request.getRequestType() != null, "RequestType cannot be null or missing"); if (request.getOwners().isPresent()) { checkBadRequest(!request.getOwners().get().contains(null), "Request owners cannot contain null values"); checkBadRequest(request.getOwners().isPresent() && !request.getOwners().get().isEmpty(), "Request must have owners defined (this can be turned off in Singularity configuration)"); checkBadRequest(request.getId().length() <= maxRequestIdSize, "Request id must be less %s characters or less, it is %s (%s)", maxRequestIdSize, request.getId().length(), request.getId()); checkBadRequest(!request.getInstances().isPresent() || request.getInstances().get() > 0, "Instances must be greater than 0"); checkBadRequest(request.getInstancesSafe() <= maxInstancesPerRequest, "Instances (%s) be greater than %s (maxInstancesPerRequest in mesos configuration)", request.getInstancesSafe(), maxInstancesPerRequest); if (request.getTaskPriorityLevel().isPresent()) { checkBadRequest(request.getTaskPriorityLevel().get() >= 0 && request.getTaskPriorityLevel().get() <= 1, "Request taskPriorityLevel %s is invalid, must be between 0 and 1 (inclusive).", request.getTaskPriorityLevel().get()); if (request.isScheduled()) { checkBadRequest(request.getQuartzSchedule().isPresent() || request.getSchedule().isPresent(), "Specify at least one of schedule or quartzSchedule"); String originalSchedule = request.getQuartzScheduleSafe(); if (request.getScheduleType().or(ScheduleType.QUARTZ) != ScheduleType.RFC5545) { if (request.getQuartzSchedule().isPresent() && !request.getSchedule().isPresent()) { checkBadRequest(request.getScheduleType().or(ScheduleType.QUARTZ) == ScheduleType.QUARTZ, "If using quartzSchedule specify scheduleType QUARTZ or leave it blank"); if (request.getQuartzSchedule().isPresent() || (request.getScheduleType().isPresent() && request.getScheduleType().get() == ScheduleType.QUARTZ)) { quartzSchedule = originalSchedule; } else { checkBadRequest(request.getScheduleType().or(ScheduleType.CRON) == ScheduleType.CRON, "If not using quartzSchedule specify scheduleType CRON or leave it blank"); checkBadRequest(!request.getQuartzSchedule().isPresent(), "If using schedule type CRON do not specify quartzSchedule");
private boolean shouldReschedule(SingularityRequest newRequest, SingularityRequest oldRequest) { if (newRequest.getInstancesSafe() != oldRequest.getInstancesSafe()) { return true; } if (newRequest.isScheduled() && oldRequest.isScheduled()) { if (!newRequest.getQuartzScheduleSafe().equals(oldRequest.getQuartzScheduleSafe())) { return true; } } return false; }
private void checkForIllegalChanges(SingularityRequest request, SingularityRequest existingRequest) { if (request.getRequestType() != existingRequest.getRequestType()) { boolean validWorkerServiceTransition = (existingRequest.getRequestType() == RequestType.SERVICE && !existingRequest.isLoadBalanced() && request.getRequestType() == RequestType.WORKER) || (request.getRequestType() == RequestType.SERVICE && !request.isLoadBalanced() && existingRequest.getRequestType() == RequestType.WORKER); checkBadRequest(validWorkerServiceTransition, String.format("Request can not change requestType from %s to %s", existingRequest.getRequestType(), request.getRequestType())); } checkBadRequest(request.isLoadBalanced() == existingRequest.isLoadBalanced(), "Request can not change whether it is load balanced"); }
private void populateRequestEmailProperties(Map<String, Object> templateProperties, SingularityRequest request, SingularityEmailType emailType) { templateProperties.put("requestId", request.getId()); templateProperties.put("singularityRequestLink", mailTemplateHelpers.getSingularityRequestLink(request.getId())); templateProperties.put("requestAlwaysRunning", request.isAlwaysRunning()); templateProperties.put("requestRunOnce", request.getRequestType() == RequestType.RUN_ONCE); templateProperties.put("requestScheduled", request.isScheduled()); templateProperties.put("requestOneOff", request.isOneOff()); templateProperties.put("taskWillRetry", request.getNumRetriesOnFailure().or(0) > 0); templateProperties.put("numRetries", request.getNumRetriesOnFailure().or(0)); templateProperties.put("color", emailType.getColor()); }
if (!request.isOneOff() && !(request.getRequestType() == RequestType.RUN_ONCE)) { cleanupTasks(pendingDeploy, request, deployResult, tasksToKill); .setCmdLineArgsList(runNowRequest.getCommandLineArgs()) .setRunId(runNowRequest.getRunId().or(Optional.of(UUID.randomUUID().toString()))) .setSkipHealthchecks(runNowRequest.getSkipHealthchecks().or(request.getSkipHealthchecks())) .setMessage(runNowRequest.getMessage() .or(pendingDeploy.getDeployMarker().getMessage())) if (request.isScheduled()) { if (activeTasks.isEmpty()) { pendingType = PendingType.IMMEDIATE; } else if (!request.isLongRunning()) { if (request.getInstances().isPresent() && (activeTasks.size() + pendingTasks.size() < request.getInstances().get())) { pendingType = PendingType.ONEOFF; } else { } else if (!request.isDeployable() && !request.isOneOff()) { PendingType pendingType = canceledOr(deployResult.getDeployState(), PendingType.NEW_DEPLOY); requestManager.addToPendingQueue(new SingularityPendingRequest(request.getId(), pendingDeploy.getDeployMarker().getDeployId(), deployResult.getTimestamp(), pendingDeploy.getDeployMarker().getUser(), pendingType, deploy.isPresent() ? deploy.get().getSkipHealthchecksOnDeploy() : Optional.absent(), pendingDeploy.getDeployMarker().getMessage())); if (request.isDeployable() && !request.isOneOff()) { requestManager.markBounceComplete(request.getId()); requestManager.removeExpiringBounce(request.getId());
validator.checkActionEnabled(SingularityAction.SCALE_REQUEST); SingularityRequest newRequest = oldRequest.toBuilder().setInstances(scaleRequest.getInstances()).build(); validator.checkScale(newRequest, Optional.<Integer>absent()); checkBadRequest(oldRequest.getInstancesSafe() != newRequest.getInstancesSafe(), "Scale request has no affect on the # of instances (%s)", newRequest.getInstancesSafe()); String scaleMessage = String.format("Scaling from %d -> %d", oldRequest.getInstancesSafe(), newRequest.getInstancesSafe()); if (scaleRequest.getMessage().isPresent()) { scaleMessage = String.format("%s -- %s", scaleRequest.getMessage().get(), scaleMessage); if (scaleRequest.getBounce().or(newRequest.getBounceAfterScale().or(false))) { validator.checkActionEnabled(SingularityAction.BOUNCE_REQUEST); checkBadRequest(newRequest.isLongRunning(), "Can not bounce a %s request (%s)", newRequest.getRequestType(), newRequest); checkConflict(oldRequestWithState.getState() != RequestState.PAUSED, "Request %s is paused. Unable to bounce (it must be manually unpaused first)", newRequest.getId()); checkConflict(!requestManager.cleanupRequestExists(newRequest.getId(), RequestCleanupType.BOUNCE), "Request %s is already bouncing cannot bounce again", newRequest.getId()); System.currentTimeMillis(), scaleRequest, oldRequest.getInstances(), scaleRequest.getActionId().or(UUID.randomUUID().toString()), scaleRequest.getBounce())); } else { requestManager.deleteExpiringObject(SingularityExpiringScale.class, requestId); mailer.sendRequestScaledMail(newRequest, Optional.of(scaleRequest), oldRequest.getInstances(), user.getEmail());
if (request.isScheduled()) { if (pendingType == PendingType.IMMEDIATE || pendingType == PendingType.RETRY) { LOG.info("Scheduling requested immediate run of {}", request.getId()); } else { try { if (request.getScheduleTypeSafe() == ScheduleType.RFC5545) { final RFC5545Schedule rfc5545Schedule = new RFC5545Schedule(request.getSchedule().get()); nextRunAtDate = rfc5545Schedule.getNextValidTime(); scheduleFrom = new Date(rfc5545Schedule.getStartDateTime().getMillis()); } else { scheduleFrom = new Date(now); final CronExpression cronExpression = new CronExpression(request.getQuartzScheduleSafe()); if (request.getScheduleTimeZone().isPresent()) { cronExpression.setTimeZone(TimeZone.getTimeZone(request.getScheduleTimeZone().get())); LOG.trace("Calculating nextRunAtDate for {} (schedule: {}): {} (from: {})", request.getId(), request.getSchedule(), nextRunAtDate, scheduleFrom); LOG.trace("Scheduling next run of {} (schedule: {}) at {} (from: {})", request.getId(), request.getSchedule(), nextRunAtDate, scheduleFrom); } catch (ParseException | InvalidRecurrenceRuleException pe) { throw Throwables.propagate(pe); if (!request.isLongRunning() && pendingRequest.getRunAt().isPresent()) { nextRunAt = Math.max(nextRunAt, pendingRequest.getRunAt().get()); if (pendingType == PendingType.TASK_DONE && request.getWaitAtLeastMillisAfterTaskFinishesForReschedule().or(0L) > 0) { nextRunAt = Math.max(nextRunAt, now + request.getWaitAtLeastMillisAfterTaskFinishesForReschedule().get()); LOG.trace("Adjusted next run of {} to {} (by {}) due to waitAtLeastMillisAfterTaskFinishesForReschedule", request.getId(), nextRunAt,
@Override protected void handleExpiringObject(SingularityExpiringScale expiringObject, SingularityRequestWithState requestWithState, String message) { final SingularityRequest oldRequest = requestWithState.getRequest(); final SingularityRequest newRequest = oldRequest.toBuilder().setInstances(expiringObject.getRevertToInstances()).build(); try { Optional<SingularityBounceRequest> maybeBounceRequest = Optional.absent(); if (expiringObject.getBounce().or(false) || newRequest.getBounceAfterScale().or(false)) { LOG.info("Attempting to bounce request {} after expiring scale", newRequest.getId()); Optional<String> maybeActiveDeployId = deployManager.getInUseDeployId(newRequest.getId()); if (maybeActiveDeployId.isPresent()) { maybeBounceRequest = Optional.of(SingularityBounceRequest.defaultRequest()); } else { LOG.debug("No active deploy id present for request {}, not bouncing after expiring scale", newRequest.getId()); } } requestHelper.updateRequest(newRequest, Optional.of(oldRequest), requestWithState.getState(), Optional.of(RequestHistoryType.SCALE_REVERTED), expiringObject.getUser(), Optional.<Boolean>absent(), Optional.of(message), maybeBounceRequest); mailer.sendRequestScaledMail(newRequest, Optional.<SingularityScaleRequest>absent(), oldRequest.getInstances(), expiringObject.getUser()); } catch (WebApplicationException wae) { LOG.error("While trying to apply {} for {}", expiringObject, expiringObject.getRequestId(), wae); } }
public void checkScale(SingularityRequest request, Optional<Integer> previousScale) { SlavePlacement placement = request.getSlavePlacement().or(defaultSlavePlacement); if (placement != SlavePlacement.GREEDY && placement != SlavePlacement.OPTIMISTIC) { int currentActiveSlaveCount = slaveManager.getNumObjectsAtState(MachineState.ACTIVE); int requiredSlaveCount = request.getInstancesSafe(); if (previousScale.isPresent() && placement == SlavePlacement.SEPARATE_BY_REQUEST) { requiredSlaveCount += previousScale.get(); } checkBadRequest(currentActiveSlaveCount >= requiredSlaveCount, "Not enough active slaves to successfully complete a bounce of request %s (minimum required: %s, current: %s). Consider deploying, or changing the slave placement strategy instead.", request.getId(), requiredSlaveCount, currentActiveSlaveCount); } }
subject = String.format("%s notifications for %s are being rate limited", emailType.name(), request.getId()); body = Jade4J.render(rateLimitedTemplate, getRateLimitTemplateProperties(request, emailType)); if (destination.contains(SingularityEmailDestination.OWNERS) && request.getOwners().isPresent() && !request.getOwners().get().isEmpty()) { toList.addAll(request.getOwners().get());
private boolean isTooManyInstancesForRequest(SingularityTaskRequest taskRequest, List<SingularityTaskId> activeTaskIdsForRequest) { if (taskRequest.getRequest().getRequestType() == RequestType.ON_DEMAND) { int maxActiveOnDemandTasks = taskRequest.getRequest().getInstances().or(configuration.getMaxActiveOnDemandTasksPerRequest()); if (maxActiveOnDemandTasks > 0) { int activeTasksForRequest = activeTaskIdsForRequest.size(); LOG.debug("Running {} instances for request {}. Max is {}", activeTasksForRequest, taskRequest.getRequest().getId(), maxActiveOnDemandTasks); if (activeTasksForRequest >= maxActiveOnDemandTasks) { return true; } } } return false; } }
@Test public void testCronScheduleChanges() throws Exception { final String requestId = "test-change-cron"; final String oldSchedule = "*/5 * * * *"; final String oldScheduleQuartz = "0 */5 * * * ?"; final String newSchedule = "*/30 * * * *"; final String newScheduleQuartz = "0 */30 * * * ?"; SingularityRequest request = new SingularityRequestBuilder(requestId, RequestType.SCHEDULED) .setSchedule(Optional.of(oldSchedule)) .build(); request = validator.checkSingularityRequest(request, Optional.<SingularityRequest>absent(), Optional.<SingularityDeploy>absent(), Optional.<SingularityDeploy>absent()); saveRequest(request); Assert.assertEquals(oldScheduleQuartz, requestManager.getRequest(requestId).get().getRequest().getQuartzScheduleSafe()); initAndFinishDeploy(request, "1"); scheduler.drainPendingQueue(); final SingularityRequest newRequest = request.toBuilder() .setSchedule(Optional.of(newSchedule)) .setQuartzSchedule(Optional.<String>absent()) .build(); final SingularityDeploy newDeploy = new SingularityDeployBuilder(request.getId(), "2").setCommand(Optional.of("sleep 100")).build(); deployResource.deploy(new SingularityDeployRequest(newDeploy, Optional.absent(), Optional.absent(), Optional.of(newRequest)), singularityUser); deployChecker.checkDeploys(); scheduler.drainPendingQueue(); Assert.assertEquals(newScheduleQuartz, requestManager.getRequest(requestId).get().getRequest().getQuartzScheduleSafe()); }
SingularityRunNowRequest runNowRequest = fillRunNowRequest(maybeRunNowRequest); PendingType pendingType; if (request.isScheduled()) { pendingType = PendingType.IMMEDIATE; checkConflict(activeTasks.isEmpty(), "Cannot request immediate run of a scheduled job which is currently running (%s)", activeTasks); } else if (request.isOneOff()) { pendingType = PendingType.ONEOFF; if (request.getInstances().isPresent()) { checkRateLimited( activeTasks.size() + pendingTasks.size() < request.getInstances().get(), "No more than %s tasks allowed to run concurrently for request %s (%s active, %s pending)", request.getInstances().get(), request, activeTasks.size(), pendingTasks.size()); request.getId(), deployId, System.currentTimeMillis(),
private boolean isPreemptibleTask(SingularityTaskRequest taskRequest) { // A long running task can be replaced + killed easily if (taskRequest.getRequest().getRequestType().isLongRunning()) { return true; } // A short, non-long-running task Optional<SingularityDeployStatistics> deployStatistics = deployManager.getDeployStatistics(taskRequest.getRequest().getId(), taskRequest.getDeploy().getId()); return deployStatistics.isPresent() && deployStatistics.get().getAverageRuntimeMillis().isPresent() && deployStatistics.get().getAverageRuntimeMillis().get() < configuration.getPreemptibleTaskMaxExpectedRuntimeMs(); }
(!(taskCleanup.getCleanupType() == TaskCleanupType.PAUSING) || request.isLongRunning())) { LOG.debug("Killing a task {} immediately because the request was paused", taskCleanup); return true; if (!request.isLongRunning()) { final long timeSinceCleanup = System.currentTimeMillis() - taskCleanup.getTimestamp(); final long maxWaitTime = request.getKillOldNonLongRunningTasksAfterMillis().or(killNonLongRunningTasksInCleanupAfterMillis); final boolean tooOld = (maxWaitTime < 1) || (timeSinceCleanup > maxWaitTime); final String requestId = request.getId(); return shouldKillIncrementalDeployCleanupTask(request, taskCleanup, matchingTasksDeployId, matchingTasks, key, incrementalCleaningTasks); } else { if (matchingTasks.size() < request.getInstancesSafe()) { LOG.trace("Not killing a task {} yet, only {} matching out of a required {}", taskCleanup, matchingTasks.size(), request.getInstancesSafe()); return false; LOG.debug("Killing a task {}, at least {} replacement tasks are healthy [{}]", taskCleanup, request.getInstancesSafe(), matchingTasks); return true; case WAITING:
validator.checkScale(request, Optional.of(taskManager.getActiveTaskIdsForRequest(request.getId()).size())); checkConflict(requestWithState.getState() != RequestState.PAUSED, "Request %s is paused. Unable to deploy (it must be manually unpaused first)", requestWithState.getRequest().getId()); if (request.isLongRunning()) { deployProgress = Optional.of(new SingularityDeployProgress( Math.min(deploy.getDeployInstanceCountPerStep().or(request.getInstancesSafe()), request.getInstancesSafe()), 0, deploy.getDeployInstanceCountPerStep().or(request.getInstancesSafe()), deploy.getDeployStepWaitTimeMs().or(configuration.getDefaultDeployStepWaitTimeMs()), false, if (request.isDeployable() && !(requestWithState.getState() == RequestState.PAUSED && configuration.isAllowDeployOfPausedRequests())) { requestManager.addToPendingQueue(new SingularityPendingRequest(requestId, deployMarker.getDeployId(), now, deployUser, PendingType.NEW_DEPLOY, deployRequest.getDeploy().getSkipHealthchecksOnDeploy(), deployRequest.getMessage()));
private void updatePossiblyUnderProvisionedAndOverProvisionedIds(SingularityRequestWithState requestWithState, Map<String, Long> numInstances, List<String> overProvisionedRequestIds, Set<String> possiblyUnderProvisionedRequestIds) { if (requestWithState.getState().isRunnable() && requestWithState.getRequest().isAlwaysRunning()) { SingularityRequest request = requestWithState.getRequest(); final int expectedInstances = request.getInstancesSafe(); final Long numActualInstances = numInstances.get(request.getId()); if (numActualInstances == null || numActualInstances < expectedInstances) { possiblyUnderProvisionedRequestIds.add(request.getId()); } else if (numActualInstances > expectedInstances) { overProvisionedRequestIds.add(request.getId()); } } }
validator.checkActionEnabled(SingularityAction.BOUNCE_REQUEST); checkBadRequest(requestWithState.getRequest().isLongRunning(), "Can not bounce a %s request (%s)", requestWithState.getRequest().getRequestType(), requestWithState); checkConflict(requestWithState.getState() != RequestState.PAUSED, "Request %s is paused. Unable to bounce (it must be manually unpaused first)", requestWithState.getRequest().getId());
private boolean shouldRetryImmediately(SingularityRequest request, SingularityDeployStatistics deployStatistics, Optional<SingularityTask> task) { if (!request.getNumRetriesOnFailure().isPresent()) { return false; } if (task.isPresent() && task.get().getTaskRequest().getPendingTask().getPendingTaskId().getPendingType() == PendingType.IMMEDIATE && request.getRequestType() == RequestType.SCHEDULED) { return false; // don't retry UI triggered scheduled jobs (UI triggered on-demand jobs are okay to retry though) } final int numRetriesInARow = deployStatistics.getNumSequentialRetries(); if (numRetriesInARow >= request.getNumRetriesOnFailure().get()) { LOG.debug("Request {} had {} retries in a row, not retrying again (num retries on failure: {})", request.getId(), numRetriesInARow, request.getNumRetriesOnFailure()); return false; } LOG.debug("Request {} had {} retries in a row - retrying again (num retries on failure: {})", request.getId(), numRetriesInARow, request.getNumRetriesOnFailure()); return true; }