scheduler.checkForDecomissions(); scheduler.drainPendingQueue();
final Map<SingularitySlave, MachineState> slaves = getDefaultMap(slaveManager.getObjectsFiltered(MachineState.STARTING_DECOMMISSION)); cleanupTaskDueToDecomission(requestIdsToUserToReschedule, matchingTaskIds, activeTask, slave); foundTask = true; final Map<SingularityRack, MachineState> racks = getDefaultMap(rackManager.getObjectsFiltered(MachineState.STARTING_DECOMMISSION)); cleanupTaskDueToDecomission(requestIdsToUserToReschedule, matchingTaskIds, maybeTask.get(), rack); changeState(slaves, slaveManager); changeState(racks, rackManager);
private void handlePendingRequestsForDeployKey(AtomicInteger obsoleteRequests, AtomicInteger heldForScheduledActiveTask, AtomicInteger totalNewScheduledTasks, SingularityDeployKey deployKey, List<SingularityPendingRequest> pendingRequestsForDeploy) { final String requestId = deployKey.getRequestId(); final Optional<SingularityRequestWithState> maybeRequest = requestManager.getRequest(requestId); final SingularityDeployStatistics deployStatistics = getDeployStatistics(deployKey.getRequestId(), deployKey.getDeployId()); if (!isRequestActive(maybeRequest)) { LOG.debug("Pending request {} was obsolete (request {})", requestId, SingularityRequestWithState.getRequestState(maybeRequest)); obsoleteRequests.getAndIncrement(); List<SingularityTaskId> matchingTaskIds = getMatchingTaskIds(request.getRequest(), deployKey); int scheduledTasks = 0; for (SingularityPendingRequest pendingRequest : pendingRequestsForDeploy) { final SingularityRequest updatedRequest = updatedRequest(maybePendingDeploy, pendingRequest, request); if (!shouldScheduleTasks(updatedRequest, pendingRequest, maybePendingDeploy, maybeRequestDeployState)) { LOG.debug("Pending request {} was obsolete (request {})", pendingRequest, SingularityRequestWithState.getRequestState(maybeRequest)); obsoleteRequests.getAndIncrement(); int missingInstances = getNumMissingInstances(matchingTaskIds, updatedRequest, pendingRequest, maybePendingDeploy); if (missingInstances == 0 && !matchingTaskIds.isEmpty() && updatedRequest.isScheduled() && pendingRequest.getPendingType() == PendingType.NEW_DEPLOY) { LOG.trace("Holding pending request {} because it is scheduled and has an active task", pendingRequest); RequestState requestState = checkCooldown(request.getState(), request.getRequest(), deployStatistics); scheduledTasks += scheduleTasks(request.getRequest(), requestState, deployStatistics, pendingRequest, matchingTaskIds, maybePendingDeploy); requestManager.deletePendingRequest(pendingRequest); } else if (pendingRequest.getPendingType() == PendingType.IMMEDIATE) { effectivePendingRequests.add(pendingRequest); RequestState requestState = checkCooldown(request.getState(), request.getRequest(), deployStatistics);
List<SingularityTaskId> matchingTaskIds, Optional<SingularityPendingDeploy> maybePendingDeploy) { if (request.getRequestType() != RequestType.ON_DEMAND) { deleteScheduledTasks(leaderCache.getPendingTasks(), pendingRequest); final int numMissingInstances = getNumMissingInstances(matchingTaskIds, request, pendingRequest, maybePendingDeploy); schedule(numMissingInstances, matchingTaskIds, request, state, deployStatistics, pendingRequest, maybePendingDeploy); } else if (numMissingInstances < 0) { final long now = System.currentTimeMillis(); rebalanceRacks(request, state, deployStatistics, pendingRequest, maybePendingDeploy, remainingActiveTasks); rebalanceAttributeDistribution(request, state, deployStatistics, pendingRequest, maybePendingDeploy, remainingActiveTasks);
protected void saveAndSchedule(SingularityRequestBuilder bldr) { SingularityRequest build = bldr.build(); requestManager.activate(build, RequestHistoryType.UPDATED, System.currentTimeMillis(), Optional.<String> absent(), Optional.<String> absent()); requestManager.addToPendingQueue(new SingularityPendingRequest(build.getId(), firstDeployId, System.currentTimeMillis(), Optional.<String> absent(), PendingType.UPDATED_REQUEST, Optional.<Boolean> absent(), Optional.<String> absent())); scheduler.drainPendingQueue(); }
@Timed public void handleCompletedTask(Optional<SingularityTask> task, SingularityTaskId taskId, boolean wasActive, long timestamp, ExtendedTaskState state, SingularityCreateResult taskHistoryUpdateCreateResult, Protos.TaskStatus status) { final SingularityDeployStatistics deployStatistics = getDeployStatistics(taskId.getRequestId(), taskId.getDeployId()); final Optional<PendingType> scheduleResult = handleCompletedTaskWithStatistics(task, taskId, timestamp, state, deployStatistics, taskHistoryUpdateCreateResult, status); updateDeployStatistics(deployStatistics, taskId, task, timestamp, state, scheduleResult);
private List<SingularityTaskRequestHolder> getSortedDueTaskRequests() { final List<SingularityTaskRequest> taskRequests = taskPrioritizer.getSortedDueTasks(scheduler.getDueTasks()); taskRequests.forEach((taskRequest) -> LOG.trace("Task {} is due", taskRequest.getPendingTask().getPendingTaskId())); taskPrioritizer.removeTasksAffectedByPriorityFreeze(taskRequests); return taskRequests.stream() .map((taskRequest) -> new SingularityTaskRequestHolder(taskRequest, defaultResources, defaultCustomExecutorResources)) .collect(Collectors.toList()); }
@Timed public List<SingularityTaskRequest> getDueTasks() { final List<SingularityPendingTask> tasks = taskManager.getPendingTasks(); final long now = System.currentTimeMillis(); final List<SingularityPendingTask> dueTasks = Lists.newArrayListWithCapacity(tasks.size()); for (SingularityPendingTask task : tasks) { if (task.getPendingTaskId().getNextRunAt() <= now) { dueTasks.add(task); } } final List<SingularityTaskRequest> dueTaskRequests = taskRequestManager.getTaskRequests(dueTasks); return checkForStaleScheduledTasks(dueTasks, dueTaskRequests); }
@Test public void testQueueMultipleOneOffs() { SingularityRequestBuilder bldr = new SingularityRequestBuilder(requestId, RequestType.ON_DEMAND); requestResource.postRequest(bldr.build(), singularityUser); deploy("on_demand_deploy"); deployChecker.checkDeploys(); requestManager.addToPendingQueue(new SingularityPendingRequest(requestId, "on_demand_deploy", System.currentTimeMillis(), Optional.<String>absent(), PendingType.ONEOFF, Optional.<List<String>>absent(), Optional.<String>absent(), Optional.<Boolean>absent(), Optional.<String>absent(), Optional.<String>absent())); requestManager.addToPendingQueue(new SingularityPendingRequest(requestId, "on_demand_deploy", System.currentTimeMillis(), Optional.<String>absent(), PendingType.ONEOFF, Optional.<List<String>>absent(), Optional.<String>absent(), Optional.<Boolean>absent(), Optional.<String>absent(), Optional.<String>absent())); scheduler.drainPendingQueue(); Assert.assertEquals(2, taskManager.getPendingTaskIds().size()); }
List<SingularityTaskId> matchingTaskIds, Optional<SingularityPendingDeploy> maybePendingDeploy) { if (request.getRequestType() != RequestType.ON_DEMAND) { deleteScheduledTasks(leaderCache.getPendingTasks(), pendingRequest); final int numMissingInstances = getNumMissingInstances(matchingTaskIds, request, pendingRequest, maybePendingDeploy); schedule(numMissingInstances, matchingTaskIds, request, state, deployStatistics, pendingRequest, maybePendingDeploy); } else if (numMissingInstances < 0) { final long now = System.currentTimeMillis(); rebalanceRacks(request, state, deployStatistics, pendingRequest, maybePendingDeploy, remainingActiveTasks); rebalanceAttributeDistribution(request, state, deployStatistics, pendingRequest, maybePendingDeploy, remainingActiveTasks);
@Timed public void handleCompletedTask(Optional<SingularityTask> task, SingularityTaskId taskId, boolean wasActive, long timestamp, ExtendedTaskState state, SingularityCreateResult taskHistoryUpdateCreateResult, Protos.TaskStatus status) { final SingularityDeployStatistics deployStatistics = getDeployStatistics(taskId.getRequestId(), taskId.getDeployId()); final Optional<PendingType> scheduleResult = handleCompletedTaskWithStatistics(task, taskId, timestamp, state, deployStatistics, taskHistoryUpdateCreateResult, status); updateDeployStatistics(deployStatistics, taskId, task, timestamp, state, scheduleResult);
private List<SingularityTaskRequestHolder> getSortedDueTaskRequests() { final List<SingularityTaskRequest> taskRequests = taskPrioritizer.getSortedDueTasks(scheduler.getDueTasks()); taskRequests.forEach((taskRequest) -> LOG.trace("Task {} is due", taskRequest.getPendingTask().getPendingTaskId())); taskPrioritizer.removeTasksAffectedByPriorityFreeze(taskRequests); return taskRequests.stream() .map((taskRequest) -> new SingularityTaskRequestHolder(taskRequest, defaultResources, defaultCustomExecutorResources)) .collect(Collectors.toList()); }
@Timed public List<SingularityTaskRequest> getDueTasks() { final List<SingularityPendingTask> tasks = taskManager.getPendingTasks(); final long now = System.currentTimeMillis(); final List<SingularityPendingTask> dueTasks = Lists.newArrayListWithCapacity(tasks.size()); for (SingularityPendingTask task : tasks) { if (task.getPendingTaskId().getNextRunAt() <= now) { dueTasks.add(task); } } final List<SingularityTaskRequest> dueTaskRequests = taskRequestManager.getTaskRequests(dueTasks); return checkForStaleScheduledTasks(dueTasks, dueTaskRequests); }
@Test public void testMaxOnDemandTasks() { SingularityRequestBuilder bldr = new SingularityRequestBuilder(requestId, RequestType.ON_DEMAND); bldr.setInstances(Optional.of(1)); requestResource.postRequest(bldr.build(), singularityUser); deploy("on_demand_deploy"); deployChecker.checkDeploys(); requestManager.addToPendingQueue(new SingularityPendingRequest(requestId, "on_demand_deploy", System.currentTimeMillis(), Optional.<String>absent(), PendingType.ONEOFF, Optional.<List<String>>absent(), Optional.<String>absent(), Optional.<Boolean>absent(), Optional.<String>absent(), Optional.<String>absent())); requestManager.addToPendingQueue(new SingularityPendingRequest(requestId, "on_demand_deploy", System.currentTimeMillis(), Optional.<String>absent(), PendingType.ONEOFF, Optional.<List<String>>absent(), Optional.<String>absent(), Optional.<Boolean>absent(), Optional.<String>absent(), Optional.<String>absent())); scheduler.drainPendingQueue(); resourceOffers(); Assert.assertEquals(1, taskManager.getActiveTaskIds().size()); }
private void handlePendingRequestsForDeployKey(AtomicInteger obsoleteRequests, AtomicInteger heldForScheduledActiveTask, AtomicInteger totalNewScheduledTasks, SingularityDeployKey deployKey, List<SingularityPendingRequest> pendingRequestsForDeploy) { final String requestId = deployKey.getRequestId(); final Optional<SingularityRequestWithState> maybeRequest = requestManager.getRequest(requestId); final SingularityDeployStatistics deployStatistics = getDeployStatistics(deployKey.getRequestId(), deployKey.getDeployId()); if (!isRequestActive(maybeRequest)) { LOG.debug("Pending request {} was obsolete (request {})", requestId, SingularityRequestWithState.getRequestState(maybeRequest)); obsoleteRequests.getAndIncrement(); List<SingularityTaskId> matchingTaskIds = getMatchingTaskIds(request.getRequest(), deployKey); int scheduledTasks = 0; for (SingularityPendingRequest pendingRequest : pendingRequestsForDeploy) { final SingularityRequest updatedRequest = updatedRequest(maybePendingDeploy, pendingRequest, request); if (!shouldScheduleTasks(updatedRequest, pendingRequest, maybePendingDeploy, maybeRequestDeployState)) { LOG.debug("Pending request {} was obsolete (request {})", pendingRequest, SingularityRequestWithState.getRequestState(maybeRequest)); obsoleteRequests.getAndIncrement(); int missingInstances = getNumMissingInstances(matchingTaskIds, updatedRequest, pendingRequest, maybePendingDeploy); if (missingInstances == 0 && !matchingTaskIds.isEmpty() && updatedRequest.isScheduled() && pendingRequest.getPendingType() == PendingType.NEW_DEPLOY) { LOG.trace("Holding pending request {} because it is scheduled and has an active task", pendingRequest); RequestState requestState = checkCooldown(request.getState(), request.getRequest(), deployStatistics); scheduledTasks += scheduleTasks(request.getRequest(), requestState, deployStatistics, pendingRequest, matchingTaskIds, maybePendingDeploy); requestManager.deletePendingRequest(pendingRequest); } else if (pendingRequest.getPendingType() == PendingType.IMMEDIATE) { effectivePendingRequests.add(pendingRequest); RequestState requestState = checkCooldown(request.getState(), request.getRequest(), deployStatistics);
@Test public void testValidTransitionToDecommissioned() { initRequest(); initFirstDeploy(); requestResource.postRequest(request.toBuilder().setInstances(Optional.of(2)).build(), singularityUser); scheduler.drainPendingQueue(); resourceOffers(1); SingularitySlave slave = slaveManager.getObjects().get(0); slaveResource.decommissionSlave(singularityUser, slave.getId(), new SingularityMachineChangeRequest(Optional.of(1L), Optional.absent(), Optional.absent(), Optional.of(MachineState.DECOMMISSIONED), Optional.of(true))); Assert.assertEquals(MachineState.STARTING_DECOMMISSION, slaveManager.getObjects().get(0).getCurrentState().getState()); scheduler.checkForDecomissions(); scheduler.drainPendingQueue(); Assert.assertEquals(TaskCleanupType.DECOMISSIONING, taskManager.getCleanupTasks().get(0).getCleanupType()); expiringUserActionPoller.runActionOnPoll(); Assert.assertEquals(MachineState.DECOMMISSIONED, slaveManager.getObjects().get(0).getCurrentState().getState()); Assert.assertEquals(TaskCleanupType.DECOMMISSION_TIMEOUT, taskManager.getCleanupTasks().get(0).getCleanupType()); }
final Map<SingularitySlave, MachineState> slaves = getDefaultMap(slaveManager.getObjectsFiltered(MachineState.STARTING_DECOMMISSION)); cleanupTaskDueToDecomission(requestIdsToUserToReschedule, matchingTaskIds, activeTask, slave); foundTask = true; final Map<SingularityRack, MachineState> racks = getDefaultMap(rackManager.getObjectsFiltered(MachineState.STARTING_DECOMMISSION)); cleanupTaskDueToDecomission(requestIdsToUserToReschedule, matchingTaskIds, maybeTask.get(), rack); changeState(slaves, slaveManager); changeState(racks, rackManager);
protected void killKilledTasks() { for (SingularityKilledTaskIdRecord killed : taskManager.getKilledTaskIdRecords()) { statusUpdate(taskManager.getTask(killed.getTaskId()).get(), TaskState.TASK_KILLED); } scheduler.drainPendingQueue(); }
@Test public void testDecommissionDoesntKillPendingDeploy() { initRequest(); deployResource.deploy(new SingularityDeployRequest(new SingularityDeployBuilder(requestId, "d1").setCommand(Optional.of("cmd")).build(), Optional.absent(), Optional.absent()), singularityUser); scheduler.drainPendingQueue(); deployChecker.checkDeploys(); resourceOffers(); Assert.assertEquals(1, taskManager.getNumActiveTasks()); slaveResource.decommissionSlave(singularityUser, taskManager.getActiveTasks().get(0).getAgentId().getValue(), null); scheduler.checkForDecomissions(); cleaner.drainCleanupQueue(); killKilledTasks(); Assert.assertEquals(1, taskManager.getNumActiveTasks()); Assert.assertEquals(1, taskManager.getNumCleanupTasks()); Assert.assertEquals(0, taskManager.getKilledTaskIdRecords().size()); configuration.setPendingDeployHoldTaskDuringDecommissionMillis(1); try { Thread.sleep(2); } catch (InterruptedException e) {} cleaner.drainCleanupQueue(); killKilledTasks(); Assert.assertEquals(0, taskManager.getNumActiveTasks()); Assert.assertEquals(0, taskManager.getNumCleanupTasks()); }
@Test public void testScaleWithBounceDoesNotLaunchExtraInstances() { initRequest(); initFirstDeploy(); launchTask(request, firstDeploy, 1, TaskState.TASK_RUNNING); requestResource.scale(requestId, new SingularityScaleRequest(Optional.of(5), Optional.of(1L), Optional.absent(), Optional.absent(), Optional.absent(), Optional.of(true), Optional.absent(), Optional.absent()), singularityUser); Assert.assertEquals(1, requestManager.getCleanupRequests().size()); cleaner.drainCleanupQueue(); Assert.assertEquals(1, taskManager.getNumCleanupTasks()); scheduler.drainPendingQueue(); Assert.assertEquals(5, taskManager.getPendingTaskIds().size()); }