public SingularityPendingRequest build() { return new SingularityPendingRequest( requestId, deployId, timestamp, user, pendingType, cmdLineArgsList, runId, skipHealthchecks, message, actionId, resources, s3UploaderAdditionalFiles, runAsUserOverride, envOverrides, extraArtifacts, runAt ); }
private void checkPendingRequests() { try { if (curator.checkExists().forPath(REQUEST_PENDING_PATH) == null) { return; } } catch (Exception e) { throw Throwables.propagate(e); } try { for (String pendingRequest : curator.getChildren().forPath(REQUEST_PENDING_PATH)) { SingularityPendingRequestPrevious previous = objectMapper.readValue(curator.getData().forPath(ZKPaths.makePath(REQUEST_PENDING_PATH, pendingRequest)), SingularityPendingRequestPrevious.class); SingularityPendingRequest newRequest = new SingularityPendingRequest(previous.requestId, previous.deployId, previous.timestamp, previous.user, previous.pendingType, getCmdLineArgs(previous.cmdLineArgs), Optional.<String> absent(), Optional.<Boolean> absent(), Optional.<String> absent(), Optional.<String> absent()); LOG.info("Re-saving {}", newRequest); curator.setData().forPath(ZKPaths.makePath(REQUEST_PENDING_PATH, pendingRequest), pendingRequestTranscoder.toBytes(newRequest)); } } catch (Exception e) { throw Throwables.propagate(e); } }
public long unpause(SingularityRequest request, Optional<String> user, Optional<String> message, Optional<Boolean> skipHealthchecks) { mailer.sendRequestUnpausedMail(request, user, message); Optional<String> maybeDeployId = deployManager.getInUseDeployId(request.getId()); final long now = System.currentTimeMillis(); requestManager.unpause(request, now, user, message); if (maybeDeployId.isPresent() && !request.isOneOff()) { requestManager.addToPendingQueue(new SingularityPendingRequest(request.getId(), maybeDeployId.get(), now, user, PendingType.UNPAUSED, skipHealthchecks, message)); } return now; }
private void maybeUpdatePendingRequest(SingularityPendingDeploy pendingDeploy, Optional<SingularityDeploy> deploy, SingularityRequest request, Optional<SingularityUpdatePendingDeployRequest> updatePendingDeployRequest, Optional<SingularityLoadBalancerUpdate> lbUpdate) { if (updatePendingDeployRequest.isPresent() && pendingDeploy.getDeployProgress().isPresent()) { SingularityDeployProgress newProgress = pendingDeploy.getDeployProgress().get().withNewTargetInstances(Math.min(updatePendingDeployRequest.get().getTargetActiveInstances(), request.getInstancesSafe())); updatePendingDeploy(pendingDeploy, lbUpdate.or(pendingDeploy.getLastLoadBalancerUpdate()), DeployState.WAITING, Optional.of(newProgress)); requestManager .addToPendingQueue(new SingularityPendingRequest(request.getId(), pendingDeploy.getDeployMarker().getDeployId(), System.currentTimeMillis(), pendingDeploy.getDeployMarker().getUser(), PendingType.NEXT_DEPLOY_STEP, deploy.isPresent() ? deploy.get().getSkipHealthchecksOnDeploy() : Optional.<Boolean> absent(), pendingDeploy.getDeployMarker().getMessage())); } }
private SingularityDeployResult checkCanMoveToNextDeployStep(SingularityRequest request, Optional<SingularityDeploy> deploy, SingularityPendingDeploy pendingDeploy, Optional<SingularityUpdatePendingDeployRequest> updatePendingDeployRequest) { SingularityDeployProgress deployProgress = pendingDeploy.getDeployProgress().get(); if (canMoveToNextStep(deployProgress) || updatePendingDeployRequest.isPresent()) { SingularityDeployProgress newProgress = deployProgress.withNewTargetInstances(getNewTargetInstances(deployProgress, request, updatePendingDeployRequest)); updatePendingDeploy(pendingDeploy, pendingDeploy.getLastLoadBalancerUpdate(), DeployState.WAITING, Optional.of(newProgress)); requestManager.addToPendingQueue( new SingularityPendingRequest(request.getId(), pendingDeploy.getDeployMarker().getDeployId(), System.currentTimeMillis(), pendingDeploy.getDeployMarker().getUser(), PendingType.NEXT_DEPLOY_STEP, deploy.isPresent() ? deploy.get().getSkipHealthchecksOnDeploy() : Optional.<Boolean> absent(), pendingDeploy.getDeployMarker().getMessage())); } return new SingularityDeployResult(DeployState.WAITING); }
protected void saveAndSchedule(SingularityRequestBuilder bldr) { SingularityRequest build = bldr.build(); requestManager.activate(build, RequestHistoryType.UPDATED, System.currentTimeMillis(), Optional.<String> absent(), Optional.<String> absent()); requestManager.addToPendingQueue(new SingularityPendingRequest(build.getId(), firstDeployId, System.currentTimeMillis(), Optional.<String> absent(), PendingType.UPDATED_REQUEST, Optional.<Boolean> absent(), Optional.<String> absent())); scheduler.drainPendingQueue(); }
private void checkActiveRequest(SingularityRequestWithState requestWithState, Map<SingularityDeployKey, SingularityPendingTaskId> deployKeyToPendingTaskId, final long timestamp) { final SingularityRequest request = requestWithState.getRequest(); if (request.getRequestType() == RequestType.ON_DEMAND || request.getRequestType() == RequestType.RUN_ONCE) { return; // There's no situation where we'd want to schedule an On Demand or Run Once request at startup, so don't even bother with them. } Optional<SingularityRequestDeployState> requestDeployState = deployManager.getRequestDeployState(request.getId()); if (!requestDeployState.isPresent() || !requestDeployState.get().getActiveDeploy().isPresent()) { LOG.debug("No active deploy for {} - not scheduling on startup", request.getId()); return; } final String activeDeployId = requestDeployState.get().getActiveDeploy().get().getDeployId(); if (request.isScheduled()) { SingularityDeployKey deployKey = new SingularityDeployKey(request.getId(), activeDeployId); SingularityPendingTaskId pendingTaskId = deployKeyToPendingTaskId.get(deployKey); if (pendingTaskId != null && pendingTaskId.getCreatedAt() >= requestWithState.getTimestamp()) { LOG.info("Not rescheduling {} because {} is newer than {}", request.getId(), pendingTaskId, requestWithState.getTimestamp()); return; } } requestManager.addToPendingQueue(new SingularityPendingRequest(request.getId(), activeDeployId, timestamp, Optional.<String> absent(), PendingType.STARTUP, Optional.<Boolean> absent(), Optional.<String> absent())); }
public SingularityRequestParent exitCooldown(String requestId, Optional<SingularityExitCooldownRequest> exitCooldownRequest, SingularityUser user) { final SingularityRequestWithState requestWithState = fetchRequestWithState(requestId, user); authorizationHelper.checkForAuthorization(requestWithState.getRequest(), user, SingularityAuthorizationScope.WRITE); checkConflict(requestWithState.getState() == RequestState.SYSTEM_COOLDOWN, "Request %s is not in SYSTEM_COOLDOWN state, it is in %s", requestId, requestWithState.getState()); final Optional<String> maybeDeployId = deployManager.getInUseDeployId(requestId); final long now = System.currentTimeMillis(); Optional<String> message = Optional.absent(); Optional<Boolean> skipHealthchecks = Optional.absent(); if (exitCooldownRequest.isPresent()) { message = exitCooldownRequest.get().getMessage(); skipHealthchecks = exitCooldownRequest.get().getSkipHealthchecks(); } requestManager.exitCooldown(requestWithState.getRequest(), now, Optional.of(user.getId()), message); if (maybeDeployId.isPresent() && !requestWithState.getRequest().isOneOff()) { requestManager.addToPendingQueue(new SingularityPendingRequest(requestId, maybeDeployId.get(), now, Optional.of(user.getId()), PendingType.IMMEDIATE, skipHealthchecks, message)); } return fillEntireRequest(requestWithState); }
@Test public void testQueueMultipleOneOffs() { SingularityRequestBuilder bldr = new SingularityRequestBuilder(requestId, RequestType.ON_DEMAND); requestResource.postRequest(bldr.build(), singularityUser); deploy("on_demand_deploy"); deployChecker.checkDeploys(); requestManager.addToPendingQueue(new SingularityPendingRequest(requestId, "on_demand_deploy", System.currentTimeMillis(), Optional.<String>absent(), PendingType.ONEOFF, Optional.<List<String>>absent(), Optional.<String>absent(), Optional.<Boolean>absent(), Optional.<String>absent(), Optional.<String>absent())); requestManager.addToPendingQueue(new SingularityPendingRequest(requestId, "on_demand_deploy", System.currentTimeMillis(), Optional.<String>absent(), PendingType.ONEOFF, Optional.<List<String>>absent(), Optional.<String>absent(), Optional.<Boolean>absent(), Optional.<String>absent(), Optional.<String>absent())); scheduler.drainPendingQueue(); Assert.assertEquals(2, taskManager.getPendingTaskIds().size()); }
@Test public void testMaxOnDemandTasks() { SingularityRequestBuilder bldr = new SingularityRequestBuilder(requestId, RequestType.ON_DEMAND); bldr.setInstances(Optional.of(1)); requestResource.postRequest(bldr.build(), singularityUser); deploy("on_demand_deploy"); deployChecker.checkDeploys(); requestManager.addToPendingQueue(new SingularityPendingRequest(requestId, "on_demand_deploy", System.currentTimeMillis(), Optional.<String>absent(), PendingType.ONEOFF, Optional.<List<String>>absent(), Optional.<String>absent(), Optional.<Boolean>absent(), Optional.<String>absent(), Optional.<String>absent())); requestManager.addToPendingQueue(new SingularityPendingRequest(requestId, "on_demand_deploy", System.currentTimeMillis(), Optional.<String>absent(), PendingType.ONEOFF, Optional.<List<String>>absent(), Optional.<String>absent(), Optional.<Boolean>absent(), Optional.<String>absent(), Optional.<String>absent())); scheduler.drainPendingQueue(); resourceOffers(); Assert.assertEquals(1, taskManager.getActiveTaskIds().size()); }
@Test public void testObsoletePendingRequestsRemoved() { initRequest(); initFirstDeploy(); SingularityTask taskOne = startTask(firstDeploy); requestResource.pause(requestId, Optional.absent(), singularityUser); requestManager.addToPendingQueue(new SingularityPendingRequest(requestId, firstDeployId, System.currentTimeMillis(), Optional.<String>absent(), PendingType.NEW_DEPLOY, Optional.<Boolean>absent(), Optional.<String>absent())); Assert.assertEquals(requestManager.getPendingRequests().size(), 1); scheduler.drainPendingQueue(); Assert.assertEquals(requestManager.getPendingRequests().size(), 0); }
@Test public void itDoesntFlagPendingRequestsForUnderOrOverProvisioning() { initRequest(); initFirstDeploy(); SingularityRequest request = requestResource.getRequest(requestId, singularityUser).getRequest(); requestManager.activate(request.toBuilder().setInstances(Optional.of(0)).build(), RequestHistoryType.UPDATED, System.currentTimeMillis(), Optional.<String> absent(), Optional.<String> absent()); requestManager.addToPendingQueue(new SingularityPendingRequest(request.getId(), firstDeployId, System.currentTimeMillis(), Optional.<String> absent(), PendingType.ONEOFF, Optional.<Boolean> absent(), Optional.<String> absent())); Assert.assertEquals(0, taskManager.getActiveTaskIds().size()); SingularityState state = stateManager.getState(true, false); Assert.assertEquals(0, state.getOverProvisionedRequests()); Assert.assertEquals(0, state.getUnderProvisionedRequests()); } }
@Test public void testOfferCombination() { configuration.setOfferCacheSize(2); // Each are half of needed memory Offer offer1 = createOffer(1, 64, 1024, "slave1", "host1"); Offer offer2 = createOffer(1, 64, 1024, "slave1", "host1"); sms.resourceOffers(ImmutableList.of(offer1, offer2)); initRequest(); initFirstDeploy(); requestManager.addToPendingQueue(new SingularityPendingRequest(requestId, firstDeployId, System.currentTimeMillis(), Optional.absent(), PendingType.TASK_DONE, Optional.absent(), Optional.absent())); schedulerPoller.runActionOnPoll(); Assert.assertEquals(1, taskManager.getActiveTasks().size()); Assert.assertEquals(2, taskManager.getActiveTasks().get(0).getOffers().size()); } }
@Test public void testLeftoverCachedOffersAreReturnedToCache() throws Exception { configuration.setCacheOffers(true); Offer neededOffer = createOffer(1, 128, 1024, "slave1", "host1", Optional.absent(), Collections.emptyMap(), new String[]{"80:81"}); Offer extraOffer = createOffer(4, 256, 1024, "slave1", "host1", Optional.absent(), Collections.emptyMap(), new String[]{"83:84"}); sms.resourceOffers(ImmutableList.of(neededOffer, extraOffer)); initRequest(); firstDeploy = initAndFinishDeploy(request, new SingularityDeployBuilder(request.getId(), firstDeployId) .setCommand(Optional.of("sleep 100")), Optional.of(new Resources(1, 128, 2, 0)) ); requestManager.addToPendingQueue( new SingularityPendingRequest( requestId, firstDeployId, System.currentTimeMillis(), Optional.absent(), PendingType.TASK_DONE, Optional.absent(), Optional.absent() ) ); schedulerPoller.runActionOnPoll(); List<Offer> cachedOffers = offerCache.peekOffers(); Assert.assertEquals(1, cachedOffers.size()); }
@Test public void testRequestsInPendingQueueAreOrderedByTimestamp() { long now = System.currentTimeMillis(); initRequestWithType(RequestType.SCHEDULED, false); startFirstDeploy(); SingularityPendingRequest pendingDeployRequest = new SingularityPendingRequest(requestId, firstDeploy.getId(), now, Optional.absent(), PendingType.NEW_DEPLOY, firstDeploy.getSkipHealthchecksOnDeploy(), Optional.absent()); SingularityPendingRequest pendingRunNowRequest = new SingularityPendingRequest(requestId, firstDeploy.getId(), now + 200, Optional.absent(), PendingType.IMMEDIATE, firstDeploy.getSkipHealthchecksOnDeploy(), Optional.absent()); requestManager.addToPendingQueue(pendingDeployRequest); requestManager.addToPendingQueue(pendingRunNowRequest); Assert.assertEquals(2, requestManager.getPendingRequests().size()); // Was added first Assert.assertEquals(PendingType.NEW_DEPLOY, requestManager.getPendingRequests().get(0).getPendingType()); // Was added second Assert.assertEquals(PendingType.IMMEDIATE, requestManager.getPendingRequests().get(1).getPendingType()); resourceOffers(); }
@Test public void testLeftoverNewOffersAreCached() { configuration.setCacheOffers(true); Offer neededOffer = createOffer(1, 128, 1024, "slave1", "host1"); Offer extraOffer = createOffer(4, 256, 0, "slave1", "host1"); initRequest(); initFirstDeploy(); requestManager.addToPendingQueue( new SingularityPendingRequest( requestId, firstDeployId, System.currentTimeMillis(), Optional.absent(), PendingType.TASK_DONE, Optional.absent(), Optional.absent() ) ); sms.resourceOffers(ImmutableList.of(neededOffer, extraOffer)); List<Offer> cachedOffers = offerCache.peekOffers(); Assert.assertEquals(1, cachedOffers.size()); }
@Test public void testOnDemandDoesntGetRescheduled() { saveRequest(new SingularityRequestBuilder(requestId, RequestType.ON_DEMAND).build()); deploy(firstDeployId); deployChecker.checkDeploys(); Assert.assertTrue(requestManager.getPendingRequests().isEmpty()); Assert.assertTrue(taskManager.getPendingTaskIds().isEmpty()); startup.checkSchedulerForInconsistentState(); Assert.assertTrue(requestManager.getPendingRequests().isEmpty()); Assert.assertTrue(taskManager.getPendingTaskIds().isEmpty()); requestManager.addToPendingQueue(new SingularityPendingRequest(requestId, firstDeployId, System.currentTimeMillis(), Optional.<String> absent(), PendingType.ONEOFF, Optional.<Boolean> absent(), Optional.<String> absent())); startup.checkSchedulerForInconsistentState(); Assert.assertTrue(requestManager.getPendingRequests().get(0).getPendingType() == PendingType.ONEOFF); }
@Test public void testImmediateRequestsAreConsistentlyDeleted() { long now = System.currentTimeMillis(); initRequestWithType(RequestType.SCHEDULED, false); startFirstDeploy(); SingularityPendingRequest pendingDeployRequest = new SingularityPendingRequest(requestId, firstDeploy.getId(), now, Optional.absent(), PendingType.NEW_DEPLOY, firstDeploy.getSkipHealthchecksOnDeploy(), Optional.absent()); SingularityPendingRequest pendingRunNowRequest = new SingularityPendingRequest(requestId, firstDeploy.getId(), now + 200, Optional.absent(), PendingType.IMMEDIATE, firstDeploy.getSkipHealthchecksOnDeploy(), Optional.absent()); requestManager.addToPendingQueue(pendingDeployRequest); requestManager.addToPendingQueue(pendingRunNowRequest); // Pending queue has two requests: NEW_DEPLOY & IMMEDIATE Assert.assertEquals(2, requestManager.getPendingRequests().size()); finishNewTaskChecks(); requestManager.deletePendingRequest(pendingDeployRequest); // Just the immediate run Assert.assertEquals(1, requestManager.getPendingRequests().size()); requestManager.deletePendingRequest(pendingRunNowRequest); // Immediate run was successfully deleted Assert.assertEquals(0, requestManager.getPendingRequests().size()); }
@Test public void testImmediateRunReplacesScheduledTask() { initScheduledRequest(); SingularityDeploy deploy = SingularityDeploy.newBuilder(requestId, firstDeployId) .setCommand(Optional.of("sleep 100")) .build(); SingularityDeployRequest singularityDeployRequest = new SingularityDeployRequest(deploy, Optional.absent(), Optional.absent(), Optional.absent()); deployResource.deploy(singularityDeployRequest, singularityUser); scheduler.drainPendingQueue(); SingularityPendingTask task1 = createAndSchedulePendingTask(firstDeployId); Assert.assertEquals(1, taskManager.getPendingTaskIds().size()); Assert.assertEquals(PendingType.NEW_DEPLOY, taskManager.getPendingTaskIds().get(0).getPendingType()); requestManager.addToPendingQueue(new SingularityPendingRequest(requestId, deploy.getId(), System.currentTimeMillis(), Optional.absent(), PendingType.IMMEDIATE, deploy.getSkipHealthchecksOnDeploy(), Optional.absent())); scheduler.drainPendingQueue(); Assert.assertEquals(1, taskManager.getPendingTaskIds().size()); Assert.assertEquals(PendingType.IMMEDIATE, taskManager.getPendingTaskIds().get(0).getPendingType()); }
@Test public void testNewlyDeployedScheduledTasksAreScheduledAfterStartup() { initScheduledRequest(); initFirstDeploy(); SingularityTask runningTask = launchTask(request, firstDeploy, 1, TaskState.TASK_RUNNING); long now = System.currentTimeMillis(); initSecondDeploy(); requestManager.addToPendingQueue(new SingularityPendingRequest(requestId, secondDeployId, now, Optional.absent(), PendingType.STARTUP, Optional.absent(), Optional.absent())); deployChecker.checkDeploys(); resourceOffers(); // There's an instance running, so we shouldn't schedule a pending task yet Assert.assertTrue(taskManager.getPendingTaskIds().isEmpty()); statusUpdate(runningTask, TaskState.TASK_FINISHED); scheduler.drainPendingQueue(); // Now a pending task should be scheduled with the new deploy Assert.assertEquals(1, taskManager.getPendingTaskIds().size()); Assert.assertEquals(PendingType.NEW_DEPLOY, taskManager.getPendingTaskIds().get(0).getPendingType()); Assert.assertEquals(secondDeployId, taskManager.getPendingTaskIds().get(0).getDeployId()); }