private SingularityRequestParent getAndLogRequestAndDeployStatus(SingularityRequestParent singularityRequestParent) { String activeDeployId = singularityRequestParent.getActiveDeploy().isPresent() ? singularityRequestParent.getActiveDeploy().get().getId() : "No Active Deploy"; String pendingDeployId = singularityRequestParent.getPendingDeploy().isPresent() ? singularityRequestParent.getPendingDeploy().get().getId() : "No Pending deploy"; LOG.info("Deploy status: Singularity request {} -> pending deploy: '{}', active deploy: '{}'", singularityRequestParent.getRequest().getId(), pendingDeployId, activeDeployId); return singularityRequestParent; }
public static SingularityDeployKey fromDeploy(SingularityDeploy deploy) { return new SingularityDeployKey(deploy.getRequestId(), deploy.getId()); }
public SingularityRequestParent createDeployForSingularityRequest(String requestId, SingularityDeploy pendingDeploy, Optional<Boolean> deployUnpause, Optional<String> message, Optional<SingularityRequest> updatedRequest) { final Function<String, String> requestUri = (String host) -> String.format(DEPLOYS_FORMAT, getApiBase(host)); HttpResponse response = post(requestUri, String.format("new deploy %s", new SingularityDeployKey(requestId, pendingDeploy.getId())), Optional.of(new SingularityDeployRequest(pendingDeploy, deployUnpause, message, updatedRequest))); return getAndLogRequestAndDeployStatus(response.getAs(SingularityRequestParent.class)); }
private boolean matchesDeploy(SingularityRequestDeployState requestDeployState, SingularityTaskRequest taskRequest) { if (requestDeployState == null) { return false; } return matchesDeployMarker(requestDeployState.getActiveDeploy(), taskRequest.getDeploy().getId()) || matchesDeployMarker(requestDeployState.getPendingDeploy(), taskRequest.getDeploy().getId()); }
private void checkForIllegalResources(SingularityRequest request, SingularityDeploy deploy) { int instances = request.getInstancesSafe(); double cpusPerInstance = deploy.getResources().or(defaultResources).getCpus(); double memoryMbPerInstance = deploy.getResources().or(defaultResources).getMemoryMb(); double diskMbPerInstance = deploy.getResources().or(defaultResources).getDiskMb(); checkBadRequest(cpusPerInstance > 0, "Request must have more than 0 cpus"); checkBadRequest(memoryMbPerInstance > 0, "Request must have more than 0 memoryMb"); checkBadRequest(diskMbPerInstance >= 0, "Request must have non-negative diskMb"); checkBadRequest(cpusPerInstance <= maxCpusPerInstance, "Deploy %s uses too many cpus %s (maxCpusPerInstance %s in mesos configuration)", deploy.getId(), cpusPerInstance, maxCpusPerInstance); checkBadRequest(cpusPerInstance * instances <= maxCpusPerRequest, "Deploy %s uses too many cpus %s (%s*%s) (cpusPerRequest %s in mesos configuration)", deploy.getId(), cpusPerInstance * instances, cpusPerInstance, instances, maxCpusPerRequest); checkBadRequest(memoryMbPerInstance <= maxMemoryMbPerInstance, "Deploy %s uses too much memoryMb %s (maxMemoryMbPerInstance %s in mesos configuration)", deploy.getId(), memoryMbPerInstance, maxMemoryMbPerInstance); checkBadRequest(memoryMbPerInstance * instances <= maxMemoryMbPerRequest, "Deploy %s uses too much memoryMb %s (%s*%s) (maxMemoryMbPerRequest %s in mesos configuration)", deploy.getId(), memoryMbPerInstance * instances, memoryMbPerInstance, instances, maxMemoryMbPerRequest); checkBadRequest(diskMbPerInstance <= maxDiskMbPerInstance, "Deploy %s uses too much diskMb %s (maxDiskMbPerInstance %s in mesos configuration)", deploy.getId(), diskMbPerInstance, maxDiskMbPerInstance); checkBadRequest(diskMbPerInstance * instances <= maxDiskMbPerRequest, "Deploy %s uses too much diskMb %s (%s*%s) (maxDiskMbPerRequest %s in mesos configuration)", deploy.getId(), diskMbPerInstance * instances, diskMbPerInstance, instances, maxDiskMbPerRequest); }
private boolean isPreemptibleTask(SingularityTaskRequest taskRequest) { // A long running task can be replaced + killed easily if (taskRequest.getRequest().getRequestType().isLongRunning()) { return true; } // A short, non-long-running task Optional<SingularityDeployStatistics> deployStatistics = deployManager.getDeployStatistics(taskRequest.getRequest().getId(), taskRequest.getDeploy().getId()); return deployStatistics.isPresent() && deployStatistics.get().getAverageRuntimeMillis().isPresent() && deployStatistics.get().getAverageRuntimeMillis().get() < configuration.getPreemptibleTaskMaxExpectedRuntimeMs(); }
public SingularityCreateResult saveDeploy(SingularityRequest request, SingularityDeployMarker deployMarker, SingularityDeploy deploy) { final SingularityCreateResult deploySaveResult = create(getDeployDataPath(deploy.getRequestId(), deploy.getId()), deploy, deployTranscoder); if (deploySaveResult == SingularityCreateResult.EXISTED) { LOG.info("Deploy object for {} already existed (new marker: {})", deploy, deployMarker); } singularityEventListener.deployHistoryEvent(new SingularityDeployUpdate(deployMarker, Optional.of(deploy), DeployEventType.STARTING, Optional.<SingularityDeployResult>absent())); create(getDeployMarkerPath(deploy.getRequestId(), deploy.getId()), deployMarker, deployMarkerTranscoder); final Optional<SingularityRequestDeployState> currentState = getRequestDeployState(deploy.getRequestId()); Optional<SingularityDeployMarker> activeDeploy = Optional.absent(); Optional<SingularityDeployMarker> pendingDeploy = Optional.absent(); if (request.isDeployable()) { if (currentState.isPresent()) { activeDeploy = currentState.get().getActiveDeploy(); } pendingDeploy = Optional.of(deployMarker); } else { activeDeploy = Optional.of(deployMarker); } final SingularityRequestDeployState newState = new SingularityRequestDeployState(deploy.getRequestId(), activeDeploy, pendingDeploy); return saveNewRequestDeployState(newState); }
@Before public void setup() { Mockito.when(taskRequest.getRequest()).thenReturn(request); Mockito.when(request.getId()).thenReturn("requestId"); Mockito.when(taskRequest.getDeploy()).thenReturn(deploy); Mockito.when(deploy.getId()).thenReturn("deployId"); Mockito.when(taskRequest.getPendingTask()).thenReturn(task); Mockito.when(task.getPendingTaskId()).thenReturn(taskId); }
protected SingularityDeploy initAndFinishDeploy(SingularityRequest request, SingularityDeployBuilder builder, Optional<Resources> maybeResources) { SingularityDeploy deploy = builder.setResources(maybeResources).build(); SingularityDeployMarker marker = new SingularityDeployMarker(deploy.getRequestId(), deploy.getId(), System.currentTimeMillis(), Optional.<String> absent(), Optional.<String> absent()); deployManager.saveDeploy(request, marker, deploy); finishDeploy(marker, deploy); return deploy; }
@Test public void testDeployCountPurging() { SingularityRequest requestOne = buildRequest("request1"); SingularityRequest requestTwo = buildRequest("request2"); SingularityDeploy requestOneDeployOne = initAndFinishDeploy(requestOne, "r1d1"); SingularityDeploy requestOneDeployTwo = initAndFinishDeploy(requestOne, "r1d2"); SingularityDeploy requestOneDeployThree = initAndFinishDeploy(requestOne, "r1d3"); SingularityDeploy requestOneDeployFour = initAndFinishDeploy(requestOne, "r1d4"); // r1d4 is the active deploy, not eligible for purging SingularityDeploy requestTwoDeployOne = initAndFinishDeploy(requestTwo, "r2d1"); SingularityDeploy requestTwoDeployTwo = initAndFinishDeploy(requestTwo, "r2d2"); // r2d2 is the active deploy, not eligible for purging configuration.setMaxStaleDeploysPerRequestInZkWhenNoDatabase(Optional.of(2)); deployHistoryPersister.runActionOnPoll(); Assert.assertTrue(!deployManager.getDeployHistory(requestOneDeployOne.getRequestId(), requestOneDeployOne.getId(), true).isPresent()); Assert.assertTrue(deployManager.getDeployHistory(requestOneDeployTwo.getRequestId(), requestOneDeployTwo.getId(), true).isPresent()); Assert.assertTrue(deployManager.getDeployHistory(requestOneDeployThree.getRequestId(), requestOneDeployThree.getId(), true).isPresent()); Assert.assertTrue(deployManager.getDeployHistory(requestOneDeployFour.getRequestId(), requestOneDeployFour.getId(), true).isPresent()); Assert.assertTrue(deployManager.getDeployHistory(requestTwoDeployOne.getRequestId(), requestTwoDeployOne.getId(), true).isPresent()); Assert.assertTrue(deployManager.getDeployHistory(requestTwoDeployTwo.getRequestId(), requestTwoDeployTwo.getId(), true).isPresent()); }
SingularityDeployMarker deployMarker = new SingularityDeployMarker(requestId, deploy.getId(), now, deployUser, deployRequest.getMessage());
@Test public void testRequestsInPendingQueueAreOrderedByTimestamp() { long now = System.currentTimeMillis(); initRequestWithType(RequestType.SCHEDULED, false); startFirstDeploy(); SingularityPendingRequest pendingDeployRequest = new SingularityPendingRequest(requestId, firstDeploy.getId(), now, Optional.absent(), PendingType.NEW_DEPLOY, firstDeploy.getSkipHealthchecksOnDeploy(), Optional.absent()); SingularityPendingRequest pendingRunNowRequest = new SingularityPendingRequest(requestId, firstDeploy.getId(), now + 200, Optional.absent(), PendingType.IMMEDIATE, firstDeploy.getSkipHealthchecksOnDeploy(), Optional.absent()); requestManager.addToPendingQueue(pendingDeployRequest); requestManager.addToPendingQueue(pendingRunNowRequest); Assert.assertEquals(2, requestManager.getPendingRequests().size()); // Was added first Assert.assertEquals(PendingType.NEW_DEPLOY, requestManager.getPendingRequests().get(0).getPendingType()); // Was added second Assert.assertEquals(PendingType.IMMEDIATE, requestManager.getPendingRequests().get(1).getPendingType()); resourceOffers(); }
@Test public void testImmediateRequestsAreConsistentlyDeleted() { long now = System.currentTimeMillis(); initRequestWithType(RequestType.SCHEDULED, false); startFirstDeploy(); SingularityPendingRequest pendingDeployRequest = new SingularityPendingRequest(requestId, firstDeploy.getId(), now, Optional.absent(), PendingType.NEW_DEPLOY, firstDeploy.getSkipHealthchecksOnDeploy(), Optional.absent()); SingularityPendingRequest pendingRunNowRequest = new SingularityPendingRequest(requestId, firstDeploy.getId(), now + 200, Optional.absent(), PendingType.IMMEDIATE, firstDeploy.getSkipHealthchecksOnDeploy(), Optional.absent()); requestManager.addToPendingQueue(pendingDeployRequest); requestManager.addToPendingQueue(pendingRunNowRequest); // Pending queue has two requests: NEW_DEPLOY & IMMEDIATE Assert.assertEquals(2, requestManager.getPendingRequests().size()); finishNewTaskChecks(); requestManager.deletePendingRequest(pendingDeployRequest); // Just the immediate run Assert.assertEquals(1, requestManager.getPendingRequests().size()); requestManager.deletePendingRequest(pendingRunNowRequest); // Immediate run was successfully deleted Assert.assertEquals(0, requestManager.getPendingRequests().size()); }
final String sanitizedHost = offerHolder.getSanitizedHost(); final SingularityTaskId taskId = new SingularityTaskId(taskRequest.getPendingTask().getPendingTaskId().getRequestId(), taskRequest.getDeploy().getId(), System.currentTimeMillis(), taskRequest.getPendingTask().getPendingTaskId().getInstanceNo(), sanitizedHost, sanitizedRackId);
@Test public void testImmediateRunReplacesScheduledTask() { initScheduledRequest(); SingularityDeploy deploy = SingularityDeploy.newBuilder(requestId, firstDeployId) .setCommand(Optional.of("sleep 100")) .build(); SingularityDeployRequest singularityDeployRequest = new SingularityDeployRequest(deploy, Optional.absent(), Optional.absent(), Optional.absent()); deployResource.deploy(singularityDeployRequest, singularityUser); scheduler.drainPendingQueue(); SingularityPendingTask task1 = createAndSchedulePendingTask(firstDeployId); Assert.assertEquals(1, taskManager.getPendingTaskIds().size()); Assert.assertEquals(PendingType.NEW_DEPLOY, taskManager.getPendingTaskIds().get(0).getPendingType()); requestManager.addToPendingQueue(new SingularityPendingRequest(requestId, deploy.getId(), System.currentTimeMillis(), Optional.absent(), PendingType.IMMEDIATE, deploy.getSkipHealthchecksOnDeploy(), Optional.absent())); scheduler.drainPendingQueue(); Assert.assertEquals(1, taskManager.getPendingTaskIds().size()); Assert.assertEquals(PendingType.IMMEDIATE, taskManager.getPendingTaskIds().get(0).getPendingType()); }
protected SingularityPendingTask buildPendingTask(SingularityRequest request, SingularityDeploy deploy, long launchTime, int instanceNo, Optional<String> runId) { SingularityPendingTaskId pendingTaskId = new SingularityPendingTaskId(request.getId(), deploy.getId(), launchTime, instanceNo, PendingType.IMMEDIATE, launchTime); SingularityPendingTask pendingTask = new SingularityPendingTaskBuilder() .setPendingTaskId(pendingTaskId) .setRunId(runId) .build(); return pendingTask; }
protected SingularityTask prepTask(SingularityRequest request, SingularityDeploy deploy, long launchTime, int instanceNo, boolean separateHosts, Optional<String> runId) { SingularityPendingTask pendingTask = buildPendingTask(request, deploy, launchTime, instanceNo, runId); SingularityTaskRequest taskRequest = new SingularityTaskRequest(request, deploy, pendingTask); Offer offer; if (separateHosts) { offer = createOffer(125, 1024, 2048, String.format("slave%s", instanceNo), String.format("host%s", instanceNo)); } else { offer = createOffer(125, 1024, 2048); } SingularityTaskId taskId = new SingularityTaskId(request.getId(), deploy.getId(), launchTime, instanceNo, offer.getHostname(), "rack1"); TaskID taskIdProto = TaskID.newBuilder().setValue(taskId.toString()).build(); TaskInfo taskInfo = TaskInfo.newBuilder() .setAgentId(offer.getAgentId()) .setExecutor(ExecutorInfo.newBuilder().setExecutorId(ExecutorID.newBuilder().setValue("executorID"))) .setTaskId(taskIdProto) .setName("name") .build(); SingularityTask task = new SingularityTask(taskRequest, taskId, Collections.singletonList(mesosProtosUtils.offerFromProtos(offer)), mesosProtosUtils.taskFromProtos(taskInfo), Optional.of("rack1")); taskManager.savePendingTask(pendingTask); return task; }
private void relaunchTask(SingularityTask task) { SingularityPendingTask pendingTask = task.getTaskRequest().getPendingTask(); SingularityPendingRequest pendingRequest = new SingularityPendingRequestBuilder() .setRequestId(task.getTaskRequest().getRequest().getId()) .setDeployId(task.getTaskRequest().getDeploy().getId()) .setPendingType(PendingType.RETRY) .setUser(pendingTask.getUser()) .setRunId(pendingTask.getRunId()) .setCmdLineArgsList(pendingTask.getCmdLineArgsList()) .setSkipHealthchecks(pendingTask.getSkipHealthchecks()) .setMessage(pendingTask.getMessage()) .setResources(pendingTask.getResources()) .setS3UploaderAdditionalFiles(pendingTask.getS3UploaderAdditionalFiles()) .setRunAsUserOverride(pendingTask.getRunAsUserOverride()) .setEnvOverrides(pendingTask.getEnvOverrides()) .setExtraArtifacts(pendingTask.getExtraArtifacts()) .setActionId(pendingTask.getActionId()) .setRunAt(pendingTask.getPendingTaskId().getNextRunAt()) .setTimestamp(System.currentTimeMillis()) .build(); requestManager.addToPendingQueue(pendingRequest); }
assertTrue(task.getMesosTask().getContainer().getDocker().getParametersList().contains(envParameter)); assertEquals(String.format("/container/%s/%s", task.getTask().getTaskRequest().getDeploy().getRequestId(), task.getTask().getTaskRequest().getDeploy().getId()), task.getMesosTask().getContainer().getVolumes(1).getContainerPath()); assertEquals(String.format("/host/%s", task.getMesosTask().getTaskId().getValue()), task.getMesosTask().getContainer().getVolumes(1).getHostPath()); assertEquals(Mode.RO, task.getMesosTask().getContainer().getVolumes(1).getMode());