protected SingularityTaskRequest buildTaskRequest(SingularityRequest request, SingularityDeploy deploy, long launchTime) { return new SingularityTaskRequest(request, deploy, buildPendingTask(request, deploy, launchTime, 100, Optional.<String>absent())); }
public List<SingularityTaskRequest> getTaskRequests(List<SingularityPendingTask> tasks) { final Multimap<String, SingularityPendingTask> requestIdToPendingTaskId = ArrayListMultimap.create(tasks.size(), 1); for (SingularityPendingTask task : tasks) { requestIdToPendingTaskId.put(task.getPendingTaskId().getRequestId(), task); } final List<SingularityRequestWithState> matchingRequests = requestManager.getRequests(requestIdToPendingTaskId.keySet()); final Map<SingularityPendingTask, SingularityDeployKey> deployKeys = SingularityDeployKey.fromPendingTasks(requestIdToPendingTaskId.values()); final Map<SingularityDeployKey, SingularityDeploy> matchingDeploys = deployManager.getDeploysForKeys(Sets.newHashSet(deployKeys.values())); final List<SingularityTaskRequest> taskRequests = Lists.newArrayListWithCapacity(matchingRequests.size()); for (SingularityRequestWithState request : matchingRequests) { Optional<SingularityPendingDeploy> maybePendingDeploy = deployManager.getPendingDeploy(request.getRequest().getId()); for (SingularityPendingTask task : requestIdToPendingTaskId.get(request.getRequest().getId())) { SingularityDeploy foundDeploy = matchingDeploys.get(deployKeys.get(task)); if (foundDeploy == null) { LOG.warn("Couldn't find a matching deploy for pending task {}", task); continue; } if (!request.getState().isRunnable()) { LOG.warn("Request was in state {} for pending task {}", request.getState(), task); continue; } Optional<SingularityRequest> updatedRequest = maybePendingDeploy.isPresent() && maybePendingDeploy.get().getDeployMarker().getDeployId().equals(task.getPendingTaskId().getDeployId()) ? maybePendingDeploy.get().getUpdatedRequest() : Optional.<SingularityRequest>absent(); taskRequests.add(new SingularityTaskRequest(updatedRequest.or(request.getRequest()), foundDeploy, task)); } } return taskRequests; }
SingularityTask getSizeOptimizedTask(SingularityMesosTaskHolder taskHolder) { if (configuration.isStoreAllMesosTaskInfoForDebugging()) { return taskHolder.getTask(); } SingularityTask task = taskHolder.getTask(); TaskInfo.Builder mesosTask = taskHolder.getMesosTask().toBuilder(); mesosTask.clearData(); List<MesosOfferObject> offers = task.getOffers() .stream() .map(MesosOfferObject::sizeOptimized) .collect(Collectors.toList()); SingularityTaskRequest taskRequest = task.getTaskRequest(); if (task.getTaskRequest().getDeploy().getExecutorData().isPresent()) { SingularityDeployBuilder deploy = task.getTaskRequest().getDeploy().toBuilder(); deploy.setExecutorData(Optional.absent()); taskRequest = new SingularityTaskRequest(task.getTaskRequest().getRequest(), deploy.build(), task.getTaskRequest().getPendingTask()); } return new SingularityTask(taskRequest, task.getTaskId(), offers, mesosProtosUtils.taskFromProtos(mesosTask.build()), task.getRackId()); }
@Test public void testEnvironmentVariableOverrides() { Map<String, String> overrideVariables = new HashMap<>(); overrideVariables.put("MY_NEW_ENV_VAR", "test"); overrideVariables.put("STARTED_BY_USER", "notTestUser"); final SingularityRequest request = new SingularityRequestBuilder("test", RequestType.WORKER) .build(); final SingularityDeploy deploy = new SingularityDeployBuilder("test", "1") .setCommand(Optional.of("/bin/echo hi")) .build(); final SingularityPendingTask pendingTask = new SingularityPendingTaskBuilder() .setPendingTaskId(new SingularityPendingTaskId("test", "1", 0, 1, PendingType.IMMEDIATE, 0)) .setUser(user) .setEnvOverrides(overrideVariables) .build(); final SingularityTaskRequest taskRequest = new SingularityTaskRequest(request, deploy, pendingTask); final TaskInfo task = builder.buildTask(offerHolder, null, taskRequest, taskResources, executorResources).getMesosTask(); Map<String, String> environmentVariables = task .getCommand() .getEnvironment() .getVariablesList() .stream() .collect(Collectors.toMap(Variable::getName, Variable::getValue)); for (String key : overrideVariables.keySet()) { assertEquals( "Environment variable " + key + " not overridden.", environmentVariables.get(key), overrideVariables.get(key)); } }
@Test public void testDockerMinimalNetworking() { taskResources = new Resources(1, 1, 0, 0); final SingularityRequest request = new SingularityRequestBuilder("test", RequestType.WORKER).build(); final SingularityContainerInfo containerInfo = new SingularityContainerInfo( SingularityContainerType.DOCKER, Optional.absent(), Optional.of(new SingularityDockerInfo("docker-image", true, SingularityDockerNetworkType.NONE, Optional.absent(), Optional.absent(), Optional.absent(), Optional.absent()))); final SingularityDeploy deploy = new SingularityDeployBuilder("test", "1") .setContainerInfo(Optional.of(containerInfo)) .build(); final SingularityTaskRequest taskRequest = new SingularityTaskRequest(request, deploy, pendingTask); final SingularityMesosTaskHolder task = builder.buildTask(offerHolder, Collections.emptyList(), taskRequest, taskResources, executorResources); assertEquals(Type.DOCKER, task.getMesosTask().getContainer().getType()); assertEquals(Protos.ContainerInfo.DockerInfo.Network.NONE, task.getMesosTask().getContainer().getDocker().getNetwork()); }
@Test public void testJobUserPassedAsEnvironmentVariable() { final SingularityRequest request = new SingularityRequestBuilder("test", RequestType.WORKER) .build(); final SingularityDeploy deploy = new SingularityDeployBuilder("test", "1") .setCommand(Optional.of("/bin/echo hi")) .build(); final SingularityTaskRequest taskRequest = new SingularityTaskRequest(request, deploy, pendingTask); final SingularityMesosTaskHolder task = builder.buildTask(offerHolder, null, taskRequest, taskResources, executorResources); List<Variable> environmentVariables = task.getMesosTask() .getCommand() .getEnvironment() .getVariablesList(); boolean success = false; for (Variable environmentVariable : environmentVariables) { success = success || (environmentVariable.getName().equals("STARTED_BY_USER") && environmentVariable.getValue().equals(user)); } assertTrue("Expected env variable STARTED_BY_USER to be set to " + user, success); }
@Test public void testAutomaticPortMapping() { NetworkConfiguration netConf = new NetworkConfiguration(); netConf.setDefaultPortMapping(true); configuration.setNetworkConfiguration(netConf); taskResources = new Resources(1, 1, 2); final SingularityRequest request = new SingularityRequestBuilder("test", RequestType.WORKER).build(); final SingularityContainerInfo containerInfo = new SingularityContainerInfo( SingularityContainerType.DOCKER, Optional.absent(), Optional.of(new SingularityDockerInfo("docker-image", false, SingularityDockerNetworkType.BRIDGE, Optional.absent(), Optional.absent(), Optional.absent(), Optional.absent()))); final SingularityDeploy deploy = new SingularityDeployBuilder("test", "1") .setContainerInfo(Optional.of(containerInfo)) .build(); final SingularityTaskRequest taskRequest = new SingularityTaskRequest(request, deploy, pendingTask); final SingularityMesosTaskHolder task = builder.buildTask(offerHolder, Collections.singletonList(MesosUtils.getPortRangeResource(31010, 31011)), taskRequest, taskResources, executorResources); assertEquals(Type.DOCKER, task.getMesosTask().getContainer().getType()); assertEquals(Protos.ContainerInfo.DockerInfo.Network.BRIDGE, task.getMesosTask().getContainer().getDocker().getNetwork()); List<PortMapping> portMappings = task.getMesosTask().getContainer().getDocker().getPortMappingsList(); assertEquals(2, portMappings.size()); assertEquals(31010, portMappings.get(0).getHostPort()); assertEquals(31010, portMappings.get(0).getContainerPort()); assertEquals(31011, portMappings.get(1).getHostPort()); assertEquals(31011, portMappings.get(1).getContainerPort()); }
@Test public void testGetPortByIndex() throws Exception{ taskResources = new Resources(1, 1, 4, 0); final Protos.Resource portsResource = Protos.Resource.newBuilder() .setName("ports") .setType(Protos.Value.Type.RANGES) .setRanges(Protos.Value.Ranges.newBuilder() .addRange( Protos.Value.Range.newBuilder() .setBegin(31003) .setEnd(31006).build()) .build()).build(); final SingularityRequest request = new SingularityRequestBuilder("test", RequestType.WORKER).build(); final SingularityDeploy deploy = new SingularityDeployBuilder("test", "1") .setCommand(Optional.of("/bin/echo")) .setArguments(Optional.of(Collections.singletonList("wat"))) .build(); final SingularityTaskRequest taskRequest = new SingularityTaskRequest(request, deploy, pendingTask); final SingularityMesosTaskHolder task = builder.buildTask(offerHolder, Collections.singletonList(portsResource), taskRequest, taskResources, executorResources); assertEquals(31005L, task.getTask().getPortByIndex(2).get().longValue()); }
.setContainerInfo(Optional.of(containerInfo)) .build(); final SingularityTaskRequest taskRequest = new SingularityTaskRequest(request, deploy, pendingTask); final SingularityMesosTaskHolder task = builder.buildTask(offerHolder, Collections.singletonList(MesosUtils.getPortRangeResource(31010, 31011)), taskRequest, taskResources, executorResources);
@Test public void testArgumentCommand() { final SingularityRequest request = new SingularityRequestBuilder("test", RequestType.WORKER).build(); final SingularityDeploy deploy = new SingularityDeployBuilder("test", "1") .setCommand(Optional.of("/bin/echo")) .setArguments(Optional.of(Collections.singletonList("wat"))) .build(); final SingularityTaskRequest taskRequest = new SingularityTaskRequest(request, deploy, pendingTask); final SingularityMesosTaskHolder task = builder.buildTask(offerHolder, null, taskRequest, taskResources, executorResources); assertEquals("/bin/echo", task.getMesosTask().getCommand().getValue()); assertEquals(1, task.getMesosTask().getCommand().getArgumentsCount()); assertEquals("wat", task.getMesosTask().getCommand().getArguments(0)); assertFalse(task.getMesosTask().getCommand().getShell()); }
protected SingularityTask prepTask(SingularityRequest request, SingularityDeploy deploy, long launchTime, int instanceNo, boolean separateHosts, Optional<String> runId) { SingularityPendingTask pendingTask = buildPendingTask(request, deploy, launchTime, instanceNo, runId); SingularityTaskRequest taskRequest = new SingularityTaskRequest(request, deploy, pendingTask); Offer offer; if (separateHosts) { offer = createOffer(125, 1024, 2048, String.format("slave%s", instanceNo), String.format("host%s", instanceNo)); } else { offer = createOffer(125, 1024, 2048); } SingularityTaskId taskId = new SingularityTaskId(request.getId(), deploy.getId(), launchTime, instanceNo, offer.getHostname(), "rack1"); TaskID taskIdProto = TaskID.newBuilder().setValue(taskId.toString()).build(); TaskInfo taskInfo = TaskInfo.newBuilder() .setAgentId(offer.getAgentId()) .setExecutor(ExecutorInfo.newBuilder().setExecutorId(ExecutorID.newBuilder().setValue("executorID"))) .setTaskId(taskIdProto) .setName("name") .build(); SingularityTask task = new SingularityTask(taskRequest, taskId, Collections.singletonList(mesosProtosUtils.offerFromProtos(offer)), mesosProtosUtils.taskFromProtos(taskInfo), Optional.of("rack1")); taskManager.savePendingTask(pendingTask); return task; }
@Test public void testShellCommand() { final SingularityRequest request = new SingularityRequestBuilder("test", RequestType.WORKER).build(); final SingularityDeploy deploy = new SingularityDeployBuilder("test", "1") .setCommand(Optional.of("/bin/echo hi")) .build(); final SingularityTaskRequest taskRequest = new SingularityTaskRequest(request, deploy, pendingTask); final SingularityMesosTaskHolder task = builder.buildTask(offerHolder, null, taskRequest, taskResources, executorResources); assertEquals("/bin/echo hi", task.getMesosTask().getCommand().getValue()); assertEquals(0, task.getMesosTask().getCommand().getArgumentsCount()); assertTrue(task.getMesosTask().getCommand().getShell()); }
.setArguments(Optional.of(Collections.singletonList("wat"))) .build(); final SingularityTaskRequest taskRequest = new SingularityTaskRequest(request, deploy, pendingTask); final SingularityMesosTaskHolder task = builder.buildTask(offerHolder, Collections.singletonList(portsResource), taskRequest, taskResources, executorResources);
protected SingularityTaskRequest buildTaskRequest(SingularityRequest request, SingularityDeploy deploy, long launchTime) { return new SingularityTaskRequest(request, deploy, buildPendingTask(request, deploy, launchTime, 100, Optional.<String>absent())); }
SingularityTask getSizeOptimizedTask(SingularityMesosTaskHolder taskHolder) { if (configuration.isStoreAllMesosTaskInfoForDebugging()) { return taskHolder.getTask(); } SingularityTask task = taskHolder.getTask(); TaskInfo.Builder mesosTask = taskHolder.getMesosTask().toBuilder(); mesosTask.clearData(); List<MesosOfferObject> offers = task.getOffers() .stream() .map(MesosOfferObject::sizeOptimized) .collect(Collectors.toList()); SingularityTaskRequest taskRequest = task.getTaskRequest(); if (task.getTaskRequest().getDeploy().getExecutorData().isPresent()) { SingularityDeployBuilder deploy = task.getTaskRequest().getDeploy().toBuilder(); deploy.setExecutorData(Optional.absent()); taskRequest = new SingularityTaskRequest(task.getTaskRequest().getRequest(), deploy.build(), task.getTaskRequest().getPendingTask()); } return new SingularityTask(taskRequest, task.getTaskId(), offers, mesosProtosUtils.taskFromProtos(mesosTask.build()), task.getRackId()); }
@Test public void testDockerMinimalNetworking() { taskResources = new Resources(1, 1, 0, 0); final SingularityRequest request = new SingularityRequestBuilder("test", RequestType.WORKER).build(); final SingularityContainerInfo containerInfo = new SingularityContainerInfo( SingularityContainerType.DOCKER, Optional.absent(), Optional.of(new SingularityDockerInfo("docker-image", true, SingularityDockerNetworkType.NONE, Optional.absent(), Optional.absent(), Optional.absent(), Optional.absent()))); final SingularityDeploy deploy = new SingularityDeployBuilder("test", "1") .setContainerInfo(Optional.of(containerInfo)) .build(); final SingularityTaskRequest taskRequest = new SingularityTaskRequest(request, deploy, pendingTask); final SingularityMesosTaskHolder task = builder.buildTask(offerHolder, Collections.emptyList(), taskRequest, taskResources, executorResources); assertEquals(Type.DOCKER, task.getMesosTask().getContainer().getType()); assertEquals(Protos.ContainerInfo.DockerInfo.Network.NONE, task.getMesosTask().getContainer().getDocker().getNetwork()); }
@Test public void testJobUserPassedAsEnvironmentVariable() { final SingularityRequest request = new SingularityRequestBuilder("test", RequestType.WORKER) .build(); final SingularityDeploy deploy = new SingularityDeployBuilder("test", "1") .setCommand(Optional.of("/bin/echo hi")) .build(); final SingularityTaskRequest taskRequest = new SingularityTaskRequest(request, deploy, pendingTask); final SingularityMesosTaskHolder task = builder.buildTask(offerHolder, null, taskRequest, taskResources, executorResources); List<Variable> environmentVariables = task.getMesosTask() .getCommand() .getEnvironment() .getVariablesList(); boolean success = false; for (Variable environmentVariable : environmentVariables) { success = success || (environmentVariable.getName().equals("STARTED_BY_USER") && environmentVariable.getValue().equals(user)); } assertTrue("Expected env variable STARTED_BY_USER to be set to " + user, success); }
@Test public void testGetPortByIndex() throws Exception{ taskResources = new Resources(1, 1, 4, 0); final Protos.Resource portsResource = Protos.Resource.newBuilder() .setName("ports") .setType(Protos.Value.Type.RANGES) .setRanges(Protos.Value.Ranges.newBuilder() .addRange( Protos.Value.Range.newBuilder() .setBegin(31003) .setEnd(31006).build()) .build()).build(); final SingularityRequest request = new SingularityRequestBuilder("test", RequestType.WORKER).build(); final SingularityDeploy deploy = new SingularityDeployBuilder("test", "1") .setCommand(Optional.of("/bin/echo")) .setArguments(Optional.of(Collections.singletonList("wat"))) .build(); final SingularityTaskRequest taskRequest = new SingularityTaskRequest(request, deploy, pendingTask); final SingularityMesosTaskHolder task = builder.buildTask(offerHolder, Collections.singletonList(portsResource), taskRequest, taskResources, executorResources); assertEquals(31005L, task.getTask().getPortByIndex(2).get().longValue()); }
@Test public void testArgumentCommand() { final SingularityRequest request = new SingularityRequestBuilder("test", RequestType.WORKER).build(); final SingularityDeploy deploy = new SingularityDeployBuilder("test", "1") .setCommand(Optional.of("/bin/echo")) .setArguments(Optional.of(Collections.singletonList("wat"))) .build(); final SingularityTaskRequest taskRequest = new SingularityTaskRequest(request, deploy, pendingTask); final SingularityMesosTaskHolder task = builder.buildTask(offerHolder, null, taskRequest, taskResources, executorResources); assertEquals("/bin/echo", task.getMesosTask().getCommand().getValue()); assertEquals(1, task.getMesosTask().getCommand().getArgumentsCount()); assertEquals("wat", task.getMesosTask().getCommand().getArguments(0)); assertFalse(task.getMesosTask().getCommand().getShell()); }
@Test public void testShellCommand() { final SingularityRequest request = new SingularityRequestBuilder("test", RequestType.WORKER).build(); final SingularityDeploy deploy = new SingularityDeployBuilder("test", "1") .setCommand(Optional.of("/bin/echo hi")) .build(); final SingularityTaskRequest taskRequest = new SingularityTaskRequest(request, deploy, pendingTask); final SingularityMesosTaskHolder task = builder.buildTask(offerHolder, null, taskRequest, taskResources, executorResources); assertEquals("/bin/echo hi", task.getMesosTask().getCommand().getValue()); assertEquals(0, task.getMesosTask().getCommand().getArgumentsCount()); assertTrue(task.getMesosTask().getCommand().getShell()); }