@Override public void applyMigration() { final long start = System.currentTimeMillis(); try { if (curator.checkExists().forPath(PENDING_TASKS_ROOT) == null) { return; } } catch (Exception e) { throw Throwables.propagate(e); } try { for (String pendingTaskId : curator.getChildren().forPath(PENDING_TASKS_ROOT)) { SingularityPendingTaskId newPendingTaskId = createFrom(pendingTaskId, start); if (!newPendingTaskId.toString().equals(pendingTaskId)) { LOG.info("Migrating {} to {}", pendingTaskId, newPendingTaskId); Optional<String> cmdLineArgs = getCmdLineArgs(pendingTaskId); taskManager.savePendingTask( new SingularityPendingTaskBuilder() .setPendingTaskId(newPendingTaskId) .setCmdLineArgsList(cmdLineArgs.isPresent() ? Optional.of(Collections.singletonList(cmdLineArgs.get())) : Optional.<List<String>> absent()) .build()); curator.delete().forPath(ZKPaths.makePath(PENDING_TASKS_ROOT, pendingTaskId)); } } } catch (Exception e) { throw Throwables.propagate(e); } }
private void checkPendingTasks() { try { if (curator.checkExists().forPath(TASK_PENDING_PATH) == null) { return; } } catch (Exception e) { throw Throwables.propagate(e); } try { for (SingularityPendingTaskId pendingTaskId : taskManager.getPendingTaskIds()) { Optional<String> cmdLineArgs = getCmdLineArgs(pendingTaskId); SingularityCreateResult result = taskManager.savePendingTask( new SingularityPendingTaskBuilder() .setPendingTaskId(pendingTaskId) .setCmdLineArgsList(getCmdLineArgs(cmdLineArgs)) .build() ); LOG.info("Saving {} ({}) {}", pendingTaskId, cmdLineArgs, result); } } catch (Exception e) { throw Throwables.propagate(e); } }
protected SingularityPendingTask createAndSchedulePendingTask(String deployId) { Random random = new Random(); SingularityPendingTaskId pendingTaskId = new SingularityPendingTaskId(requestId, deployId, System.currentTimeMillis() + TimeUnit.DAYS.toMillis(random.nextInt(3)), random.nextInt(10), PendingType.NEW_DEPLOY, System.currentTimeMillis()); SingularityPendingTask pendingTask = new SingularityPendingTaskBuilder().setPendingTaskId(pendingTaskId).build(); taskManager.savePendingTask(pendingTask); return pendingTask; }
private SingularityPendingTask pendingTask(String requestId, String deployId, PendingType pendingType) { return new SingularityPendingTaskBuilder() .setPendingTaskId(new SingularityPendingTaskId(requestId, deployId, System.currentTimeMillis(), 1, pendingType, System.currentTimeMillis())) .build(); }
@Test public void testDeployClearsObsoleteScheduledTasks() { initRequest(); initFirstDeploy(); initSecondDeploy(); SingularityPendingTaskId taskIdOne = new SingularityPendingTaskId(requestId, firstDeployId, System.currentTimeMillis() + TimeUnit.DAYS.toMillis(3), 1, PendingType.IMMEDIATE, System.currentTimeMillis()); SingularityPendingTask taskOne = new SingularityPendingTaskBuilder().setPendingTaskId(taskIdOne).build(); SingularityPendingTaskId taskIdTwo = new SingularityPendingTaskId(requestId, firstDeployId, System.currentTimeMillis() + TimeUnit.DAYS.toMillis(1), 2, PendingType.IMMEDIATE, System.currentTimeMillis()); SingularityPendingTask taskTwo = new SingularityPendingTaskBuilder().setPendingTaskId(taskIdTwo).build(); SingularityPendingTaskId taskIdThree = new SingularityPendingTaskId(requestId, secondDeployId, System.currentTimeMillis() + TimeUnit.DAYS.toMillis(3), 1, PendingType.IMMEDIATE, System.currentTimeMillis()); SingularityPendingTask taskThree = new SingularityPendingTaskBuilder().setPendingTaskId(taskIdThree).build(); SingularityPendingTaskId taskIdFour = new SingularityPendingTaskId(requestId + "hi", firstDeployId, System.currentTimeMillis() + TimeUnit.DAYS.toMillis(3), 5, PendingType.IMMEDIATE, System.currentTimeMillis()); SingularityPendingTask taskFour = new SingularityPendingTaskBuilder().setPendingTaskId(taskIdFour).build(); taskManager.savePendingTask(taskOne); taskManager.savePendingTask(taskTwo); taskManager.savePendingTask(taskThree); taskManager.savePendingTask(taskFour); launchTask(request, secondDeploy, 1, TaskState.TASK_RUNNING); deployChecker.checkDeploys(); List<SingularityPendingTaskId> pendingTaskIds = taskManager.getPendingTaskIds(); Assert.assertTrue(!pendingTaskIds.contains(taskIdOne)); Assert.assertTrue(!pendingTaskIds.contains(taskIdTwo)); Assert.assertTrue(pendingTaskIds.contains(taskIdThree)); Assert.assertTrue(pendingTaskIds.contains(taskIdFour)); }
@Test public void testEnvironmentVariableOverrides() { Map<String, String> overrideVariables = new HashMap<>(); overrideVariables.put("MY_NEW_ENV_VAR", "test"); overrideVariables.put("STARTED_BY_USER", "notTestUser"); final SingularityRequest request = new SingularityRequestBuilder("test", RequestType.WORKER) .build(); final SingularityDeploy deploy = new SingularityDeployBuilder("test", "1") .setCommand(Optional.of("/bin/echo hi")) .build(); final SingularityPendingTask pendingTask = new SingularityPendingTaskBuilder() .setPendingTaskId(new SingularityPendingTaskId("test", "1", 0, 1, PendingType.IMMEDIATE, 0)) .setUser(user) .setEnvOverrides(overrideVariables) .build(); final SingularityTaskRequest taskRequest = new SingularityTaskRequest(request, deploy, pendingTask); final TaskInfo task = builder.buildTask(offerHolder, null, taskRequest, taskResources, executorResources).getMesosTask(); Map<String, String> environmentVariables = task .getCommand() .getEnvironment() .getVariablesList() .stream() .collect(Collectors.toMap(Variable::getName, Variable::getValue)); for (String key : overrideVariables.keySet()) { assertEquals( "Environment variable " + key + " not overridden.", environmentVariables.get(key), overrideVariables.get(key)); } }
protected SingularityPendingTask buildPendingTask(SingularityRequest request, SingularityDeploy deploy, long launchTime, int instanceNo, Optional<String> runId) { SingularityPendingTaskId pendingTaskId = new SingularityPendingTaskId(request.getId(), deploy.getId(), launchTime, instanceNo, PendingType.IMMEDIATE, launchTime); SingularityPendingTask pendingTask = new SingularityPendingTaskBuilder() .setPendingTaskId(pendingTaskId) .setRunId(runId) .build(); return pendingTask; }
@Before public void createMocks() { pendingTask = new SingularityPendingTaskBuilder() .setPendingTaskId(new SingularityPendingTaskId("test", "1", 0, 1, PendingType.IMMEDIATE, 0)) .setUser(user) .build();
@Override public void applyMigration() { final long start = System.currentTimeMillis(); try { if (curator.checkExists().forPath(PENDING_TASKS_ROOT) == null) { return; } } catch (Exception e) { throw Throwables.propagate(e); } try { for (String pendingTaskId : curator.getChildren().forPath(PENDING_TASKS_ROOT)) { SingularityPendingTaskId newPendingTaskId = createFrom(pendingTaskId, start); if (!newPendingTaskId.toString().equals(pendingTaskId)) { LOG.info("Migrating {} to {}", pendingTaskId, newPendingTaskId); Optional<String> cmdLineArgs = getCmdLineArgs(pendingTaskId); taskManager.savePendingTask( new SingularityPendingTaskBuilder() .setPendingTaskId(newPendingTaskId) .setCmdLineArgsList(cmdLineArgs.isPresent() ? Optional.of(Collections.singletonList(cmdLineArgs.get())) : Optional.<List<String>> absent()) .build()); curator.delete().forPath(ZKPaths.makePath(PENDING_TASKS_ROOT, pendingTaskId)); } } } catch (Exception e) { throw Throwables.propagate(e); } }
private void checkPendingTasks() { try { if (curator.checkExists().forPath(TASK_PENDING_PATH) == null) { return; } } catch (Exception e) { throw Throwables.propagate(e); } try { for (SingularityPendingTaskId pendingTaskId : taskManager.getPendingTaskIds()) { Optional<String> cmdLineArgs = getCmdLineArgs(pendingTaskId); SingularityCreateResult result = taskManager.savePendingTask( new SingularityPendingTaskBuilder() .setPendingTaskId(pendingTaskId) .setCmdLineArgsList(getCmdLineArgs(cmdLineArgs)) .build() ); LOG.info("Saving {} ({}) {}", pendingTaskId, cmdLineArgs, result); } } catch (Exception e) { throw Throwables.propagate(e); } }
protected SingularityPendingTask createAndSchedulePendingTask(String deployId) { Random random = new Random(); SingularityPendingTaskId pendingTaskId = new SingularityPendingTaskId(requestId, deployId, System.currentTimeMillis() + TimeUnit.DAYS.toMillis(random.nextInt(3)), random.nextInt(10), PendingType.NEW_DEPLOY, System.currentTimeMillis()); SingularityPendingTask pendingTask = new SingularityPendingTaskBuilder().setPendingTaskId(pendingTaskId).build(); taskManager.savePendingTask(pendingTask); return pendingTask; }
private SingularityPendingTask pendingTask(String requestId, String deployId, PendingType pendingType) { return new SingularityPendingTaskBuilder() .setPendingTaskId(new SingularityPendingTaskId(requestId, deployId, System.currentTimeMillis(), 1, pendingType, System.currentTimeMillis())) .build(); }
@Test public void testDeployClearsObsoleteScheduledTasks() { initRequest(); initFirstDeploy(); initSecondDeploy(); SingularityPendingTaskId taskIdOne = new SingularityPendingTaskId(requestId, firstDeployId, System.currentTimeMillis() + TimeUnit.DAYS.toMillis(3), 1, PendingType.IMMEDIATE, System.currentTimeMillis()); SingularityPendingTask taskOne = new SingularityPendingTaskBuilder().setPendingTaskId(taskIdOne).build(); SingularityPendingTaskId taskIdTwo = new SingularityPendingTaskId(requestId, firstDeployId, System.currentTimeMillis() + TimeUnit.DAYS.toMillis(1), 2, PendingType.IMMEDIATE, System.currentTimeMillis()); SingularityPendingTask taskTwo = new SingularityPendingTaskBuilder().setPendingTaskId(taskIdTwo).build(); SingularityPendingTaskId taskIdThree = new SingularityPendingTaskId(requestId, secondDeployId, System.currentTimeMillis() + TimeUnit.DAYS.toMillis(3), 1, PendingType.IMMEDIATE, System.currentTimeMillis()); SingularityPendingTask taskThree = new SingularityPendingTaskBuilder().setPendingTaskId(taskIdThree).build(); SingularityPendingTaskId taskIdFour = new SingularityPendingTaskId(requestId + "hi", firstDeployId, System.currentTimeMillis() + TimeUnit.DAYS.toMillis(3), 5, PendingType.IMMEDIATE, System.currentTimeMillis()); SingularityPendingTask taskFour = new SingularityPendingTaskBuilder().setPendingTaskId(taskIdFour).build(); taskManager.savePendingTask(taskOne); taskManager.savePendingTask(taskTwo); taskManager.savePendingTask(taskThree); taskManager.savePendingTask(taskFour); launchTask(request, secondDeploy, 1, TaskState.TASK_RUNNING); deployChecker.checkDeploys(); List<SingularityPendingTaskId> pendingTaskIds = taskManager.getPendingTaskIds(); Assert.assertTrue(!pendingTaskIds.contains(taskIdOne)); Assert.assertTrue(!pendingTaskIds.contains(taskIdTwo)); Assert.assertTrue(pendingTaskIds.contains(taskIdThree)); Assert.assertTrue(pendingTaskIds.contains(taskIdFour)); }
protected SingularityPendingTask buildPendingTask(SingularityRequest request, SingularityDeploy deploy, long launchTime, int instanceNo, Optional<String> runId) { SingularityPendingTaskId pendingTaskId = new SingularityPendingTaskId(request.getId(), deploy.getId(), launchTime, instanceNo, PendingType.IMMEDIATE, launchTime); SingularityPendingTask pendingTask = new SingularityPendingTaskBuilder() .setPendingTaskId(pendingTaskId) .setRunId(runId) .build(); return pendingTask; }
@Test public void testEnvironmentVariableOverrides() { Map<String, String> overrideVariables = new HashMap<>(); overrideVariables.put("MY_NEW_ENV_VAR", "test"); overrideVariables.put("STARTED_BY_USER", "notTestUser"); final SingularityRequest request = new SingularityRequestBuilder("test", RequestType.WORKER) .build(); final SingularityDeploy deploy = new SingularityDeployBuilder("test", "1") .setCommand(Optional.of("/bin/echo hi")) .build(); final SingularityPendingTask pendingTask = new SingularityPendingTaskBuilder() .setPendingTaskId(new SingularityPendingTaskId("test", "1", 0, 1, PendingType.IMMEDIATE, 0)) .setUser(user) .setEnvOverrides(overrideVariables) .build(); final SingularityTaskRequest taskRequest = new SingularityTaskRequest(request, deploy, pendingTask); final TaskInfo task = builder.buildTask(offerHolder, null, taskRequest, taskResources, executorResources).getMesosTask(); Map<String, String> environmentVariables = task .getCommand() .getEnvironment() .getVariablesList() .stream() .collect(Collectors.toMap(Variable::getName, Variable::getValue)); for (String key : overrideVariables.keySet()) { assertEquals( "Environment variable " + key + " not overridden.", environmentVariables.get(key), overrideVariables.get(key)); } }
@Before public void createMocks() { pendingTask = new SingularityPendingTaskBuilder() .setPendingTaskId(new SingularityPendingTaskId("test", "1", 0, 1, PendingType.IMMEDIATE, 0)) .setUser(user) .build();