public void sendUpdate(UpdateType updateType, Optional<String> message, Optional<String> outputFilename) { SingularityTaskShellCommandUpdate update = new SingularityTaskShellCommandUpdate(shellRequest.getId(), System.currentTimeMillis(), message, outputFilename, updateType); try { byte[] data = objectMapper.writeValueAsBytes(update); task.getLog().info("Sending update {} ({}) for shell command {}", updateType, message.or(""), shellRequest.getId()); task.getDriver().sendFrameworkMessage(data); } catch (JsonProcessingException e) { task.getLog().error("Unable to serialize update {}", update, e); } }
public Collection<SingularityTaskShellCommandUpdate> getShellCommandUpdates(SingularityTaskShellCommandRequest shellCommandRequest) { final Function<String, String> requestUri = (host) -> String.format(SHELL_COMMAND_UPDATES_FORMAT, getApiBase(host), shellCommandRequest.getTaskId(), shellCommandRequest.getShellCommand().getName(), shellCommandRequest.getTimestamp()); return getCollection(requestUri, "get shell command update history", SHELL_COMMAND_UPDATES); }
private void delete(SingularityRequestCleanup requestCleanup, Iterable<SingularityTaskId> activeTaskIds){ final long start = System.currentTimeMillis(); for (SingularityTaskId taskId : activeTaskIds) { LOG.debug("Adding task {} to cleanup (delete)", taskId.getId()); Optional<SingularityTaskShellCommandRequestId> runBeforeKillId = Optional.absent(); if (requestCleanup.getRunShellCommandBeforeKill().isPresent()) { SingularityTaskShellCommandRequest shellRequest = new SingularityTaskShellCommandRequest(taskId, requestCleanup.getUser(), System.currentTimeMillis(), requestCleanup.getRunShellCommandBeforeKill().get()); taskManager.saveTaskShellCommandRequestToQueue(shellRequest); runBeforeKillId = Optional.of(shellRequest.getId()); } taskManager.saveTaskCleanup(new SingularityTaskCleanup(requestCleanup.getUser(), TaskCleanupType.REQUEST_DELETING, start, taskId, requestCleanup.getMessage(), requestCleanup.getActionId(), runBeforeKillId, requestCleanup.getRemoveFromLoadBalancer())); } }
public void start() { List<String> command = null; try { command = buildCommand(); } catch (InvalidShellCommandException isce) { shellCommandUpdater.sendUpdate(UpdateType.INVALID, Optional.of(isce.getMessage()), Optional.<String>absent()); return; } final String outputFilename = executorConfiguration.getShellCommandOutFile() .replace("{NAME}", shellRequest.getShellCommand().getLogfileName().or(convertCommandNameToLogfileName(shellRequest.getShellCommand().getName()))) .replace("{TIMESTAMP}", Long.toString(shellRequest.getTimestamp())); shellCommandUpdater.sendUpdate(UpdateType.ACKED, Optional.of(Joiner.on(" ").join(command)), Optional.of(outputFilename)); final File outputFile = MesosUtils.getTaskDirectoryPath(getTask().getTaskId()).resolve(outputFilename).toFile(); Map<String, String> additionalEnv = new HashMap<>(); additionalEnv.put("TASK_SANDBOX_DIR", MesosUtils.getTaskDirectoryPath(getTask().getTaskId()).toString()); SingularityExecutorShellCommandRunnerCallable callable = new SingularityExecutorShellCommandRunnerCallable(task.getLog(), shellCommandUpdater, buildProcessBuilder(command, outputFile, additionalEnv), outputFile); ListenableFuture<Integer> shellFuture = shellCommandExecutorService.submit(callable); Futures.addCallback(shellFuture, new FutureCallback<Integer>() { @Override public void onSuccess(Integer result) { task.getLog().info("ShellRequest {} finished with {}", shellRequest, result); shellCommandUpdater.sendUpdate(UpdateType.FINISHED, Optional.of(String.format("Finished with code %s", result)), Optional.<String>absent()); } @Override public void onFailure(Throwable t) { task.getLog().warn("ShellRequest {} failed", shellRequest, t); shellCommandUpdater.sendUpdate(UpdateType.FAILED, Optional.of(String.format("Failed - %s (%s)", t.getClass().getSimpleName(), t.getMessage())), Optional.<String>absent()); } }); }
private void handleShellRequest(SingularityTaskShellCommandRequest shellRequest) { Optional<SingularityExecutorTask> matchingTask = monitor.getTask(shellRequest.getTaskId().getId()); if (!matchingTask.isPresent()) { LOG.warn("Missing task for {}, ignoring shell request", shellRequest.getTaskId()); return; } matchingTask.get().getLog().info("Received shell request {}", shellRequest); SingularityExecutorShellCommandUpdater updater = new SingularityExecutorShellCommandUpdater(objectMapper, shellRequest, matchingTask.get()); Optional<SingularityExecutorTaskProcessCallable> taskProcess = monitor.getTaskProcess(shellRequest.getTaskId().getId()); if (!taskProcess.isPresent()) { updater.sendUpdate(UpdateType.INVALID, Optional.of("No task process found"), Optional.<String>absent()); return; } SingularityExecutorShellCommandRunner shellRunner = new SingularityExecutorShellCommandRunner(shellRequest, executorConfiguration, matchingTask.get(), taskProcess.get(), monitor.getShellCommandExecutorServiceForTask(shellRequest.getTaskId().getId()), updater); shellRunner.start(); }
@Override public int compareTo(SingularityTaskShellCommandRequest o) { return Longs.compare(o.getTimestamp(), timestamp); }
@Override public boolean apply(SingularityExecutorShellCommandDescriptor input) { return input.getName().equals(shellRequest.getShellCommand().getName()); }
private SingularityTaskShellCommandRequest startShellCommand(SingularityTaskId taskId, final SingularityShellCommand shellCommand, SingularityUser user) { validator.checkValidShellCommand(shellCommand); SingularityTaskShellCommandRequest shellRequest = new SingularityTaskShellCommandRequest(taskId, user.getEmail(), System.currentTimeMillis(), shellCommand); taskManager.saveTaskShellCommandRequestToQueue(shellRequest); return shellRequest; }
@Override public void runActionOnPoll() { final long start = System.currentTimeMillis(); final List<SingularityTaskShellCommandRequest> shellRequests = taskManager.getAllQueuedTaskShellCommandRequests(); if (!schedulerClient.isRunning()) { LOG.warn("Unable to process shell requests because scheduler driver isn't present ({} tasks waiting)", shellRequests.size()); return; } if (shellRequests.isEmpty()) { LOG.trace("No shell requests to send."); return; } for (SingularityTaskShellCommandRequest shellRequest : shellRequests) { Optional<SingularityTask> task = taskManager.getTask(shellRequest.getTaskId()); if (!task.isPresent() || !taskManager.isActiveTask(shellRequest.getTaskId().getId())) { LOG.info("Skipping shell request {} because {} didn't exist or isn't active", shellRequest, shellRequest.getTaskId()); continue; } final ExecutorID executorId = MesosProtosUtils.toExecutorId(task.get().getMesosTask().getExecutor().getExecutorId()); final AgentID slaveId = MesosProtosUtils.toAgentId(task.get().getMesosTask().getAgentId()); final byte[] bytes = transcoder.toBytes(shellRequest); schedulerClient.frameworkMessage(executorId, slaveId, bytes); LOG.info("Sent {} ({} bytes) to {} on {}", shellRequest, bytes.length, executorId, slaveId); taskManager.saveTaskShellCommandRequestToTask(shellRequest); taskManager.deleteTaskShellCommandRequestFromQueue(shellRequest); } LOG.info("Sent {} shell requests to executors in {}", shellRequests.size(), JavaUtils.duration(start)); }
throw new InvalidShellCommandException(String.format( "\"%s\" command not found. If this command was added or modified after the task launched, then the task must be restarted to find it. Currently available commands are: %s", shellRequest.getShellCommand().getName(), executorConfiguration.getShellCommands().stream().map(SingularityExecutorShellCommandDescriptor::getName).collect(Collectors.toList()))); if (shellRequest.getShellCommand().getOptions().isPresent()) { for (SingularityExecutorShellCommandOptionDescriptor option : shellCommandDescriptor.getOptions()) { if (shellRequest.getShellCommand().getOptions().get().contains(option.getName())) { command.add(option.getFlag());
private SingularityTaskShellCommandRequest startShellCommand(SingularityTaskId taskId, final SingularityShellCommand shellCommand, SingularityUser user) { validator.checkValidShellCommand(shellCommand); SingularityTaskShellCommandRequest shellRequest = new SingularityTaskShellCommandRequest(taskId, user.getEmail(), System.currentTimeMillis(), shellCommand); taskManager.saveTaskShellCommandRequestToQueue(shellRequest); return shellRequest; }
public List<SingularityTaskShellCommandHistory> getTaskShellCommandHistory(SingularityTaskId taskId) { List<SingularityTaskShellCommandRequest> shellRequests = getTaskShellCommandRequestsForTask(taskId); List<SingularityTaskShellCommandHistory> shellCommandHistory = new ArrayList<>(shellRequests.size()); for (SingularityTaskShellCommandRequest shellRequest : shellRequests) { shellCommandHistory.add(new SingularityTaskShellCommandHistory(shellRequest, getTaskShellCommandUpdates(shellRequest.getId()))); } return shellCommandHistory; }
private TaskCleanupType pause(SingularityRequestCleanup requestCleanup, Iterable<SingularityTaskId> activeTaskIds) { final long start = System.currentTimeMillis(); boolean killTasks = requestCleanup.getKillTasks().or(configuration.isDefaultValueForKillTasksOfPausedRequests()); if (requestCleanup.getRunShellCommandBeforeKill().isPresent()) { killTasks = false; } TaskCleanupType cleanupType = killTasks ? TaskCleanupType.PAUSE : TaskCleanupType.PAUSING; for (SingularityTaskId taskId : activeTaskIds) { LOG.debug("Adding task {} to cleanup (pause)", taskId.getId()); Optional<SingularityTaskShellCommandRequestId> runBeforeKillId = Optional.absent(); if (requestCleanup.getRunShellCommandBeforeKill().isPresent()) { SingularityTaskShellCommandRequest shellRequest = new SingularityTaskShellCommandRequest(taskId, requestCleanup.getUser(), System.currentTimeMillis(), requestCleanup.getRunShellCommandBeforeKill().get()); taskManager.saveTaskShellCommandRequestToQueue(shellRequest); runBeforeKillId = Optional.of(shellRequest.getId()); } taskManager.createTaskCleanup(new SingularityTaskCleanup(requestCleanup.getUser(), cleanupType, start, taskId, requestCleanup.getMessage(), requestCleanup.getActionId(), runBeforeKillId)); } return cleanupType; }
@Override public void runActionOnPoll() { final long start = System.currentTimeMillis(); final List<SingularityTaskShellCommandRequest> shellRequests = taskManager.getAllQueuedTaskShellCommandRequests(); if (!schedulerClient.isRunning()) { LOG.warn("Unable to process shell requests because scheduler driver isn't present ({} tasks waiting)", shellRequests.size()); return; } if (shellRequests.isEmpty()) { LOG.trace("No shell requests to send."); return; } for (SingularityTaskShellCommandRequest shellRequest : shellRequests) { Optional<SingularityTask> task = taskManager.getTask(shellRequest.getTaskId()); if (!task.isPresent() || !taskManager.isActiveTask(shellRequest.getTaskId().getId())) { LOG.info("Skipping shell request {} because {} didn't exist or isn't active", shellRequest, shellRequest.getTaskId()); continue; } final ExecutorID executorId = MesosProtosUtils.toExecutorId(task.get().getMesosTask().getExecutor().getExecutorId()); final AgentID slaveId = MesosProtosUtils.toAgentId(task.get().getMesosTask().getAgentId()); final byte[] bytes = transcoder.toBytes(shellRequest); schedulerClient.frameworkMessage(executorId, slaveId, bytes); LOG.info("Sent {} ({} bytes) to {} on {}", shellRequest, bytes.length, executorId, slaveId); taskManager.saveTaskShellCommandRequestToTask(shellRequest); taskManager.deleteTaskShellCommandRequestFromQueue(shellRequest); } LOG.info("Sent {} shell requests to executors in {}", shellRequests.size(), JavaUtils.duration(start)); }
private String getShellRequestQueuePath(SingularityTaskShellCommandRequest shellRequest) { return ZKPaths.makePath(SHELL_REQUESTS_QUEUE_PATH_ROOT, shellRequest.getId().getId()); }
private void bounce(SingularityRequestCleanup requestCleanup, final List<SingularityTaskId> activeTaskIds) { final long start = System.currentTimeMillis(); final List<SingularityTaskId> matchingTaskIds = new ArrayList<>(); for (SingularityTaskId activeTaskId : activeTaskIds) { if (activeTaskId.getRequestId().equals(requestCleanup.getRequestId()) && activeTaskId.getDeployId().equals(requestCleanup.getDeployId().get())) { matchingTaskIds.add(activeTaskId); } } for (SingularityTaskId matchingTaskId : matchingTaskIds) { LOG.debug("Adding task {} to cleanup (bounce)", matchingTaskId.getId()); Optional<SingularityTaskShellCommandRequestId> runBeforeKillId = Optional.absent(); if (requestCleanup.getRunShellCommandBeforeKill().isPresent()) { SingularityTaskShellCommandRequest shellRequest = new SingularityTaskShellCommandRequest(matchingTaskId, requestCleanup.getUser(), System.currentTimeMillis(), requestCleanup.getRunShellCommandBeforeKill().get()); taskManager.saveTaskShellCommandRequestToQueue(shellRequest); runBeforeKillId = Optional.of(shellRequest.getId()); } taskManager.createTaskCleanup(new SingularityTaskCleanup(requestCleanup.getUser(), requestCleanup.getCleanupType().getTaskCleanupType().get(), start, matchingTaskId, requestCleanup.getMessage(), requestCleanup.getActionId(), runBeforeKillId)); } if (matchingTaskIds.isEmpty() && requestCleanup.getDeployId().isPresent()) { Optional<SingularityExpiringBounce> expiringBounce = requestManager.getExpiringBounce(requestCleanup.getRequestId()); if (expiringBounce.isPresent() && expiringBounce.get().getDeployId().equalsIgnoreCase(requestCleanup.getDeployId().get())) { LOG.info("No running tasks for request {}. Marking bounce {} complete and starting new tasks", expiringBounce.get().getRequestId(), expiringBounce.get()); requestManager.removeExpiringBounce(requestCleanup.getRequestId()); } requestManager.markBounceComplete(requestCleanup.getRequestId()); } requestManager.addToPendingQueue(new SingularityPendingRequest(requestCleanup.getRequestId(), requestCleanup.getDeployId().get(), requestCleanup.getTimestamp(), requestCleanup.getUser(), PendingType.BOUNCE, Optional.absent(), Optional.absent(), requestCleanup.getSkipHealthchecks(), requestCleanup.getMessage(), requestCleanup.getActionId())); LOG.info("Added {} tasks for request {} to cleanup bounce queue in {}", matchingTaskIds.size(), requestCleanup.getRequestId(), JavaUtils.duration(start)); }
public SingularityCreateResult saveTaskShellCommandRequestToTask(SingularityTaskShellCommandRequest shellRequest) { return save(getShellHistoryRequestPath(shellRequest.getId()), shellRequest, taskShellCommandRequestTranscoder); }
private void delete(SingularityRequestCleanup requestCleanup, Iterable<SingularityTaskId> activeTaskIds){ final long start = System.currentTimeMillis(); for (SingularityTaskId taskId : activeTaskIds) { LOG.debug("Adding task {} to cleanup (delete)", taskId.getId()); Optional<SingularityTaskShellCommandRequestId> runBeforeKillId = Optional.absent(); if (requestCleanup.getRunShellCommandBeforeKill().isPresent()) { SingularityTaskShellCommandRequest shellRequest = new SingularityTaskShellCommandRequest(taskId, requestCleanup.getUser(), System.currentTimeMillis(), requestCleanup.getRunShellCommandBeforeKill().get()); taskManager.saveTaskShellCommandRequestToQueue(shellRequest); runBeforeKillId = Optional.of(shellRequest.getId()); } taskManager.saveTaskCleanup(new SingularityTaskCleanup(requestCleanup.getUser(), TaskCleanupType.REQUEST_DELETING, start, taskId, requestCleanup.getMessage(), requestCleanup.getActionId(), runBeforeKillId, requestCleanup.getRemoveFromLoadBalancer())); } }
if (killTaskRequest.get().getRunShellCommandBeforeKill().isPresent()) { SingularityTaskShellCommandRequest shellCommandRequest = startShellCommand(task.getTaskId(), killTaskRequest.get().getRunShellCommandBeforeKill().get(), user); runBeforeKillId = Optional.of(shellCommandRequest.getId());
private TaskCleanupType pause(SingularityRequestCleanup requestCleanup, Iterable<SingularityTaskId> activeTaskIds) { final long start = System.currentTimeMillis(); boolean killTasks = requestCleanup.getKillTasks().or(configuration.isDefaultValueForKillTasksOfPausedRequests()); if (requestCleanup.getRunShellCommandBeforeKill().isPresent()) { killTasks = false; } TaskCleanupType cleanupType = killTasks ? TaskCleanupType.PAUSE : TaskCleanupType.PAUSING; for (SingularityTaskId taskId : activeTaskIds) { LOG.debug("Adding task {} to cleanup (pause)", taskId.getId()); Optional<SingularityTaskShellCommandRequestId> runBeforeKillId = Optional.absent(); if (requestCleanup.getRunShellCommandBeforeKill().isPresent()) { SingularityTaskShellCommandRequest shellRequest = new SingularityTaskShellCommandRequest(taskId, requestCleanup.getUser(), System.currentTimeMillis(), requestCleanup.getRunShellCommandBeforeKill().get()); taskManager.saveTaskShellCommandRequestToQueue(shellRequest); runBeforeKillId = Optional.of(shellRequest.getId()); } taskManager.createTaskCleanup(new SingularityTaskCleanup(requestCleanup.getUser(), cleanupType, start, taskId, requestCleanup.getMessage(), requestCleanup.getActionId(), runBeforeKillId)); } return cleanupType; }