private Collection<String> getS3PrefixesForDeploy(S3Configuration s3Configuration, String requestId, String deployId, Optional<Long> startArg, Optional<Long> endArg, String group, SingularityUser user) { SingularityDeployHistory deployHistory = getDeployHistory(requestId, deployId, user); long start = deployHistory.getDeployMarker().getTimestamp(); if (startArg.isPresent()) { start = Math.max(startArg.get(), start); } long end = System.currentTimeMillis(); if (!isCurrentDeploy(requestId, deployId) && deployHistory.getDeployStatistics().isPresent() && deployHistory.getDeployStatistics().get().getLastFinishAt().isPresent()) { end = deployHistory.getDeployStatistics().get().getLastFinishAt().get() + TimeUnit.DAYS.toMillis(1); } if (endArg.isPresent()) { end = Math.min(endArg.get(), end); } Optional<String> tag = Optional.absent(); if (deployHistory.getDeploy().isPresent() && deployHistory.getDeploy().get().getExecutorData().isPresent()) { tag = deployHistory.getDeploy().get().getExecutorData().get().getLoggingTag(); } Collection<String> prefixes = SingularityS3FormatHelper.getS3KeyPrefixes(s3Configuration.getS3KeyFormat(), requestId, deployId, tag, start, end, group); for (SingularityS3UploaderFile additionalFile : s3Configuration.getS3UploaderAdditionalFiles()) { if (additionalFile.getS3UploaderKeyPattern().isPresent() && !additionalFile.getS3UploaderKeyPattern().get().equals(s3Configuration.getS3KeyFormat())) { prefixes.addAll(SingularityS3FormatHelper.getS3KeyPrefixes(additionalFile.getS3UploaderKeyPattern().get(), requestId, deployId, tag, start, end, group)); } } LOG.trace("Request {}, deploy {} got S3 prefixes {} for start {}, end {}, tag {}", requestId, deployId, prefixes, start, end, tag); return prefixes; }
if (history.isPresent() && history.get().getTask().getTaskRequest().getDeploy().getExecutorData().isPresent()) { tag = history.get().getTask().getTaskRequest().getDeploy().getExecutorData().get().getLoggingTag();
checkBadRequest(deploy.getCommand().isPresent() && !deploy.getExecutorData().isPresent() || deploy.getExecutorData().isPresent() && deploy.getCustomExecutorCmd().isPresent() && !deploy.getCommand().isPresent() || deploy.getContainerInfo().isPresent(), "If not using custom executor, specify a command or containerInfo. If using custom executor, specify executorData and customExecutorCmd and no command.");
SingularityTask getSizeOptimizedTask(SingularityMesosTaskHolder taskHolder) { if (configuration.isStoreAllMesosTaskInfoForDebugging()) { return taskHolder.getTask(); } SingularityTask task = taskHolder.getTask(); TaskInfo.Builder mesosTask = taskHolder.getMesosTask().toBuilder(); mesosTask.clearData(); List<MesosOfferObject> offers = task.getOffers() .stream() .map(MesosOfferObject::sizeOptimized) .collect(Collectors.toList()); SingularityTaskRequest taskRequest = task.getTaskRequest(); if (task.getTaskRequest().getDeploy().getExecutorData().isPresent()) { SingularityDeployBuilder deploy = task.getTaskRequest().getDeploy().toBuilder(); deploy.setExecutorData(Optional.absent()); taskRequest = new SingularityTaskRequest(task.getTaskRequest().getRequest(), deploy.build(), task.getTaskRequest().getPendingTask()); } return new SingularityTask(taskRequest, task.getTaskId(), offers, mesosProtosUtils.taskFromProtos(mesosTask.build()), task.getRackId()); }
); if (task.getDeploy().getExecutorData().isPresent()) { final ExecutorDataBuilder executorDataBldr = task.getDeploy().getExecutorData().get().toBuilder();
private Collection<String> getS3PrefixesForDeploy(S3Configuration s3Configuration, String requestId, String deployId, Optional<Long> startArg, Optional<Long> endArg, String group, SingularityUser user) { SingularityDeployHistory deployHistory = getDeployHistory(requestId, deployId, user); long start = deployHistory.getDeployMarker().getTimestamp(); if (startArg.isPresent()) { start = Math.max(startArg.get(), start); } long end = System.currentTimeMillis(); if (!isCurrentDeploy(requestId, deployId) && deployHistory.getDeployStatistics().isPresent() && deployHistory.getDeployStatistics().get().getLastFinishAt().isPresent()) { end = deployHistory.getDeployStatistics().get().getLastFinishAt().get() + TimeUnit.DAYS.toMillis(1); } if (endArg.isPresent()) { end = Math.min(endArg.get(), end); } Optional<String> tag = Optional.absent(); if (deployHistory.getDeploy().isPresent() && deployHistory.getDeploy().get().getExecutorData().isPresent()) { tag = deployHistory.getDeploy().get().getExecutorData().get().getLoggingTag(); } Collection<String> prefixes = SingularityS3FormatHelper.getS3KeyPrefixes(s3Configuration.getS3KeyFormat(), requestId, deployId, tag, start, end, group); for (SingularityS3UploaderFile additionalFile : s3Configuration.getS3UploaderAdditionalFiles()) { if (additionalFile.getS3UploaderKeyPattern().isPresent() && !additionalFile.getS3UploaderKeyPattern().get().equals(s3Configuration.getS3KeyFormat())) { prefixes.addAll(SingularityS3FormatHelper.getS3KeyPrefixes(additionalFile.getS3UploaderKeyPattern().get(), requestId, deployId, tag, start, end, group)); } } LOG.trace("Request {}, deploy {} got S3 prefixes {} for start {}, end {}, tag {}", requestId, deployId, prefixes, start, end, tag); return prefixes; }
if (history.isPresent() && history.get().getTask().getTaskRequest().getDeploy().getExecutorData().isPresent()) { tag = history.get().getTask().getTaskRequest().getDeploy().getExecutorData().get().getLoggingTag();
checkBadRequest(deploy.getCommand().isPresent() && !deploy.getExecutorData().isPresent() || deploy.getExecutorData().isPresent() && deploy.getCustomExecutorCmd().isPresent() && !deploy.getCommand().isPresent() || deploy.getContainerInfo().isPresent(), "If not using custom executor, specify a command or containerInfo. If using custom executor, specify executorData and customExecutorCmd and no command.");
SingularityTask getSizeOptimizedTask(SingularityMesosTaskHolder taskHolder) { if (configuration.isStoreAllMesosTaskInfoForDebugging()) { return taskHolder.getTask(); } SingularityTask task = taskHolder.getTask(); TaskInfo.Builder mesosTask = taskHolder.getMesosTask().toBuilder(); mesosTask.clearData(); List<MesosOfferObject> offers = task.getOffers() .stream() .map(MesosOfferObject::sizeOptimized) .collect(Collectors.toList()); SingularityTaskRequest taskRequest = task.getTaskRequest(); if (task.getTaskRequest().getDeploy().getExecutorData().isPresent()) { SingularityDeployBuilder deploy = task.getTaskRequest().getDeploy().toBuilder(); deploy.setExecutorData(Optional.absent()); taskRequest = new SingularityTaskRequest(task.getTaskRequest().getRequest(), deploy.build(), task.getTaskRequest().getPendingTask()); } return new SingularityTask(taskRequest, task.getTaskId(), offers, mesosProtosUtils.taskFromProtos(mesosTask.build()), task.getRackId()); }
); if (task.getDeploy().getExecutorData().isPresent()) { final ExecutorDataBuilder executorDataBldr = task.getDeploy().getExecutorData().get().toBuilder();