String deployId = deploy.getId(); SingularityDeployBuilder builder = deploy.toBuilder(); builder.setId(createUniqueDeployId()); deploy = builder.build(); deployId = deploy.getId(); checkBadRequest(deploy.getRequestId() != null && deploy.getRequestId().equals(request.getId()), "Deploy id must match request id"); checkBadRequest(deploy.getServiceBasePath().isPresent(), "Deploy for loadBalanced request must include serviceBasePath"); checkBadRequest(deploy.getLoadBalancerGroups().isPresent() && !deploy.getLoadBalancerGroups().get().isEmpty(), "Deploy for a loadBalanced request must include at least one load balacner group"); if (deploy.getResources().isPresent()) { if (deploy.getHealthcheck().isPresent()) { HealthcheckOptions healthcheck = deploy.getHealthcheck().get(); checkBadRequest(!(healthcheck.getPortIndex().isPresent() && healthcheck.getPortNumber().isPresent()), "Can only specify one of portIndex or portNumber for healthchecks"); if (healthcheck.getPortIndex().isPresent()) { checkBadRequest(healthcheck.getPortIndex().get() >= 0, "healthcheckPortIndex cannot be negative"); checkBadRequest(deploy.getResources().get().getNumPorts() > healthcheck.getPortIndex().get(), String .format("Must request %s ports for healthcheckPortIndex %s, only requested %s", healthcheck.getPortIndex().get() + 1, healthcheck.getPortIndex().get(), deploy.getResources().get().getNumPorts())); if (deploy.getLoadBalancerPortIndex().isPresent()) { checkBadRequest(deploy.getLoadBalancerPortIndex().get() >= 0, "loadBalancerPortIndex must be greater than 0"); checkBadRequest(deploy.getResources().get().getNumPorts() > deploy.getLoadBalancerPortIndex().get(), String .format("Must request %s ports for loadBalancerPortIndex %s, only requested %s", deploy.getLoadBalancerPortIndex().get() + 1, deploy.getLoadBalancerPortIndex().get(), deploy.getResources().get().getNumPorts()));
private long getAllowedMillis(SingularityDeploy deploy) { long seconds = deploy.getDeployHealthTimeoutSeconds().or(configuration.getDeployHealthyBySeconds()); if (deploy.getHealthcheck().isPresent() && !deploy.getSkipHealthchecksOnDeploy().or(false)) { seconds += deployHealthHelper.getMaxHealthcheckTimeoutSeconds(deploy.getHealthcheck().get()); } else { seconds += deploy.getConsiderHealthyAfterRunningForSeconds().or(configuration.getConsiderTaskHealthyAfterRunningForSeconds()); } return TimeUnit.SECONDS.toMillis(seconds); }
public SingularityDeploy build() { return new SingularityDeploy(requestId, id, command, arguments, containerInfo, customExecutorCmd, customExecutorId, customExecutorSource, customExecutorResources, resources, env, taskEnv, runImmediately, uris, metadata, executorData, version, timestamp, labels, mesosLabels, taskLabels, mesosTaskLabels, deployHealthTimeoutSeconds, healthcheckUri, healthcheckIntervalSeconds, healthcheckTimeoutSeconds, healthcheckPortIndex, healthcheckMaxRetries, healthcheckMaxTotalTimeoutSeconds, healthcheck, serviceBasePath, loadBalancerGroups, loadBalancerPortIndex, considerHealthyAfterRunningForSeconds, loadBalancerOptions, loadBalancerDomains, loadBalancerAdditionalRoutes, loadBalancerTemplate, loadBalancerServiceIdOverride, loadBalancerUpstreamGroup, skipHealthchecksOnDeploy, healthcheckProtocol, deployInstanceCountPerStep, deployStepWaitTimeMs, autoAdvanceDeploySteps, maxTaskRetries, shell, user, s3UploaderAdditionalFiles); }
public static SingularityDeployKey fromDeploy(SingularityDeploy deploy) { return new SingularityDeployKey(deploy.getRequestId(), deploy.getId()); }
public SingularityTaskRequestHolder(SingularityTaskRequest taskRequest, Resources defaultResources, Resources defaultCustomExecutorResources) { this.taskRequest = taskRequest; this.executorResources = taskRequest.getDeploy().getCustomExecutorCmd().isPresent() ? taskRequest.getDeploy().getCustomExecutorResources().or(defaultCustomExecutorResources) : Resources.EMPTY_RESOURCES;; this.taskResources = taskRequest.getPendingTask().getResources().or(taskRequest.getDeploy().getResources()).or(defaultResources); this.totalResources = Resources.add(taskResources, executorResources); this.requestedPorts = new ArrayList<>(); if (taskRequest.getDeploy().getContainerInfo().isPresent() && taskRequest.getDeploy().getContainerInfo().get().getDocker().isPresent()) { requestedPorts.addAll(taskRequest.getDeploy().getContainerInfo().get().getDocker().get().getLiteralHostPorts()); } }
final String requestId = checkNotNullBadRequest(deploy.getRequestId(), "DeployRequest must have a non-null requestId"); SingularityDeployMarker deployMarker = new SingularityDeployMarker(requestId, deploy.getId(), now, deployUser, deployRequest.getMessage()); Math.min(deploy.getDeployInstanceCountPerStep().or(request.getInstancesSafe()), request.getInstancesSafe()), 0, deploy.getDeployInstanceCountPerStep().or(request.getInstancesSafe()), deploy.getDeployStepWaitTimeMs().or(configuration.getDefaultDeployStepWaitTimeMs()), false, deploy.getAutoAdvanceDeploySteps().or(true), Collections.emptySet(), System.currentTimeMillis())); deployRequest.getDeploy().getSkipHealthchecksOnDeploy(), deployRequest.getMessage()));
if (deploy.isPresent() && deploy.get().getRunImmediately().isPresent()) { String requestId = deploy.get().getRequestId(); String deployId = deploy.get().getId(); SingularityRunNowRequest runNowRequest = deploy.get().getRunImmediately().get(); List<SingularityTaskId> activeTasks = taskManager.getActiveTaskIdsForRequest(requestId); List<SingularityPendingTaskId> pendingTasks = taskManager.getPendingTaskIdsForRequest(requestId); requestManager.addToPendingQueue(new SingularityPendingRequest(request.getId(), pendingDeploy.getDeployMarker().getDeployId(), deployResult.getTimestamp(), pendingDeploy.getDeployMarker().getUser(), pendingType, deploy.isPresent() ? deploy.get().getSkipHealthchecksOnDeploy() : Optional.absent(), pendingDeploy.getDeployMarker().getMessage())); RequestHistoryType.UPDATED, System.currentTimeMillis(), deploy.isPresent() ? deploy.get().getUser() : Optional.absent(), Optional.absent());
private SingularityRequestParent getAndLogRequestAndDeployStatus(SingularityRequestParent singularityRequestParent) { String activeDeployId = singularityRequestParent.getActiveDeploy().isPresent() ? singularityRequestParent.getActiveDeploy().get().getId() : "No Active Deploy"; String pendingDeployId = singularityRequestParent.getPendingDeploy().isPresent() ? singularityRequestParent.getPendingDeploy().get().getId() : "No Pending deploy"; LOG.info("Deploy status: Singularity request {} -> pending deploy: '{}', active deploy: '{}'", singularityRequestParent.getRequest().getId(), pendingDeployId, activeDeployId); return singularityRequestParent; }
final String sanitizedHost = offerHolder.getSanitizedHost(); final SingularityTaskId taskId = new SingularityTaskId(taskRequest.getPendingTask().getPendingTaskId().getRequestId(), taskRequest.getDeploy().getId(), System.currentTimeMillis(), taskRequest.getPendingTask().getPendingTaskId().getInstanceNo(), sanitizedHost, sanitizedRackId); Optional<Resource> portsResource = Optional.absent(); final Optional<SingularityContainerInfo> containerInfo = taskRequest.getDeploy().getContainerInfo(); if (desiredTaskResources.getNumPorts() > 0 || hasLiteralPortMapping(containerInfo)) { List<Long> requestedPorts = new ArrayList<>(); if (taskRequest.getDeploy().getCustomExecutorCmd().isPresent()) { prepareCustomExecutor(bldr, taskId, taskRequest, offerHolder, ports, desiredExecutorResources); } else { if (taskRequest.getDeploy().getMesosLabels().isPresent() && !taskRequest.getDeploy().getMesosLabels().get().isEmpty()) { for (SingularityMesosTaskLabel label : taskRequest.getDeploy().getMesosLabels().get()) { org.apache.mesos.v1.Protos.Label.Builder labelBuilder = Label.newBuilder(); labelBuilder.setKey(label.getKey()); if (taskRequest.getDeploy().getMesosTaskLabels().isPresent() && taskRequest.getDeploy().getMesosTaskLabels().get().containsKey(taskInstanceNo) && !taskRequest.getDeploy().getMesosTaskLabels().get().get(taskInstanceNo).isEmpty()) { for (SingularityMesosTaskLabel label : taskRequest.getDeploy().getMesosTaskLabels().get().get(taskInstanceNo)) { org.apache.mesos.v1.Protos.Label.Builder labelBuilder = Label.newBuilder(); labelBuilder.setKey(label.getKey());
@Test public void testRequestsInPendingQueueAreOrderedByTimestamp() { long now = System.currentTimeMillis(); initRequestWithType(RequestType.SCHEDULED, false); startFirstDeploy(); SingularityPendingRequest pendingDeployRequest = new SingularityPendingRequest(requestId, firstDeploy.getId(), now, Optional.absent(), PendingType.NEW_DEPLOY, firstDeploy.getSkipHealthchecksOnDeploy(), Optional.absent()); SingularityPendingRequest pendingRunNowRequest = new SingularityPendingRequest(requestId, firstDeploy.getId(), now + 200, Optional.absent(), PendingType.IMMEDIATE, firstDeploy.getSkipHealthchecksOnDeploy(), Optional.absent()); requestManager.addToPendingQueue(pendingDeployRequest); requestManager.addToPendingQueue(pendingRunNowRequest); Assert.assertEquals(2, requestManager.getPendingRequests().size()); // Was added first Assert.assertEquals(PendingType.NEW_DEPLOY, requestManager.getPendingRequests().get(0).getPendingType()); // Was added second Assert.assertEquals(PendingType.IMMEDIATE, requestManager.getPendingRequests().get(1).getPendingType()); resourceOffers(); }
@Test public void testImmediateRunReplacesScheduledTask() { initScheduledRequest(); SingularityDeploy deploy = SingularityDeploy.newBuilder(requestId, firstDeployId) .setCommand(Optional.of("sleep 100")) .build(); SingularityDeployRequest singularityDeployRequest = new SingularityDeployRequest(deploy, Optional.absent(), Optional.absent(), Optional.absent()); deployResource.deploy(singularityDeployRequest, singularityUser); scheduler.drainPendingQueue(); SingularityPendingTask task1 = createAndSchedulePendingTask(firstDeployId); Assert.assertEquals(1, taskManager.getPendingTaskIds().size()); Assert.assertEquals(PendingType.NEW_DEPLOY, taskManager.getPendingTaskIds().get(0).getPendingType()); requestManager.addToPendingQueue(new SingularityPendingRequest(requestId, deploy.getId(), System.currentTimeMillis(), Optional.absent(), PendingType.IMMEDIATE, deploy.getSkipHealthchecksOnDeploy(), Optional.absent())); scheduler.drainPendingQueue(); Assert.assertEquals(1, taskManager.getPendingTaskIds().size()); Assert.assertEquals(PendingType.IMMEDIATE, taskManager.getPendingTaskIds().get(0).getPendingType()); }
CommandInfo.Builder commandBldr = CommandInfo.newBuilder(); Optional<String> specifiedUser = task.getPendingTask().getRunAsUserOverride().or(task.getDeploy().getUser()); if (specifiedUser.isPresent()) { commandBldr.setUser(specifiedUser.get()); if (task.getDeploy().getCommand().isPresent()) { commandBldr.setValue(task.getDeploy().getCommand().get()); if (task.getDeploy().getArguments().isPresent()) { commandBldr.addAllArguments(task.getDeploy().getArguments().get()); if (task.getDeploy().getShell().isPresent()){ commandBldr.setShell(task.getDeploy().getShell().get()); } else if ((task.getDeploy().getArguments().isPresent() && !task.getDeploy().getArguments().get().isEmpty()) || task.getDeploy().getContainerInfo().isPresent() || (task.getPendingTask().getCmdLineArgsList().isPresent() && !task.getPendingTask().getCmdLineArgsList().get().isEmpty())) { commandBldr.setShell(false); combinedArtifacts.addAll(task.getDeploy().getUris().or(Collections.emptyList())); combinedArtifacts.addAll(task.getPendingTask().getExtraArtifacts());
private void checkForIllegalResources(SingularityRequest request, SingularityDeploy deploy) { int instances = request.getInstancesSafe(); double cpusPerInstance = deploy.getResources().or(defaultResources).getCpus(); double memoryMbPerInstance = deploy.getResources().or(defaultResources).getMemoryMb(); double diskMbPerInstance = deploy.getResources().or(defaultResources).getDiskMb(); checkBadRequest(cpusPerInstance > 0, "Request must have more than 0 cpus"); checkBadRequest(memoryMbPerInstance > 0, "Request must have more than 0 memoryMb"); checkBadRequest(diskMbPerInstance >= 0, "Request must have non-negative diskMb"); checkBadRequest(cpusPerInstance <= maxCpusPerInstance, "Deploy %s uses too many cpus %s (maxCpusPerInstance %s in mesos configuration)", deploy.getId(), cpusPerInstance, maxCpusPerInstance); checkBadRequest(cpusPerInstance * instances <= maxCpusPerRequest, "Deploy %s uses too many cpus %s (%s*%s) (cpusPerRequest %s in mesos configuration)", deploy.getId(), cpusPerInstance * instances, cpusPerInstance, instances, maxCpusPerRequest); checkBadRequest(memoryMbPerInstance <= maxMemoryMbPerInstance, "Deploy %s uses too much memoryMb %s (maxMemoryMbPerInstance %s in mesos configuration)", deploy.getId(), memoryMbPerInstance, maxMemoryMbPerInstance); checkBadRequest(memoryMbPerInstance * instances <= maxMemoryMbPerRequest, "Deploy %s uses too much memoryMb %s (%s*%s) (maxMemoryMbPerRequest %s in mesos configuration)", deploy.getId(), memoryMbPerInstance * instances, memoryMbPerInstance, instances, maxMemoryMbPerRequest); checkBadRequest(diskMbPerInstance <= maxDiskMbPerInstance, "Deploy %s uses too much diskMb %s (maxDiskMbPerInstance %s in mesos configuration)", deploy.getId(), diskMbPerInstance, maxDiskMbPerInstance); checkBadRequest(diskMbPerInstance * instances <= maxDiskMbPerRequest, "Deploy %s uses too much diskMb %s (%s*%s) (maxDiskMbPerRequest %s in mesos configuration)", deploy.getId(), diskMbPerInstance * instances, diskMbPerInstance, instances, maxDiskMbPerRequest); }
private List<SingularityTaskId> getHealthcheckedHealthyTasks(final SingularityDeploy deploy, final Collection<SingularityTaskId> matchingActiveTasks, final boolean isDeployPending) { final Map<SingularityTaskId, SingularityTaskHealthcheckResult> healthcheckResults = taskManager.getLastHealthcheck(matchingActiveTasks); final List<SingularityTaskId> healthyTaskIds = Lists.newArrayListWithCapacity(matchingActiveTasks.size()); List<SingularityRequestHistory> requestHistories = requestManager.getRequestHistory(deploy.getRequestId()); for (SingularityTaskId taskId : matchingActiveTasks) { DeployHealth individualTaskHealth; if (healthchecksSkipped(taskId, requestHistories, deploy)) { LOG.trace("Detected skipped healthchecks for {}", taskId); individualTaskHealth = DeployHealth.HEALTHY; } else { individualTaskHealth = getTaskHealth(deploy, isDeployPending, Optional.fromNullable(healthcheckResults.get(taskId)), taskId); } if (individualTaskHealth == DeployHealth.HEALTHY) { healthyTaskIds.add(taskId); } } return healthyTaskIds; }
private boolean shouldCheckHealthchecks(final SingularityRequest request, final Optional<SingularityDeploy> deploy, final Collection<SingularityTaskId> activeTasks, final boolean isDeployPending) { if (disasterManager.isDisabled(SingularityAction.RUN_HEALTH_CHECKS)) { return false; } if (!deploy.isPresent()) { return false; } if (!deploy.get().getHealthcheck().isPresent()) { return false; } if (isDeployPending && deploy.get().getSkipHealthchecksOnDeploy().or(false)) { return false; } if (request.getSkipHealthchecks().or(Boolean.FALSE)) { return false; } for (SingularityTask task : taskManager.getTasks(activeTasks).values()) { if (task.getTaskRequest().getPendingTask().getSkipHealthchecks().or(Boolean.FALSE)) { return false; } } return true; }
private void maybeUpdatePendingRequest(SingularityPendingDeploy pendingDeploy, Optional<SingularityDeploy> deploy, SingularityRequest request, Optional<SingularityUpdatePendingDeployRequest> updatePendingDeployRequest, Optional<SingularityLoadBalancerUpdate> lbUpdate) { if (updatePendingDeployRequest.isPresent() && pendingDeploy.getDeployProgress().isPresent()) { SingularityDeployProgress newProgress = pendingDeploy.getDeployProgress().get().withNewTargetInstances(Math.min(updatePendingDeployRequest.get().getTargetActiveInstances(), request.getInstancesSafe())); updatePendingDeploy(pendingDeploy, lbUpdate.or(pendingDeploy.getLastLoadBalancerUpdate()), DeployState.WAITING, Optional.of(newProgress)); requestManager .addToPendingQueue(new SingularityPendingRequest(request.getId(), pendingDeploy.getDeployMarker().getDeployId(), System.currentTimeMillis(), pendingDeploy.getDeployMarker().getUser(), PendingType.NEXT_DEPLOY_STEP, deploy.isPresent() ? deploy.get().getSkipHealthchecksOnDeploy() : Optional.<Boolean> absent(), pendingDeploy.getDeployMarker().getMessage())); } }
@Test public void whenDeployHasRunNowSetAndNotDeployedItWillRunImmediately() { String requestId = "request"; String deployID = "deploy"; SingularityRequest request = new SingularityRequestBuilder(requestId, RequestType.ON_DEMAND) .build(); Optional<SingularityRunNowRequest> runNowRequest = Optional.of(runNowRequest()); SingularityDeploy deploy = SingularityDeploy.newBuilder(requestId, deployID) .setCommand(Optional.of("printenv")) .setRunImmediately(runNowRequest) .build(); SingularityDeploy result = validator.checkDeploy(request, deploy, Collections.emptyList(), Collections.emptyList()); Assert.assertTrue(result.getRunImmediately().isPresent()); Assert.assertTrue(result.getRunImmediately().get().getRunId().isPresent()); }
private void checkDocker(SingularityDeploy deploy) { if (deploy.getResources().isPresent() && deploy.getContainerInfo().get().getDocker().isPresent()) { final SingularityDockerInfo dockerInfo = deploy.getContainerInfo().get().getDocker().get(); final int numPorts = deploy.getResources().get().getNumPorts(); checkBadRequest(dockerInfo.getImage() != null, "docker image may not be null"); for (SingularityDockerPortMapping portMapping : dockerInfo.getPortMappings()) { if (portMapping.getContainerPortType() == SingularityPortMappingType.FROM_OFFER) { checkBadRequest(portMapping.getContainerPort() >= 0 && portMapping.getContainerPort() < numPorts, "Index of port resource for containerPort must be between 0 and %d (inclusive)", numPorts - 1); } if (portMapping.getHostPortType() == SingularityPortMappingType.FROM_OFFER) { checkBadRequest(portMapping.getHostPort() >= 0 && portMapping.getHostPort() < numPorts, "Index of port resource for hostPort must be between 0 and %d (inclusive)", numPorts - 1); } } } }
private SingularityDeployResult getDeployResult(final SingularityRequest request, final RequestState requestState, final Optional<SingularityDeployMarker> cancelRequest, final SingularityPendingDeploy pendingDeploy, final Optional<SingularityUpdatePendingDeployRequest> updatePendingDeployRequest, final Optional<SingularityDeploy> deploy, final Collection<SingularityTaskId> deployActiveTasks, final Collection<SingularityTaskId> otherActiveTasks, final Collection<SingularityTaskId> inactiveDeployMatchingTasks) { if (!request.isDeployable() || (configuration.isAllowDeployOfPausedRequests() && requestState == RequestState.PAUSED)) { LOG.info("Succeeding a deploy {} because the request {} was not deployable", pendingDeploy, request); return new SingularityDeployResult(DeployState.SUCCEEDED, "Request not deployable"); } if (!pendingDeploy.getDeployProgress().isPresent()) { return new SingularityDeployResult(DeployState.FAILED, "No deploy progress data present in Zookeeper. Please reattempt your deploy"); } Set<SingularityTaskId> newInactiveDeployTasks = getNewInactiveDeployTasks(pendingDeploy, inactiveDeployMatchingTasks); if (!newInactiveDeployTasks.isEmpty()) { if (canRetryTasks(deploy, inactiveDeployMatchingTasks)) { SingularityDeployProgress newProgress = pendingDeploy.getDeployProgress().get().withFailedTasks(new HashSet<>(inactiveDeployMatchingTasks)); updatePendingDeploy(pendingDeploy, pendingDeploy.getLastLoadBalancerUpdate(), DeployState.WAITING, Optional.of(newProgress)); requestManager.addToPendingQueue( new SingularityPendingRequest(request.getId(), pendingDeploy.getDeployMarker().getDeployId(), System.currentTimeMillis(), pendingDeploy.getDeployMarker().getUser(), PendingType.NEXT_DEPLOY_STEP, deploy.isPresent() ? deploy.get().getSkipHealthchecksOnDeploy() : Optional.<Boolean> absent(), pendingDeploy.getDeployMarker().getMessage())); return new SingularityDeployResult(DeployState.WAITING); } if (request.isLoadBalanced() && shouldCancelLoadBalancer(pendingDeploy)) { LOG.info("Attempting to cancel pending load balancer request, failing deploy {} regardless", pendingDeploy); sendCancelToLoadBalancer(pendingDeploy); } int maxRetries = deploy.get().getMaxTaskRetries().or(configuration.getDefaultDeployMaxTaskRetries()); return getDeployResultWithFailures(request, deploy, pendingDeploy, DeployState.FAILED, String.format("%s task(s) for this deploy failed", inactiveDeployMatchingTasks.size() - maxRetries), inactiveDeployMatchingTasks); } return checkDeployProgress(request, cancelRequest, pendingDeploy, updatePendingDeployRequest, deploy, deployActiveTasks, otherActiveTasks); }
private TaskCleanupResult cleanTask(SingularityExecutorTaskDefinition taskDefinition, Optional<SingularityTaskHistory> taskHistory) { SingularityExecutorTaskLogManager logManager = new SingularityExecutorTaskLogManager(taskDefinition, templateManager, baseConfiguration, executorConfiguration, LOG, jsonObjectFileHelper, false); SingularityExecutorTaskCleanup taskCleanup = new SingularityExecutorTaskCleanup(logManager, executorConfiguration, taskDefinition, LOG, dockerUtils); boolean cleanupTaskAppDirectory = !taskDefinition.getExecutorData().getPreserveTaskSandboxAfterFinish().or(Boolean.FALSE); if (taskDefinition.shouldLogrotateLogFile()) { checkForUncompressedLogrotatedFile(taskDefinition); } if (taskHistory.isPresent()) { final Optional<SingularityTaskHistoryUpdate> lastUpdate = JavaUtils.getLast(taskHistory.get().getTaskUpdates()); if (lastUpdate.isPresent()) { if (lastUpdate.get().getTaskState().isDone() && System.currentTimeMillis() - lastUpdate.get().getTimestamp() > TimeUnit.MINUTES.toMillis(15)) { LOG.info("Task {} is done for > 15 minutes, removing logrotate files"); taskCleanup.cleanUpLogs(); } if (lastUpdate.get().getTaskState().isFailed()) { final long delta = System.currentTimeMillis() - lastUpdate.get().getTimestamp(); if (delta < cleanupConfiguration.getCleanupAppDirectoryOfFailedTasksAfterMillis()) { LOG.info("Not cleaning up task app directory of {} because only {} has elapsed since it failed (will cleanup after {})", taskDefinition.getTaskId(), JavaUtils.durationFromMillis(delta), JavaUtils.durationFromMillis(cleanupConfiguration.getCleanupAppDirectoryOfFailedTasksAfterMillis())); cleanupTaskAppDirectory = false; } } } } boolean isDocker = (taskHistory.isPresent() && taskHistory.get().getTask().getTaskRequest().getDeploy().getContainerInfo().isPresent() && taskHistory.get().getTask().getTaskRequest().getDeploy().getContainerInfo().get().getType() == SingularityContainerType.DOCKER); return taskCleanup.cleanup(cleanupTaskAppDirectory, isDocker); }