private Optional<Integer> getCpuHardLimit(SingularityTaskRequest task) { if (configuration.getCpuHardLimit().isPresent()) { Optional<Resources> maybeResources = task.getPendingTask().getResources().or(task.getDeploy().getResources()); if (maybeResources.isPresent()) { double requestedCpus = maybeResources.get().getCpus(); int scaledLimit = (int) Math.ceil(requestedCpus * configuration.getCpuHardLimitScaleFactor()); return Optional.of(Math.max(scaledLimit, configuration.getCpuHardLimit().get())); } } return Optional.absent(); }
private Optional<String> getStatusMessage(Protos.TaskStatus status, Optional<SingularityTask> task) { if (status.hasMessage() && !Strings.isNullOrEmpty(status.getMessage())) { return Optional.of(status.getMessage()); } else if (status.hasReason() && status.getReason() == Reason.REASON_CONTAINER_LIMITATION_MEMORY) { if (task.isPresent() && task.get().getTaskRequest().getDeploy().getResources().isPresent()) { if (task.get().getTaskRequest().getDeploy().getResources().get().getDiskMb() > 0) { return Optional.of(String.format("Task exceeded one or more memory limits (%s MB mem, %s MB disk).", task.get().getTaskRequest().getDeploy().getResources().get().getMemoryMb(), task.get().getTaskRequest().getDeploy().getResources().get().getDiskMb())); } else { return Optional.of(String.format("Task exceeded memory limit (%s MB mem).", task.get().getTaskRequest().getDeploy().getResources().get().getMemoryMb())); } } return Optional.of("Task exceeded memory limit."); } else if (status.hasReason() && status.getReason() == Reason.REASON_CONTAINER_LIMITATION_DISK) { if (task.isPresent() && task.get().getTaskRequest().getDeploy().getResources().isPresent()) { return Optional.of(String.format("Task exceeded disk limit (%s MB disk).", task.get().getTaskRequest().getDeploy().getResources().get().getDiskMb())); } else { return Optional.of("Task exceeded disk limit."); } } return Optional.absent(); }
public SingularityTaskRequestHolder(SingularityTaskRequest taskRequest, Resources defaultResources, Resources defaultCustomExecutorResources) { this.taskRequest = taskRequest; this.executorResources = taskRequest.getDeploy().getCustomExecutorCmd().isPresent() ? taskRequest.getDeploy().getCustomExecutorResources().or(defaultCustomExecutorResources) : Resources.EMPTY_RESOURCES;; this.taskResources = taskRequest.getPendingTask().getResources().or(taskRequest.getDeploy().getResources()).or(defaultResources); this.totalResources = Resources.add(taskResources, executorResources); this.requestedPorts = new ArrayList<>(); if (taskRequest.getDeploy().getContainerInfo().isPresent() && taskRequest.getDeploy().getContainerInfo().get().getDocker().isPresent()) { requestedPorts.addAll(taskRequest.getDeploy().getContainerInfo().get().getDocker().get().getLiteralHostPorts()); } }
private void checkDocker(SingularityDeploy deploy) { if (deploy.getResources().isPresent() && deploy.getContainerInfo().get().getDocker().isPresent()) { final SingularityDockerInfo dockerInfo = deploy.getContainerInfo().get().getDocker().get(); final int numPorts = deploy.getResources().get().getNumPorts(); checkBadRequest(dockerInfo.getImage() != null, "docker image may not be null"); for (SingularityDockerPortMapping portMapping : dockerInfo.getPortMappings()) { if (portMapping.getContainerPortType() == SingularityPortMappingType.FROM_OFFER) { checkBadRequest(portMapping.getContainerPort() >= 0 && portMapping.getContainerPort() < numPorts, "Index of port resource for containerPort must be between 0 and %d (inclusive)", numPorts - 1); } if (portMapping.getHostPortType() == SingularityPortMappingType.FROM_OFFER) { checkBadRequest(portMapping.getHostPort() >= 0 && portMapping.getHostPort() < numPorts, "Index of port resource for hostPort must be between 0 and %d (inclusive)", numPorts - 1); } } } }
Optional<SingularityDeploy> maybeDeploy = deployManager.getDeploy(utilization.getRequestId(), utilization.getDeployId()); if (maybeDeploy.isPresent() && maybeDeploy.get().getResources().isPresent()) { String requestId = utilization.getRequestId(); long memoryBytesReserved = (long) (maybeDeploy.get().getResources().get().getMemoryMb() * SingularitySlaveUsage.BYTES_PER_MEGABYTE); double cpuReserved = maybeDeploy.get().getResources().get().getCpus(); long diskBytesReserved = (long) maybeDeploy.get().getResources().get().getDiskMb() * SingularitySlaveUsage.BYTES_PER_MEGABYTE;
private MaxProbableUsage getMaxProbableUsageForSlave(List<SingularityTaskId> activeTaskIds, Map<String, RequestUtilization> requestUtilizations, String sanitizedHostname) { double cpu = 0; double memBytes = 0; double diskBytes = 0; for (SingularityTaskId taskId : activeTaskIds) { if (taskId.getSanitizedHost().equals(sanitizedHostname)) { if (requestUtilizations.containsKey(taskId.getRequestId())) { RequestUtilization utilization = requestUtilizations.get(taskId.getRequestId()); cpu += getEstimatedCpuUsageForRequest(utilization); memBytes += utilization.getMaxMemBytesUsed(); diskBytes += utilization.getMaxDiskBytesUsed(); } else { Optional<SingularityTask> maybeTask = taskManager.getTask(taskId); if (maybeTask.isPresent()) { Resources resources = maybeTask.get().getTaskRequest().getPendingTask().getResources() .or(maybeTask.get().getTaskRequest().getDeploy().getResources()) .or(defaultResources); cpu += resources.getCpus(); memBytes += resources.getMemoryMb() * SingularitySlaveUsage.BYTES_PER_MEGABYTE; diskBytes += resources.getDiskMb() * SingularitySlaveUsage.BYTES_PER_MEGABYTE; } } } } return new MaxProbableUsage(cpu, memBytes, diskBytes); }
private void checkForIllegalResources(SingularityRequest request, SingularityDeploy deploy) { int instances = request.getInstancesSafe(); double cpusPerInstance = deploy.getResources().or(defaultResources).getCpus(); double memoryMbPerInstance = deploy.getResources().or(defaultResources).getMemoryMb(); double diskMbPerInstance = deploy.getResources().or(defaultResources).getDiskMb(); checkBadRequest(cpusPerInstance > 0, "Request must have more than 0 cpus"); checkBadRequest(memoryMbPerInstance > 0, "Request must have more than 0 memoryMb"); checkBadRequest(diskMbPerInstance >= 0, "Request must have non-negative diskMb"); checkBadRequest(cpusPerInstance <= maxCpusPerInstance, "Deploy %s uses too many cpus %s (maxCpusPerInstance %s in mesos configuration)", deploy.getId(), cpusPerInstance, maxCpusPerInstance); checkBadRequest(cpusPerInstance * instances <= maxCpusPerRequest, "Deploy %s uses too many cpus %s (%s*%s) (cpusPerRequest %s in mesos configuration)", deploy.getId(), cpusPerInstance * instances, cpusPerInstance, instances, maxCpusPerRequest); checkBadRequest(memoryMbPerInstance <= maxMemoryMbPerInstance, "Deploy %s uses too much memoryMb %s (maxMemoryMbPerInstance %s in mesos configuration)", deploy.getId(), memoryMbPerInstance, maxMemoryMbPerInstance); checkBadRequest(memoryMbPerInstance * instances <= maxMemoryMbPerRequest, "Deploy %s uses too much memoryMb %s (%s*%s) (maxMemoryMbPerRequest %s in mesos configuration)", deploy.getId(), memoryMbPerInstance * instances, memoryMbPerInstance, instances, maxMemoryMbPerRequest); checkBadRequest(diskMbPerInstance <= maxDiskMbPerInstance, "Deploy %s uses too much diskMb %s (maxDiskMbPerInstance %s in mesos configuration)", deploy.getId(), diskMbPerInstance, maxDiskMbPerInstance); checkBadRequest(diskMbPerInstance * instances <= maxDiskMbPerRequest, "Deploy %s uses too much diskMb %s (%s*%s) (maxDiskMbPerRequest %s in mesos configuration)", deploy.getId(), diskMbPerInstance * instances, diskMbPerInstance, instances, maxDiskMbPerRequest); }
if (deploy.getResources().isPresent()) { if (deploy.getHealthcheck().isPresent()) { HealthcheckOptions healthcheck = deploy.getHealthcheck().get(); if (healthcheck.getPortIndex().isPresent()) { checkBadRequest(healthcheck.getPortIndex().get() >= 0, "healthcheckPortIndex cannot be negative"); checkBadRequest(deploy.getResources().get().getNumPorts() > healthcheck.getPortIndex().get(), String .format("Must request %s ports for healthcheckPortIndex %s, only requested %s", healthcheck.getPortIndex().get() + 1, healthcheck.getPortIndex().get(), deploy.getResources().get().getNumPorts())); checkBadRequest(deploy.getResources().get().getNumPorts() > deploy.getLoadBalancerPortIndex().get(), String .format("Must request %s ports for loadBalancerPortIndex %s, only requested %s", deploy.getLoadBalancerPortIndex().get() + 1, deploy.getLoadBalancerPortIndex().get(), deploy.getResources().get().getNumPorts())); checkBadRequest(hasUri || hasHealthCheckResultFilePath, "Must specify a uri or a healthcheck result file pathh when specifying health check parameters"); if (hasUri && (!deploy.getResources().isPresent() || deploy.getResources().get().getNumPorts() == 0)) { checkBadRequest(healthcheckOptions.getPortNumber().isPresent(), "Either an explicit port number, or port resources and port index must be specified to run healthchecks against a uri");
Optional<Resources> maybeResources = Optional.absent(); if (maybeTask.isPresent()) { maybeResources = maybeTask.get().getTaskRequest().getPendingTask().getResources().or(maybeTask.get().getTaskRequest().getDeploy().getResources()); if (maybeResources.isPresent()) { Resources taskResources = maybeResources.get();
private Optional<Integer> getCpuHardLimit(SingularityTaskRequest task) { if (configuration.getCpuHardLimit().isPresent()) { Optional<Resources> maybeResources = task.getPendingTask().getResources().or(task.getDeploy().getResources()); if (maybeResources.isPresent()) { double requestedCpus = maybeResources.get().getCpus(); int scaledLimit = (int) Math.ceil(requestedCpus * configuration.getCpuHardLimitScaleFactor()); return Optional.of(Math.max(scaledLimit, configuration.getCpuHardLimit().get())); } } return Optional.absent(); }
private Optional<String> getStatusMessage(Protos.TaskStatus status, Optional<SingularityTask> task) { if (status.hasMessage() && !Strings.isNullOrEmpty(status.getMessage())) { return Optional.of(status.getMessage()); } else if (status.hasReason() && status.getReason() == Reason.REASON_CONTAINER_LIMITATION_MEMORY) { if (task.isPresent() && task.get().getTaskRequest().getDeploy().getResources().isPresent()) { if (task.get().getTaskRequest().getDeploy().getResources().get().getDiskMb() > 0) { return Optional.of(String.format("Task exceeded one or more memory limits (%s MB mem, %s MB disk).", task.get().getTaskRequest().getDeploy().getResources().get().getMemoryMb(), task.get().getTaskRequest().getDeploy().getResources().get().getDiskMb())); } else { return Optional.of(String.format("Task exceeded memory limit (%s MB mem).", task.get().getTaskRequest().getDeploy().getResources().get().getMemoryMb())); } } return Optional.of("Task exceeded memory limit."); } else if (status.hasReason() && status.getReason() == Reason.REASON_CONTAINER_LIMITATION_DISK) { if (task.isPresent() && task.get().getTaskRequest().getDeploy().getResources().isPresent()) { return Optional.of(String.format("Task exceeded disk limit (%s MB disk).", task.get().getTaskRequest().getDeploy().getResources().get().getDiskMb())); } else { return Optional.of("Task exceeded disk limit."); } } return Optional.absent(); }
public SingularityTaskRequestHolder(SingularityTaskRequest taskRequest, Resources defaultResources, Resources defaultCustomExecutorResources) { this.taskRequest = taskRequest; this.executorResources = taskRequest.getDeploy().getCustomExecutorCmd().isPresent() ? taskRequest.getDeploy().getCustomExecutorResources().or(defaultCustomExecutorResources) : Resources.EMPTY_RESOURCES;; this.taskResources = taskRequest.getPendingTask().getResources().or(taskRequest.getDeploy().getResources()).or(defaultResources); this.totalResources = Resources.add(taskResources, executorResources); this.requestedPorts = new ArrayList<>(); if (taskRequest.getDeploy().getContainerInfo().isPresent() && taskRequest.getDeploy().getContainerInfo().get().getDocker().isPresent()) { requestedPorts.addAll(taskRequest.getDeploy().getContainerInfo().get().getDocker().get().getLiteralHostPorts()); } }
private void checkDocker(SingularityDeploy deploy) { if (deploy.getResources().isPresent() && deploy.getContainerInfo().get().getDocker().isPresent()) { final SingularityDockerInfo dockerInfo = deploy.getContainerInfo().get().getDocker().get(); final int numPorts = deploy.getResources().get().getNumPorts(); checkBadRequest(dockerInfo.getImage() != null, "docker image may not be null"); for (SingularityDockerPortMapping portMapping : dockerInfo.getPortMappings()) { if (portMapping.getContainerPortType() == SingularityPortMappingType.FROM_OFFER) { checkBadRequest(portMapping.getContainerPort() >= 0 && portMapping.getContainerPort() < numPorts, "Index of port resource for containerPort must be between 0 and %d (inclusive)", numPorts - 1); } if (portMapping.getHostPortType() == SingularityPortMappingType.FROM_OFFER) { checkBadRequest(portMapping.getHostPort() >= 0 && portMapping.getHostPort() < numPorts, "Index of port resource for hostPort must be between 0 and %d (inclusive)", numPorts - 1); } } } }
Optional<SingularityDeploy> maybeDeploy = deployManager.getDeploy(utilization.getRequestId(), utilization.getDeployId()); if (maybeDeploy.isPresent() && maybeDeploy.get().getResources().isPresent()) { String requestId = utilization.getRequestId(); long memoryBytesReserved = (long) (maybeDeploy.get().getResources().get().getMemoryMb() * SingularitySlaveUsage.BYTES_PER_MEGABYTE); double cpuReserved = maybeDeploy.get().getResources().get().getCpus(); long diskBytesReserved = (long) maybeDeploy.get().getResources().get().getDiskMb() * SingularitySlaveUsage.BYTES_PER_MEGABYTE;
private MaxProbableUsage getMaxProbableUsageForSlave(List<SingularityTaskId> activeTaskIds, Map<String, RequestUtilization> requestUtilizations, String sanitizedHostname) { double cpu = 0; double memBytes = 0; double diskBytes = 0; for (SingularityTaskId taskId : activeTaskIds) { if (taskId.getSanitizedHost().equals(sanitizedHostname)) { if (requestUtilizations.containsKey(taskId.getRequestId())) { RequestUtilization utilization = requestUtilizations.get(taskId.getRequestId()); cpu += getEstimatedCpuUsageForRequest(utilization); memBytes += utilization.getMaxMemBytesUsed(); diskBytes += utilization.getMaxDiskBytesUsed(); } else { Optional<SingularityTask> maybeTask = taskManager.getTask(taskId); if (maybeTask.isPresent()) { Resources resources = maybeTask.get().getTaskRequest().getPendingTask().getResources() .or(maybeTask.get().getTaskRequest().getDeploy().getResources()) .or(defaultResources); cpu += resources.getCpus(); memBytes += resources.getMemoryMb() * SingularitySlaveUsage.BYTES_PER_MEGABYTE; diskBytes += resources.getDiskMb() * SingularitySlaveUsage.BYTES_PER_MEGABYTE; } } } } return new MaxProbableUsage(cpu, memBytes, diskBytes); }
private void checkForIllegalResources(SingularityRequest request, SingularityDeploy deploy) { int instances = request.getInstancesSafe(); double cpusPerInstance = deploy.getResources().or(defaultResources).getCpus(); double memoryMbPerInstance = deploy.getResources().or(defaultResources).getMemoryMb(); double diskMbPerInstance = deploy.getResources().or(defaultResources).getDiskMb(); checkBadRequest(cpusPerInstance > 0, "Request must have more than 0 cpus"); checkBadRequest(memoryMbPerInstance > 0, "Request must have more than 0 memoryMb"); checkBadRequest(diskMbPerInstance >= 0, "Request must have non-negative diskMb"); checkBadRequest(cpusPerInstance <= maxCpusPerInstance, "Deploy %s uses too many cpus %s (maxCpusPerInstance %s in mesos configuration)", deploy.getId(), cpusPerInstance, maxCpusPerInstance); checkBadRequest(cpusPerInstance * instances <= maxCpusPerRequest, "Deploy %s uses too many cpus %s (%s*%s) (cpusPerRequest %s in mesos configuration)", deploy.getId(), cpusPerInstance * instances, cpusPerInstance, instances, maxCpusPerRequest); checkBadRequest(memoryMbPerInstance <= maxMemoryMbPerInstance, "Deploy %s uses too much memoryMb %s (maxMemoryMbPerInstance %s in mesos configuration)", deploy.getId(), memoryMbPerInstance, maxMemoryMbPerInstance); checkBadRequest(memoryMbPerInstance * instances <= maxMemoryMbPerRequest, "Deploy %s uses too much memoryMb %s (%s*%s) (maxMemoryMbPerRequest %s in mesos configuration)", deploy.getId(), memoryMbPerInstance * instances, memoryMbPerInstance, instances, maxMemoryMbPerRequest); checkBadRequest(diskMbPerInstance <= maxDiskMbPerInstance, "Deploy %s uses too much diskMb %s (maxDiskMbPerInstance %s in mesos configuration)", deploy.getId(), diskMbPerInstance, maxDiskMbPerInstance); checkBadRequest(diskMbPerInstance * instances <= maxDiskMbPerRequest, "Deploy %s uses too much diskMb %s (%s*%s) (maxDiskMbPerRequest %s in mesos configuration)", deploy.getId(), diskMbPerInstance * instances, diskMbPerInstance, instances, maxDiskMbPerRequest); }
if (deploy.getResources().isPresent()) { if (deploy.getHealthcheck().isPresent()) { HealthcheckOptions healthcheck = deploy.getHealthcheck().get(); if (healthcheck.getPortIndex().isPresent()) { checkBadRequest(healthcheck.getPortIndex().get() >= 0, "healthcheckPortIndex cannot be negative"); checkBadRequest(deploy.getResources().get().getNumPorts() > healthcheck.getPortIndex().get(), String .format("Must request %s ports for healthcheckPortIndex %s, only requested %s", healthcheck.getPortIndex().get() + 1, healthcheck.getPortIndex().get(), deploy.getResources().get().getNumPorts())); checkBadRequest(deploy.getResources().get().getNumPorts() > deploy.getLoadBalancerPortIndex().get(), String .format("Must request %s ports for loadBalancerPortIndex %s, only requested %s", deploy.getLoadBalancerPortIndex().get() + 1, deploy.getLoadBalancerPortIndex().get(), deploy.getResources().get().getNumPorts())); checkBadRequest(hasUri || hasHealthCheckResultFilePath, "Must specify a uri or a healthcheck result file pathh when specifying health check parameters"); if (hasUri && (!deploy.getResources().isPresent() || deploy.getResources().get().getNumPorts() == 0)) { checkBadRequest(healthcheckOptions.getPortNumber().isPresent(), "Either an explicit port number, or port resources and port index must be specified to run healthchecks against a uri");
Optional<Resources> maybeResources = Optional.absent(); if (maybeTask.isPresent()) { maybeResources = maybeTask.get().getTaskRequest().getPendingTask().getResources().or(maybeTask.get().getTaskRequest().getDeploy().getResources()); if (maybeResources.isPresent()) { Resources taskResources = maybeResources.get();