@Override public String toString() { return getId(); }
private Optional<String> getCmdLineArgs(SingularityPendingTaskId pendingTaskId) throws Exception { byte[] data = curator.getData().forPath(ZKPaths.makePath(TASK_PENDING_PATH, pendingTaskId.getId())); if (data != null && data.length > 0) { return Optional.of(StringTranscoder.INSTANCE.fromBytes(data)); } return Optional.absent(); }
private String getPendingPath(SingularityPendingTaskId pendingTaskId) { return ZKPaths.makePath(PENDING_PATH_ROOT, pendingTaskId.getId()); }
private String getPendingTasksToDeletePath(SingularityPendingTaskId pendingTaskId) { return ZKPaths.makePath(PENDING_TASKS_TO_DELETE_PATH_ROOT, pendingTaskId.getId()); }
private void deleteObsoletePendingTasks(SingularityPendingDeploy pendingDeploy) { List<SingularityPendingTaskId> obsoletePendingTasks = taskManager.getPendingTaskIds() .stream() .filter(taskId -> taskId.getRequestId().equals(pendingDeploy.getDeployMarker().getRequestId())) .filter(taskId -> !taskId.getDeployId().equals(pendingDeploy.getDeployMarker().getDeployId())) .collect(Collectors.toList()); for (SingularityPendingTaskId pendingTaskId : obsoletePendingTasks) { LOG.debug("Deleting obsolete pending task {}", pendingTaskId.getId()); taskManager.deletePendingTask(pendingTaskId); } }
private void deleteScheduledTasks(final Collection<SingularityPendingTask> scheduledTasks, SingularityPendingRequest pendingRequest) { List<SingularityPendingTask> tasksForDeploy = scheduledTasks .stream() .filter(task -> pendingRequest.getRequestId().equals(task.getPendingTaskId().getRequestId())) .filter(task -> pendingRequest.getDeployId().equals(task.getPendingTaskId().getDeployId())) .collect(Collectors.toList()); for (SingularityPendingTask task : tasksForDeploy) { LOG.debug("Deleting pending task {} in order to reschedule {}", task.getPendingTaskId().getId(), pendingRequest); taskManager.deletePendingTask(task.getPendingTaskId()); } }
private double calculateScore(SingularityOfferHolder offerHolder, Map<String, SingularitySlaveUsageWithCalculatedScores> currentSlaveUsagesBySlaveId, Map<String, Integer> tasksPerOffer, SingularityTaskRequestHolder taskRequestHolder, List<SingularityTaskId> activeTaskIdsForRequest, RequestUtilization requestUtilization) { Optional<SingularitySlaveUsageWithCalculatedScores> maybeSlaveUsage = Optional.fromNullable(currentSlaveUsagesBySlaveId.get(offerHolder.getSlaveId())); double score = score(offerHolder, tasksPerOffer, taskRequestHolder, maybeSlaveUsage, activeTaskIdsForRequest, requestUtilization); if (LOG.isTraceEnabled()) { LOG.trace("Scored {} | Task {} | Offer - mem {} - cpu {} | Slave {} | maybeSlaveUsage - {}", score, taskRequestHolder.getTaskRequest().getPendingTask().getPendingTaskId().getId(), MesosUtils.getMemory(offerHolder.getCurrentResources(), Optional.absent()), MesosUtils.getNumCpus(offerHolder.getCurrentResources(), Optional.absent()), offerHolder.getHostname(), maybeSlaveUsage); } return score; }
private String getPendingPath(SingularityPendingTaskId pendingTaskId) { return ZKPaths.makePath(SingularityCmdLineArgsMigration.TASK_PENDING_PATH, pendingTaskId.getId()); }
private List<SingularityTaskRequest> checkForStaleScheduledTasks(List<SingularityPendingTask> pendingTasks, List<SingularityTaskRequest> taskRequests) { final Set<String> foundPendingTaskId = Sets.newHashSetWithExpectedSize(taskRequests.size()); final Set<String> requestIds = Sets.newHashSetWithExpectedSize(taskRequests.size()); for (SingularityTaskRequest taskRequest : taskRequests) { foundPendingTaskId.add(taskRequest.getPendingTask().getPendingTaskId().getId()); requestIds.add(taskRequest.getRequest().getId()); } for (SingularityPendingTask pendingTask : pendingTasks) { if (!foundPendingTaskId.contains(pendingTask.getPendingTaskId().getId())) { LOG.info("Removing stale pending task {}", pendingTask.getPendingTaskId()); taskManager.deletePendingTask(pendingTask.getPendingTaskId()); } } // TODO this check isn't necessary if we keep track better during deploys final Map<String, SingularityRequestDeployState> deployStates = deployManager.getRequestDeployStatesByRequestIds(requestIds); final List<SingularityTaskRequest> taskRequestsWithValidDeploys = Lists.newArrayListWithCapacity(taskRequests.size()); for (SingularityTaskRequest taskRequest : taskRequests) { SingularityRequestDeployState requestDeployState = deployStates.get(taskRequest.getRequest().getId()); if (!matchesDeploy(requestDeployState, taskRequest) && !(taskRequest.getRequest().getRequestType() == RequestType.RUN_ONCE)) { LOG.info("Removing stale pending task {} because the deployId did not match active/pending deploys {}", taskRequest.getPendingTask().getPendingTaskId(), requestDeployState); taskManager.deletePendingTask(taskRequest.getPendingTask().getPendingTaskId()); } else { taskRequestsWithValidDeploys.add(taskRequest); } } return taskRequestsWithValidDeploys; }
LOG.trace("Slaves with attributes {} are reserved for matching tasks. No attributes specified for task {}", reservedSlaveAttributes, taskRequest.getPendingTask().getPendingTaskId().getId()); return false;
private String getPendingPath(SingularityPendingTaskId pendingTaskId) { return ZKPaths.makePath(PENDING_PATH_ROOT, pendingTaskId.getId()); }
private String getPendingTasksToDeletePath(SingularityPendingTaskId pendingTaskId) { return ZKPaths.makePath(PENDING_TASKS_TO_DELETE_PATH_ROOT, pendingTaskId.getId()); }
private Optional<String> getCmdLineArgs(SingularityPendingTaskId pendingTaskId) throws Exception { byte[] data = curator.getData().forPath(ZKPaths.makePath(TASK_PENDING_PATH, pendingTaskId.getId())); if (data != null && data.length > 0) { return Optional.of(StringTranscoder.INSTANCE.fromBytes(data)); } return Optional.absent(); }
private void deleteScheduledTasks(final Collection<SingularityPendingTask> scheduledTasks, SingularityPendingRequest pendingRequest) { List<SingularityPendingTask> tasksForDeploy = scheduledTasks .stream() .filter(task -> pendingRequest.getRequestId().equals(task.getPendingTaskId().getRequestId())) .filter(task -> pendingRequest.getDeployId().equals(task.getPendingTaskId().getDeployId())) .collect(Collectors.toList()); for (SingularityPendingTask task : tasksForDeploy) { LOG.debug("Deleting pending task {} in order to reschedule {}", task.getPendingTaskId().getId(), pendingRequest); taskManager.deletePendingTask(task.getPendingTaskId()); } }
private void deleteObsoletePendingTasks(SingularityPendingDeploy pendingDeploy) { List<SingularityPendingTaskId> obsoletePendingTasks = taskManager.getPendingTaskIds() .stream() .filter(taskId -> taskId.getRequestId().equals(pendingDeploy.getDeployMarker().getRequestId())) .filter(taskId -> !taskId.getDeployId().equals(pendingDeploy.getDeployMarker().getDeployId())) .collect(Collectors.toList()); for (SingularityPendingTaskId pendingTaskId : obsoletePendingTasks) { LOG.debug("Deleting obsolete pending task {}", pendingTaskId.getId()); taskManager.deletePendingTask(pendingTaskId); } }
private double calculateScore(SingularityOfferHolder offerHolder, Map<String, SingularitySlaveUsageWithCalculatedScores> currentSlaveUsagesBySlaveId, Map<String, Integer> tasksPerOffer, SingularityTaskRequestHolder taskRequestHolder, List<SingularityTaskId> activeTaskIdsForRequest, RequestUtilization requestUtilization) { Optional<SingularitySlaveUsageWithCalculatedScores> maybeSlaveUsage = Optional.fromNullable(currentSlaveUsagesBySlaveId.get(offerHolder.getSlaveId())); double score = score(offerHolder, tasksPerOffer, taskRequestHolder, maybeSlaveUsage, activeTaskIdsForRequest, requestUtilization); if (LOG.isTraceEnabled()) { LOG.trace("Scored {} | Task {} | Offer - mem {} - cpu {} | Slave {} | maybeSlaveUsage - {}", score, taskRequestHolder.getTaskRequest().getPendingTask().getPendingTaskId().getId(), MesosUtils.getMemory(offerHolder.getCurrentResources(), Optional.absent()), MesosUtils.getNumCpus(offerHolder.getCurrentResources(), Optional.absent()), offerHolder.getHostname(), maybeSlaveUsage); } return score; }
private String getPendingPath(SingularityPendingTaskId pendingTaskId) { return ZKPaths.makePath(SingularityCmdLineArgsMigration.TASK_PENDING_PATH, pendingTaskId.getId()); }
private List<SingularityTaskRequest> checkForStaleScheduledTasks(List<SingularityPendingTask> pendingTasks, List<SingularityTaskRequest> taskRequests) { final Set<String> foundPendingTaskId = Sets.newHashSetWithExpectedSize(taskRequests.size()); final Set<String> requestIds = Sets.newHashSetWithExpectedSize(taskRequests.size()); for (SingularityTaskRequest taskRequest : taskRequests) { foundPendingTaskId.add(taskRequest.getPendingTask().getPendingTaskId().getId()); requestIds.add(taskRequest.getRequest().getId()); } for (SingularityPendingTask pendingTask : pendingTasks) { if (!foundPendingTaskId.contains(pendingTask.getPendingTaskId().getId())) { LOG.info("Removing stale pending task {}", pendingTask.getPendingTaskId()); taskManager.deletePendingTask(pendingTask.getPendingTaskId()); } } // TODO this check isn't necessary if we keep track better during deploys final Map<String, SingularityRequestDeployState> deployStates = deployManager.getRequestDeployStatesByRequestIds(requestIds); final List<SingularityTaskRequest> taskRequestsWithValidDeploys = Lists.newArrayListWithCapacity(taskRequests.size()); for (SingularityTaskRequest taskRequest : taskRequests) { SingularityRequestDeployState requestDeployState = deployStates.get(taskRequest.getRequest().getId()); if (!matchesDeploy(requestDeployState, taskRequest) && !(taskRequest.getRequest().getRequestType() == RequestType.RUN_ONCE)) { LOG.info("Removing stale pending task {} because the deployId did not match active/pending deploys {}", taskRequest.getPendingTask().getPendingTaskId(), requestDeployState); taskManager.deletePendingTask(taskRequest.getPendingTask().getPendingTaskId()); } else { taskRequestsWithValidDeploys.add(taskRequest); } } return taskRequestsWithValidDeploys; }
LOG.trace("Slaves with attributes {} are reserved for matching tasks. No attributes specified for task {}", reservedSlaveAttributes, taskRequest.getPendingTask().getPendingTaskId().getId()); return false;