public static SingularityPendingTaskId valueOf(String string) { String[] splits = null; try { splits = JavaUtils.reverseSplit(string, 6, "-"); } catch (IllegalStateException ise) { throw new InvalidSingularityTaskIdException(String.format("PendingTaskId %s was invalid (%s)", string, ise.getMessage())); } try { final String requestId = splits[0]; final String deployId = splits[1]; final long nextRunAt = Long.parseLong(splits[2]); final int instanceNo = Integer.parseInt(splits[3]); final PendingType pendingType = PendingType.valueOf(splits[4]); final long createdAt = Long.parseLong(splits[5]); return new SingularityPendingTaskId(requestId, deployId, nextRunAt, instanceNo, pendingType, createdAt); } catch (IllegalArgumentException e) { throw new InvalidSingularityTaskIdException(String.format("PendingTaskId %s had an invalid parameter (%s)", string, e.getMessage())); } }
@Override public int compareTo(SingularityPendingTaskId o) { return ComparisonChain.start() .compare(this.getNextRunAt(), o.getNextRunAt()) .compare(this.getRequestId(), o.getRequestId()) .compare(this.getDeployId(), o.getDeployId()) .compare(this.getInstanceNo(), o.getInstanceNo()) .compare(this.getCreatedAt(), o.getCreatedAt()) .compare(this.getPendingType(), o.getPendingType()) .result(); }
private void deleteObsoletePendingTasks(SingularityPendingDeploy pendingDeploy) { List<SingularityPendingTaskId> obsoletePendingTasks = taskManager.getPendingTaskIds() .stream() .filter(taskId -> taskId.getRequestId().equals(pendingDeploy.getDeployMarker().getRequestId())) .filter(taskId -> !taskId.getDeployId().equals(pendingDeploy.getDeployMarker().getDeployId())) .collect(Collectors.toList()); for (SingularityPendingTaskId pendingTaskId : obsoletePendingTasks) { LOG.debug("Deleting obsolete pending task {}", pendingTaskId.getId()); taskManager.deletePendingTask(pendingTaskId); } }
private Map<SingularityDeployKey, SingularityPendingTaskId> getDeployKeyToPendingTaskId() { final List<SingularityPendingTaskId> pendingTaskIds = taskManager.getPendingTaskIds(); final Map<SingularityDeployKey, SingularityPendingTaskId> deployKeyToPendingTaskId = Maps.newHashMapWithExpectedSize(pendingTaskIds.size()); for (SingularityPendingTaskId taskId : pendingTaskIds) { SingularityDeployKey deployKey = new SingularityDeployKey(taskId.getRequestId(), taskId.getDeployId()); deployKeyToPendingTaskId.put(deployKey, taskId); } return deployKeyToPendingTaskId; }
public static SingularityScheduledTasksInfo getInfo(List<SingularityPendingTask> pendingTasks, long millisDeltaForLateTasks) { final long now = System.currentTimeMillis(); int numFutureTasks = 0; long maxTaskLag = 0; List<SingularityPendingTaskId> lateTasks = new ArrayList<>(); for (SingularityPendingTask pendingTask : pendingTasks) { long delta = now - pendingTask.getPendingTaskId().getNextRunAt(); if (delta > millisDeltaForLateTasks) { lateTasks.add(pendingTask.getPendingTaskId()); } else { numFutureTasks++; } if (delta > maxTaskLag) { maxTaskLag = delta; } } return new SingularityScheduledTasksInfo(lateTasks, numFutureTasks, maxTaskLag, now); } }
@Override public String toString() { return getId(); }
private boolean shouldRetryImmediately(SingularityRequest request, SingularityDeployStatistics deployStatistics, Optional<SingularityTask> task) { if (!request.getNumRetriesOnFailure().isPresent()) { return false; } if (task.isPresent() && task.get().getTaskRequest().getPendingTask().getPendingTaskId().getPendingType() == PendingType.IMMEDIATE && request.getRequestType() == RequestType.SCHEDULED) { return false; // don't retry UI triggered scheduled jobs (UI triggered on-demand jobs are okay to retry though) } final int numRetriesInARow = deployStatistics.getNumSequentialRetries(); if (numRetriesInARow >= request.getNumRetriesOnFailure().get()) { LOG.debug("Request {} had {} retries in a row, not retrying again (num retries on failure: {})", request.getId(), numRetriesInARow, request.getNumRetriesOnFailure()); return false; } LOG.debug("Request {} had {} retries in a row - retrying again (num retries on failure: {})", request.getId(), numRetriesInARow, request.getNumRetriesOnFailure()); return true; }
@Test public void testNewlyDeployedScheduledTasksAreScheduledAfterStartup() { initScheduledRequest(); initFirstDeploy(); SingularityTask runningTask = launchTask(request, firstDeploy, 1, TaskState.TASK_RUNNING); long now = System.currentTimeMillis(); initSecondDeploy(); requestManager.addToPendingQueue(new SingularityPendingRequest(requestId, secondDeployId, now, Optional.absent(), PendingType.STARTUP, Optional.absent(), Optional.absent())); deployChecker.checkDeploys(); resourceOffers(); // There's an instance running, so we shouldn't schedule a pending task yet Assert.assertTrue(taskManager.getPendingTaskIds().isEmpty()); statusUpdate(runningTask, TaskState.TASK_FINISHED); scheduler.drainPendingQueue(); // Now a pending task should be scheduled with the new deploy Assert.assertEquals(1, taskManager.getPendingTaskIds().size()); Assert.assertEquals(PendingType.NEW_DEPLOY, taskManager.getPendingTaskIds().get(0).getPendingType()); Assert.assertEquals(secondDeployId, taskManager.getPendingTaskIds().get(0).getDeployId()); }
private void prepareEnvironment(final SingularityTaskRequest task, SingularityTaskId taskId, CommandInfo.Builder commandBuilder, final SingularityOfferHolder offerHolder, final Optional<long[]> ports) { Map<String, Object> envVars = new HashMap<>(); envVars.put("INSTANCE_NO", task.getPendingTask().getPendingTaskId().getInstanceNo()); envVars.put("TASK_HOST", offerHolder.getHostname()); envVars.put("AVAILABILITY_ZONE", offerHolder.getRackId()); envVars.put("TASK_REQUEST_ID", task.getPendingTask().getPendingTaskId().getRequestId()); envVars.put("TASK_DEPLOY_ID", taskId.getDeployId()); envVars.put("TASK_ID", taskId.getId());
@Override public String apply(@Nonnull SingularityPendingTaskId input) { return input.getRequestId(); } };
@Override public boolean apply(@Nonnull SingularityPendingTaskId input) { return input.getDeployId().equals(deployId); }
final long fiveHoursInMilliseconds = TimeUnit.HOURS.toMillis(5); final List<SingularityPendingTaskId> pendingTaskIds = taskManager.getPendingTaskIds(); if (pendingTaskIds.get(0).getRequestId().equals(requestEST.getId())) { nextRunEST = pendingTaskIds.get(0).getNextRunAt(); nextRunGMT = pendingTaskIds.get(1).getNextRunAt(); } else { nextRunEST = pendingTaskIds.get(1).getNextRunAt(); nextRunGMT = pendingTaskIds.get(0).getNextRunAt();
Assert.assertEquals(PendingType.TASK_DONE, pendingTaskId.getPendingType()); Assert.assertEquals(lowPriorityRequest.getId(), pendingTaskId.getRequestId());
public SingularityPendingTaskId createFrom(String string, long createdAt) { if (Character.isDigit(string.charAt(string.length() - 1))) { LOG.warn("Not migrating {} - it appears to be migrated already", string); return SingularityPendingTaskId.valueOf(string); } String[] splits = null; try { splits = JavaUtils.reverseSplit(string, 5, "-"); } catch (IllegalStateException ise) { throw new InvalidSingularityTaskIdException(String.format("PendingTaskId %s was invalid (%s)", string, ise.getMessage())); } try { final String requestId = splits[0]; final String deployId = splits[1]; final long nextRunAt = Long.parseLong(splits[2]); final int instanceNo = Integer.parseInt(splits[3]); final PendingType pendingType = PendingType.valueOf(splits[4]); return new SingularityPendingTaskId(requestId, deployId, nextRunAt, instanceNo, pendingType, createdAt); } catch (IllegalArgumentException e) { throw new InvalidSingularityTaskIdException(String.format("PendingTaskId %s had an invalid parameter (%s)", string, e.getMessage())); } }
private void checkActiveRequest(SingularityRequestWithState requestWithState, Map<SingularityDeployKey, SingularityPendingTaskId> deployKeyToPendingTaskId, final long timestamp) { final SingularityRequest request = requestWithState.getRequest(); if (request.getRequestType() == RequestType.ON_DEMAND || request.getRequestType() == RequestType.RUN_ONCE) { return; // There's no situation where we'd want to schedule an On Demand or Run Once request at startup, so don't even bother with them. } Optional<SingularityRequestDeployState> requestDeployState = deployManager.getRequestDeployState(request.getId()); if (!requestDeployState.isPresent() || !requestDeployState.get().getActiveDeploy().isPresent()) { LOG.debug("No active deploy for {} - not scheduling on startup", request.getId()); return; } final String activeDeployId = requestDeployState.get().getActiveDeploy().get().getDeployId(); if (request.isScheduled()) { SingularityDeployKey deployKey = new SingularityDeployKey(request.getId(), activeDeployId); SingularityPendingTaskId pendingTaskId = deployKeyToPendingTaskId.get(deployKey); if (pendingTaskId != null && pendingTaskId.getCreatedAt() >= requestWithState.getTimestamp()) { LOG.info("Not rescheduling {} because {} is newer than {}", request.getId(), pendingTaskId, requestWithState.getTimestamp()); return; } } requestManager.addToPendingQueue(new SingularityPendingRequest(request.getId(), activeDeployId, timestamp, Optional.<String> absent(), PendingType.STARTUP, Optional.<Boolean> absent(), Optional.<String> absent())); }
@Override public int compareTo(SingularityTaskRequest o) { return this.getPendingTask().getPendingTaskId().compareTo(o.getPendingTask().getPendingTaskId()); }
public static SingularityDeployKey fromPendingTask(SingularityPendingTask pendingTask) { return new SingularityDeployKey(pendingTask.getPendingTaskId().getRequestId(), pendingTask.getPendingTaskId().getDeployId()); }
@Timed public List<SingularityTaskRequest> getDueTasks() { final List<SingularityPendingTask> tasks = taskManager.getPendingTasks(); final long now = System.currentTimeMillis(); final List<SingularityPendingTask> dueTasks = Lists.newArrayListWithCapacity(tasks.size()); for (SingularityPendingTask task : tasks) { if (task.getPendingTaskId().getNextRunAt() <= now) { dueTasks.add(task); } } final List<SingularityTaskRequest> dueTaskRequests = taskRequestManager.getTaskRequests(dueTasks); return checkForStaleScheduledTasks(dueTasks, dueTaskRequests); }
private String getPendingPath(SingularityPendingTaskId pendingTaskId) { return ZKPaths.makePath(PENDING_PATH_ROOT, pendingTaskId.getId()); }
private void createTaskAndDeletePendingTaskPrivate(SingularityTask task) throws Exception { // TODO: Should more of the below be done within a transaction? deletePendingTask(task.getTaskRequest().getPendingTask().getPendingTaskId()); final long now = System.currentTimeMillis(); String msg = String.format("Task launched because of %s", task.getTaskRequest().getPendingTask().getPendingTaskId().getPendingType().name()); if (task.getTaskRequest().getPendingTask().getUser().isPresent()) { msg = String.format("%s by %s", msg, task.getTaskRequest().getPendingTask().getUser().get()); } if (task.getTaskRequest().getPendingTask().getMessage().isPresent()) { msg = String.format("%s (%s)", msg, task.getTaskRequest().getPendingTask().getMessage().get()); } saveTaskHistoryUpdate(new SingularityTaskHistoryUpdate(task.getTaskId(), now, ExtendedTaskState.TASK_LAUNCHED, Optional.of(msg), Optional.<String>absent())); saveLastActiveTaskStatus(new SingularityTaskStatusHolder(task.getTaskId(), Optional.absent(), now, serverId, Optional.of(task.getAgentId().getValue()))); try { final String path = getTaskPath(task.getTaskId()); CuratorTransactionFinal transaction = curator.inTransaction().create().forPath(path, taskTranscoder.toBytes(task)).and(); transaction.create().forPath(getActivePath(task.getTaskId().getId())).and().commit(); leaderCache.putActiveTask(task); taskCache.set(path, task); } catch (KeeperException.NodeExistsException nee) { LOG.error("Task or active path already existed for {}", task.getTaskId()); } }