private String fillInTaskIdValues(String string, SingularityOfferHolder offerHolder, SingularityTaskId taskId) { if (!Strings.isNullOrEmpty(string)) { string = string.replace("${TASK_REQUEST_ID}", taskId.getRequestId()) .replace("${TASK_DEPLOY_ID}", taskId.getDeployId()) .replace("${TASK_STARTED_AT}", Long.toString(taskId.getStartedAt())) .replace("${TASK_INSTANCE_NO}", Integer.toString(taskId.getInstanceNo())) .replace("${TASK_HOST}", offerHolder.getHostname()) .replace("${TASK_RACK_ID}", offerHolder.getRackId()) .replace("${TASK_ID}", taskId.getId()); } return string; }
private SingularityTaskId activeTask() { return new SingularityTaskId( "requestId", "deployId", System.currentTimeMillis(), 1, "host", "rack" ); }
private void assertEquals(SingularityTaskId one, SingularityTaskId two) { Assert.assertEquals(one, two); Assert.assertEquals(one.getDeployId(), two.getDeployId()); Assert.assertEquals(one.getRequestId(), two.getRequestId()); Assert.assertEquals(one.getSanitizedHost(), two.getSanitizedHost()); Assert.assertEquals(one.getSanitizedRackId(), two.getSanitizedRackId()); Assert.assertEquals(one.getStartedAt(), two.getStartedAt()); Assert.assertEquals(one.getInstanceNo(), two.getInstanceNo()); }
public static String getS3KeyFormat(String s3KeyFormat, SingularityTaskId taskId, Optional<String> loggingTag, String group) { s3KeyFormat = getS3KeyFormat(s3KeyFormat, taskId.getRequestId(), taskId.getDeployId(), loggingTag, group); s3KeyFormat = s3KeyFormat.replace("%host", taskId.getSanitizedHost()); s3KeyFormat = s3KeyFormat.replace("%taskId", taskId.toString()); return s3KeyFormat; }
private boolean hasTaskLeftOnRack(SingularityTaskId taskId, SingularityLeaderCache leaderCache) { for (SingularityTaskId activeTaskId : leaderCache.getActiveTaskIds()) { if (!activeTaskId.equals(taskId) && activeTaskId.getSanitizedRackId().equals(taskId.getSanitizedRackId())) { return true; } } return false; }
public static SingularityDeployKey fromTaskId(SingularityTaskId taskId) { return new SingularityDeployKey(taskId.getRequestId(), taskId.getDeployId()); }
@Override public void saveTaskHistory(SingularityTaskHistory taskHistory) { if (history.getTaskHistoryForTask(taskHistory.getTask().getTaskId().getId()) != null) { if (LOG.isTraceEnabled()) { LOG.trace("saveTaskHistory -- existing taskHistory {}", taskHistory); } return; } SingularityTaskIdHistory taskIdHistory = SingularityTaskIdHistory.fromTaskIdAndTaskAndUpdates(taskHistory.getTask().getTaskId(), taskHistory.getTask(), taskHistory.getTaskUpdates()); String lastTaskStatus = null; if (taskIdHistory.getLastTaskState().isPresent()) { lastTaskStatus = taskIdHistory.getLastTaskState().get().name(); } if (LOG.isTraceEnabled()) { LOG.trace("saveTaskHistory -- will insert taskHistory {}", taskHistory); } history.insertTaskHistory(taskIdHistory.getTaskId().getRequestId(), taskIdHistory.getTaskId().getId(), taskHistoryTranscoder.toBytes(taskHistory), new Date(taskIdHistory.getUpdatedAt()), lastTaskStatus, taskHistory.getTask().getTaskRequest().getPendingTask().getRunId().orNull(), taskIdHistory.getTaskId().getDeployId(), taskIdHistory.getTaskId().getHost(), new Date(taskIdHistory.getTaskId().getStartedAt())); }
public Logger buildTaskLogger(String taskId, String executorId, String executorPid, String taskLogFile) { LOG.info("Building a task logger for {} pointing to {}", taskId, taskLogFile); LoggerContext context = new LoggerContext(); context.setName(executorPid); baseLogging.prepareRootLogger(context); String loggerId = taskId; try { SingularityTaskId singularityTaskId = SingularityTaskId.valueOf(taskId); loggerId = String.format("%s.%s.%s.%s.%s", singularityTaskId.getRequestId(), singularityTaskId.getDeployId(), singularityTaskId.getStartedAt(), singularityTaskId.getInstanceNo(), executorId); } catch (InvalidSingularityTaskIdException e) { LOG.info("Handling non-SingularityTaskId %s", taskId); } Logger taskLogger = context.getLogger(loggerId); taskLogger.detachAndStopAllAppenders(); if (baseLogging.getRootLogPath().isPresent()) { taskLogger.addAppender(baseLogging.buildFileAppender(context, baseLogging.getRootLogPath().get())); } taskLogger.addAppender(baseLogging.buildFileAppender(context, taskLogFile)); context.start(); return taskLogger; }
return request.getScheduledExpectedRuntimeMillis(); } else { final Optional<SingularityDeployStatistics> deployStatistics = deployManager.getDeployStatistics(taskId.getRequestId(), taskId.getDeployId()); } else { final CronExpression cronExpression = new CronExpression(scheduleExpression); final Date startDate = new Date(taskId.getStartedAt()); nextRunAtDate = cronExpression.getNextValidTimeAfter(startDate); String msg = String.format("No next run date found for %s (%s)", taskId, scheduleExpression); LOG.warn(msg); exceptionNotifier.notify(msg, ImmutableMap.of("taskId", taskId.toString())); return Optional.absent(); exceptionNotifier.notify(String.format("Unable to parse schedule (%s)", e.getMessage()), e, ImmutableMap.of("taskId", taskId.toString(), "scheduleExpression", scheduleExpression, "scheduleType", request.getScheduleTypeSafe().toString())); return Optional.absent(); return Optional.of(nextRunAtDate.getTime() - taskId.getStartedAt());
private String applyPlaceholders(String uri, SingularityTaskHistoryUpdate taskUpdate) { return uri .replaceAll("\\$REQUEST_ID", taskUpdate.getTaskId().getRequestId()) .replaceAll("\\$DEPLOY_ID", taskUpdate.getTaskId().getDeployId()) .replaceAll("\\$TASK_ID", taskUpdate.getTaskId().getId()); }
@Override public String toString() { return getId(); } }
@Test public void testSingularityTaskIdSerialization() throws Exception { ObjectMapper om = Jackson.newObjectMapper() .setSerializationInclusion(Include.NON_NULL) .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false) .registerModule(new ProtobufModule()); SingularityTaskId taskId = new SingularityTaskId("rid", "did", 100, 1, "host", "rack"); String id = taskId.getId(); SingularityTaskId fromId = SingularityTaskId.valueOf(id); SingularityTaskId fromJson = om.readValue(om.writeValueAsBytes(taskId), SingularityTaskId.class); assertEquals(taskId, fromId); assertEquals(taskId, fromJson); assertEquals(fromId, fromJson); }
if (!cleaningTasks.contains(taskId) && taskRequest.getDeploy().getId().equals(taskId.getDeployId())) { countPerRack.add(taskId.getSanitizedRackId()); if (!taskId.getSanitizedHost().equals(sanitizedHost)) { continue; if (taskRequest.getDeploy().getId().equals(taskId.getDeployId())) { if (cleaningTasks.contains(taskId)) { numCleaningOnSlave++;
private Runnable getTaskCheck(final SingularityTask task, final SingularityHealthchecker healthchecker) { return () -> { try { Optional<SingularityRequestWithState> requestWithState = requestManager.getRequest(task.getTaskId().getRequestId()); if (!requestWithState.isPresent()) { LOG.info("Ignoring task check for {}, missing request {}", task.getTaskId(), task.getTaskId().getRequestId()); return; } boolean shouldReschedule = checkTask(task, requestWithState, healthchecker); if (shouldReschedule) { reEnqueueCheck(task, healthchecker); } else { taskIdToCheck.remove(task.getTaskId().getId()); } } catch (Throwable t) { LOG.error("Uncaught throwable in task check for task {}, re-enqueing", task, t); exceptionNotifier.notify(String.format("Error in task check (%s)", t.getMessage()), t, ImmutableMap.of("taskId", task.getTaskId().toString())); reEnqueueCheckOrAbort(task, healthchecker); } }; }
envVars.put("TASK_DEPLOY_ID", taskId.getDeployId()); envVars.put("TASK_ID", taskId.getId()); envVars.put("ESTIMATED_INSTANCE_COUNT", task.getRequest().getInstancesSafe()); if (task.getDeploy().getTaskEnv().isPresent() && task.getDeploy().getTaskEnv().get().containsKey(taskId.getInstanceNo()) && !task.getDeploy().getTaskEnv().get().get(taskId.getInstanceNo()).isEmpty()) { for (Entry<String, String> envEntry : task.getDeploy().getTaskEnv().get().get(taskId.getInstanceNo()).entrySet()) { envVars.put(envEntry.getKey(), fillInTaskIdValues(envEntry.getValue(), offerHolder, taskId));
private boolean isLongRunning(SingularityTaskId task) { Optional<SingularityRequestWithState> request = requestManager.getRequest(task.getRequestId()); if (request.isPresent()) { return request.get().getRequest().getRequestType().isLongRunning(); } LOG.warn("Couldn't find request id {} for task {}", task.getRequestId(), task.getId()); return false; }
.peek((offerHolder) -> { taskManager.getActiveTaskIds().forEach((t) -> { if (t.getSanitizedHost().equals(offerHolder.getSanitizedHost())) { relevantRequestIds.add(t.getRequestId()); .filter((t) -> t.getStartedAt() > maybeSlaveUsage.get().getTimestamp() && t.getSanitizedHost().equals(offerHolder.getSanitizedHost())) .count(); if (newTaskCount >= maybeSlaveUsage.get().getNumTasks() / 2) {
@Override public int compare(SingularityTaskIdHistory o1, SingularityTaskIdHistory o2) { ComparisonChain chain = ComparisonChain.start(); if (localOrderDirection == OrderDirection.ASC) { chain = chain.compare(o1.getTaskId().getStartedAt(), o2.getTaskId().getStartedAt()); } else { chain = chain.compare(o2.getTaskId().getStartedAt(), o1.getTaskId().getStartedAt()); } return chain.compare(o1.getTaskId().getRequestId(), o2.getTaskId().getRequestId()).result(); }
final String sanitizedHost = offerHolder.getSanitizedHost(); final SingularityTaskId taskId = new SingularityTaskId(taskRequest.getPendingTask().getPendingTaskId().getRequestId(), taskRequest.getDeploy().getId(), System.currentTimeMillis(), taskRequest.getPendingTask().getPendingTaskId().getInstanceNo(), sanitizedHost, sanitizedRackId); .setTaskId(TaskID.newBuilder().setValue(taskId.toString()));
public List<SingularityTaskCurrentUsageWithId> getTaskCurrentUsages(List<SingularityTaskId> taskIds) { List<String> paths = new ArrayList<>(taskIds.size()); for (SingularityTaskId taskId : taskIds) { paths.add(getCurrentTaskUsagePath(taskId.getId())); } Map<String, SingularityTaskCurrentUsage> currentTaskUsages = getAsyncWithPath("getTaskCurrentUsages", paths, taskCurrentUsageTranscoder); List<SingularityTaskCurrentUsageWithId> currentTaskUsagesWithIds = new ArrayList<>(paths.size()); for (Entry<String, SingularityTaskCurrentUsage> entry : currentTaskUsages.entrySet()) { currentTaskUsagesWithIds.add(new SingularityTaskCurrentUsageWithId(SingularityTaskId.valueOf(getTaskIdFromCurrentUsagePath(entry.getKey())), entry.getValue())); } return currentTaskUsagesWithIds; }