public SingularityRequestHistoryParent(List<SingularityRequestHistory> history, String requestId) { this.history = history; this.requestId = requestId; long newestTimestamp = 0; for (SingularityRequestHistory historyItem : history) { if (historyItem.getCreatedAt() > newestTimestamp) { newestTimestamp = historyItem.getCreatedAt(); } } createTime = newestTimestamp; }
@Override public int compareTo(SingularityRequestHistory o) { return ComparisonChain .start() .compare(o.getCreatedAt(), createdAt) .compare(request.getId(), o.getRequest().getId()) .result(); }
private boolean healthchecksSkipped(SingularityTaskId taskId, List<SingularityRequestHistory> requestHistories, SingularityDeploy deploy) { if (deploy.getSkipHealthchecksOnDeploy().or(false)) { return true; } Optional<SingularityTask> maybeTask = taskManager.getTask(taskId); if (maybeTask.isPresent()) { if (maybeTask.get().getTaskRequest().getPendingTask().getSkipHealthchecks().or(false)) { return true; } Optional<Long> runningStartTime = getRunningAt(taskManager.getTaskHistoryUpdates(taskId)); if (runningStartTime.isPresent()) { Optional<SingularityRequestHistory> previousHistory = Optional.absent(); for (SingularityRequestHistory history : requestHistories) { if (history.getCreatedAt() < runningStartTime.get() && (!previousHistory.isPresent() || previousHistory.get().getCreatedAt() < history.getCreatedAt())) { previousHistory = Optional.of(history); } } if (previousHistory.isPresent() && previousHistory.get().getRequest().getSkipHealthchecks().or(false)) { return true; } } } return false; }
private long getLastActionTimeForRequest(SingularityRequest request, Optional<SingularityRequestHistory> lastHistory, Optional<SingularityRequestDeployState> deployState, Optional<SingularityTaskIdHistory> mostRecentTask) { long lastUpdate = 0; if (lastHistory.isPresent()) { lastUpdate = lastHistory.get().getCreatedAt(); } if (deployState.isPresent()) { if (deployState.get().getActiveDeploy().isPresent()) { lastUpdate = Math.max(lastUpdate, deployState.get().getActiveDeploy().get().getTimestamp()); } if (deployState.get().getPendingDeploy().isPresent()) { lastUpdate = Math.max(lastUpdate, deployState.get().getPendingDeploy().get().getTimestamp()); } } // Only consider most recent task time for non-long-running if (mostRecentTask.isPresent() && !request.getRequestType().isLongRunning()) { lastUpdate = Math.max(lastUpdate, mostRecentTask.get().getUpdatedAt()); } return lastUpdate; } }
private Collection<String> getS3PrefixesForRequest(S3Configuration s3Configuration, String requestId, Optional<Long> startArg, Optional<Long> endArg, String group) { Optional<SingularityRequestHistory> firstHistory = requestHistoryHelper.getFirstHistory(requestId); checkNotFound(firstHistory.isPresent(), "No request history found for %s", requestId); long start = firstHistory.get().getCreatedAt(); if (startArg.isPresent()) { start = Math.max(startArg.get(), start); } Optional<SingularityRequestHistory> lastHistory = requestHistoryHelper.getLastHistory(requestId); long end = System.currentTimeMillis(); if (lastHistory.isPresent() && (lastHistory.get().getEventType() == RequestHistoryType.DELETED || lastHistory.get().getEventType() == RequestHistoryType.PAUSED)) { end = lastHistory.get().getCreatedAt() + TimeUnit.DAYS.toMillis(1); } if (endArg.isPresent()) { end = Math.min(endArg.get(), end); } Collection<String> prefixes = SingularityS3FormatHelper.getS3KeyPrefixes(s3Configuration.getS3KeyFormat(), requestId, start, end, group); for (SingularityS3UploaderFile additionalFile : s3Configuration.getS3UploaderAdditionalFiles()) { if (additionalFile.getS3UploaderKeyPattern().isPresent() && !additionalFile.getS3UploaderKeyPattern().get().equals(s3Configuration.getS3KeyFormat())) { prefixes.addAll(SingularityS3FormatHelper.getS3KeyPrefixes(additionalFile.getS3UploaderKeyPattern().get(), requestId, start, end, group)); } } LOG.trace("Request {} got S3 prefixes {} for start {}, end {}", requestId, prefixes, start, end); return prefixes; }
private String getRequestHistoryUpdateId(SingularityRequestHistory requestUpdate) { return requestUpdate.getRequest().getId() + "-" + requestUpdate.getEventType().name() + "-" + requestUpdate.getCreatedAt(); }
private int checkRequestUpdates(SingularityWebhook webhook, List<CompletableFuture<Response>> webhookFutures) { final List<SingularityRequestHistory> requestUpdates = webhookManager.getQueuedRequestHistoryForHook(webhook.getId()); int numRequestUpdates = 0; for (SingularityRequestHistory requestUpdate : requestUpdates) { String concreteUri = applyPlaceholders(webhook.getUri(), requestUpdate); webhookFutures.add(webhookSemaphore.call(() -> executeWebhookAsync( concreteUri, requestUpdate, new SingularityRequestWebhookAsyncHandler(webhookManager, webhook, requestUpdate, shouldDeleteUpdateOnFailure(numRequestUpdates, requestUpdate.getCreatedAt()))) )); } return requestUpdates.size(); }
@Override public void saveRequestHistoryUpdate(SingularityRequestHistory requestHistory) { if (LOG.isTraceEnabled()) { LOG.trace("saveRequestHistoryUpdate requestHistory {}", requestHistory); } history.insertRequestHistory(requestHistory.getRequest().getId(), singularityRequestTranscoder.toBytes(requestHistory.getRequest()), new Date(requestHistory.getCreatedAt()), requestHistory.getEventType().name(), getUserField(requestHistory.getUser()), getMessageField(requestHistory.getMessage())); }
private String getHistoryPath(SingularityRequestHistory history) { return ZKPaths.makePath(getHistoryParentPath(history.getRequest().getId()), history.getEventType() + "-" + history.getCreatedAt()); }
Assert.assertTrue(history.get(0).getCreatedAt() == 120); Assert.assertTrue(history.get(4).getCreatedAt() == 50); Assert.assertTrue(history.get(0).getCreatedAt() == 100); Assert.assertTrue(history.get(3).getCreatedAt() == 50); Assert.assertTrue(history.get(0).getCreatedAt() == 52); Assert.assertTrue(history.get(2).getCreatedAt() == 50); Assert.assertTrue(history.get(0).getCreatedAt() == 100); Assert.assertTrue(rhh.getFirstHistory(rid).get().getCreatedAt() == 100); Assert.assertTrue(rhh.getLastHistory(rid).get().getCreatedAt() == 120); Assert.assertTrue(rhh.getFirstHistory(rid).get().getCreatedAt() == 1); Assert.assertTrue(rhh.getLastHistory(rid).get().getCreatedAt() == 120);
public SingularityRequestHistoryParent(List<SingularityRequestHistory> history, String requestId) { this.history = history; this.requestId = requestId; long newestTimestamp = 0; for (SingularityRequestHistory historyItem : history) { if (historyItem.getCreatedAt() > newestTimestamp) { newestTimestamp = historyItem.getCreatedAt(); } } createTime = newestTimestamp; }
private boolean healthchecksSkipped(SingularityTaskId taskId, List<SingularityRequestHistory> requestHistories, SingularityDeploy deploy) { if (deploy.getSkipHealthchecksOnDeploy().or(false)) { return true; } Optional<SingularityTask> maybeTask = taskManager.getTask(taskId); if (maybeTask.isPresent()) { if (maybeTask.get().getTaskRequest().getPendingTask().getSkipHealthchecks().or(false)) { return true; } Optional<Long> runningStartTime = getRunningAt(taskManager.getTaskHistoryUpdates(taskId)); if (runningStartTime.isPresent()) { Optional<SingularityRequestHistory> previousHistory = Optional.absent(); for (SingularityRequestHistory history : requestHistories) { if (history.getCreatedAt() < runningStartTime.get() && (!previousHistory.isPresent() || previousHistory.get().getCreatedAt() < history.getCreatedAt())) { previousHistory = Optional.of(history); } } if (previousHistory.isPresent() && previousHistory.get().getRequest().getSkipHealthchecks().or(false)) { return true; } } } return false; }
private long getLastActionTimeForRequest(SingularityRequest request, Optional<SingularityRequestHistory> lastHistory, Optional<SingularityRequestDeployState> deployState, Optional<SingularityTaskIdHistory> mostRecentTask) { long lastUpdate = 0; if (lastHistory.isPresent()) { lastUpdate = lastHistory.get().getCreatedAt(); } if (deployState.isPresent()) { if (deployState.get().getActiveDeploy().isPresent()) { lastUpdate = Math.max(lastUpdate, deployState.get().getActiveDeploy().get().getTimestamp()); } if (deployState.get().getPendingDeploy().isPresent()) { lastUpdate = Math.max(lastUpdate, deployState.get().getPendingDeploy().get().getTimestamp()); } } // Only consider most recent task time for non-long-running if (mostRecentTask.isPresent() && !request.getRequestType().isLongRunning()) { lastUpdate = Math.max(lastUpdate, mostRecentTask.get().getUpdatedAt()); } return lastUpdate; } }
private Collection<String> getS3PrefixesForRequest(S3Configuration s3Configuration, String requestId, Optional<Long> startArg, Optional<Long> endArg, String group) { Optional<SingularityRequestHistory> firstHistory = requestHistoryHelper.getFirstHistory(requestId); checkNotFound(firstHistory.isPresent(), "No request history found for %s", requestId); long start = firstHistory.get().getCreatedAt(); if (startArg.isPresent()) { start = Math.max(startArg.get(), start); } Optional<SingularityRequestHistory> lastHistory = requestHistoryHelper.getLastHistory(requestId); long end = System.currentTimeMillis(); if (lastHistory.isPresent() && (lastHistory.get().getEventType() == RequestHistoryType.DELETED || lastHistory.get().getEventType() == RequestHistoryType.PAUSED)) { end = lastHistory.get().getCreatedAt() + TimeUnit.DAYS.toMillis(1); } if (endArg.isPresent()) { end = Math.min(endArg.get(), end); } Collection<String> prefixes = SingularityS3FormatHelper.getS3KeyPrefixes(s3Configuration.getS3KeyFormat(), requestId, start, end, group); for (SingularityS3UploaderFile additionalFile : s3Configuration.getS3UploaderAdditionalFiles()) { if (additionalFile.getS3UploaderKeyPattern().isPresent() && !additionalFile.getS3UploaderKeyPattern().get().equals(s3Configuration.getS3KeyFormat())) { prefixes.addAll(SingularityS3FormatHelper.getS3KeyPrefixes(additionalFile.getS3UploaderKeyPattern().get(), requestId, start, end, group)); } } LOG.trace("Request {} got S3 prefixes {} for start {}, end {}", requestId, prefixes, start, end); return prefixes; }
private String getRequestHistoryUpdateId(SingularityRequestHistory requestUpdate) { return requestUpdate.getRequest().getId() + "-" + requestUpdate.getEventType().name() + "-" + requestUpdate.getCreatedAt(); }
private int checkRequestUpdates(SingularityWebhook webhook, List<CompletableFuture<Response>> webhookFutures) { final List<SingularityRequestHistory> requestUpdates = webhookManager.getQueuedRequestHistoryForHook(webhook.getId()); int numRequestUpdates = 0; for (SingularityRequestHistory requestUpdate : requestUpdates) { String concreteUri = applyPlaceholders(webhook.getUri(), requestUpdate); webhookFutures.add(webhookSemaphore.call(() -> executeWebhookAsync( concreteUri, requestUpdate, new SingularityRequestWebhookAsyncHandler(webhookManager, webhook, requestUpdate, shouldDeleteUpdateOnFailure(numRequestUpdates, requestUpdate.getCreatedAt()))) )); } return requestUpdates.size(); }
@Override public void saveRequestHistoryUpdate(SingularityRequestHistory requestHistory) { if (LOG.isTraceEnabled()) { LOG.trace("saveRequestHistoryUpdate requestHistory {}", requestHistory); } history.insertRequestHistory(requestHistory.getRequest().getId(), singularityRequestTranscoder.toBytes(requestHistory.getRequest()), new Date(requestHistory.getCreatedAt()), requestHistory.getEventType().name(), getUserField(requestHistory.getUser()), getMessageField(requestHistory.getMessage())); }
private String getHistoryPath(SingularityRequestHistory history) { return ZKPaths.makePath(getHistoryParentPath(history.getRequest().getId()), history.getEventType() + "-" + history.getCreatedAt()); }
Assert.assertTrue(history.get(0).getCreatedAt() == 120); Assert.assertTrue(history.get(4).getCreatedAt() == 50); Assert.assertTrue(history.get(0).getCreatedAt() == 100); Assert.assertTrue(history.get(3).getCreatedAt() == 50); Assert.assertTrue(history.get(0).getCreatedAt() == 52); Assert.assertTrue(history.get(2).getCreatedAt() == 50); Assert.assertTrue(history.get(0).getCreatedAt() == 100); Assert.assertTrue(rhh.getFirstHistory(rid).get().getCreatedAt() == 100); Assert.assertTrue(rhh.getLastHistory(rid).get().getCreatedAt() == 120); Assert.assertTrue(rhh.getFirstHistory(rid).get().getCreatedAt() == 1); Assert.assertTrue(rhh.getLastHistory(rid).get().getCreatedAt() == 120);