@Override protected List<SingularityRequestHistory> getFromZk(String requestId) { List<SingularityRequestHistory> requestHistory = requestManager.getRequestHistory(requestId); Collections.sort(requestHistory); return requestHistory; }
public Optional<SingularityRequestHistory> getMostRecentHistoryFromZk(String requestId) { // Most recent history is stored in zk, don't need to check mysql List<SingularityRequestHistory> requestHistory = requestManager.getRequestHistory(requestId); return JavaUtils.getFirst(requestHistory); }
@Override protected Optional<Integer> getTotalCount(String requestId) { int numFromZk = requestManager.getRequestHistory(requestId).size(); int numFromHistory = historyManager.getRequestHistoryCount(requestId); return Optional.of(numFromZk + numFromHistory); }
private List<SingularityTaskId> getHealthcheckedHealthyTasks(final SingularityDeploy deploy, final Collection<SingularityTaskId> matchingActiveTasks, final boolean isDeployPending) { final Map<SingularityTaskId, SingularityTaskHealthcheckResult> healthcheckResults = taskManager.getLastHealthcheck(matchingActiveTasks); final List<SingularityTaskId> healthyTaskIds = Lists.newArrayListWithCapacity(matchingActiveTasks.size()); List<SingularityRequestHistory> requestHistories = requestManager.getRequestHistory(deploy.getRequestId()); for (SingularityTaskId taskId : matchingActiveTasks) { DeployHealth individualTaskHealth; if (healthchecksSkipped(taskId, requestHistories, deploy)) { LOG.trace("Detected skipped healthchecks for {}", taskId); individualTaskHealth = DeployHealth.HEALTHY; } else { individualTaskHealth = getTaskHealth(deploy, isDeployPending, Optional.fromNullable(healthcheckResults.get(taskId)), taskId); } if (individualTaskHealth == DeployHealth.HEALTHY) { healthyTaskIds.add(taskId); } } return healthyTaskIds; }
@Override public void runActionOnPoll() { LOG.info("Attempting to grab persister lock"); persisterLock.lock(); try { LOG.info("Checking request history for persistence"); final long start = System.currentTimeMillis(); final List<SingularityRequestHistoryParent> requestHistoryParents = new ArrayList(); AtomicInteger numHistoryTransferred = new AtomicInteger(); for (String requestId : requestManager.getRequestIdsWithHistory()) { requestHistoryParents.add(new SingularityRequestHistoryParent(requestManager.getRequestHistory(requestId), requestId)); } Collections.sort(requestHistoryParents, Collections.reverseOrder()); // createdAt descending AtomicInteger i = new AtomicInteger(); for (SingularityRequestHistoryParent requestHistoryParent : requestHistoryParents) { lock.runWithRequestLock(() -> { if (moveToHistoryOrCheckForPurge(requestHistoryParent, i.getAndIncrement())) { numHistoryTransferred.getAndAdd(requestHistoryParent.history.size()); } }, requestHistoryParent.requestId, "request history purger"); } LOG.info("Transferred {} history updates for {} requests in {}", numHistoryTransferred, requestHistoryParents.size(), JavaUtils.duration(start)); } finally { persisterLock.unlock(); } }
private DeployHealth getHealthcheckDeployState(final SingularityDeploy deploy, final Collection<SingularityTaskId> matchingActiveTasks, final boolean isDeployPending) { Map<SingularityTaskId, SingularityTaskHealthcheckResult> healthcheckResults = taskManager.getLastHealthcheck(matchingActiveTasks); List<SingularityRequestHistory> requestHistories = requestManager.getRequestHistory(deploy.getRequestId()); for (SingularityTaskId taskId : matchingActiveTasks) { DeployHealth individualTaskHealth; if (healthchecksSkipped(taskId, requestHistories, deploy)) { LOG.trace("Detected skipped healthchecks for {}", taskId); individualTaskHealth = DeployHealth.HEALTHY; } else { individualTaskHealth = getTaskHealth(deploy, isDeployPending, Optional.fromNullable(healthcheckResults.get(taskId)), taskId); } if (individualTaskHealth != DeployHealth.HEALTHY) { return individualTaskHealth; } } return DeployHealth.HEALTHY; }
@Test public void testRequestAgePurging() { initRequest(); configuration.setDeleteStaleRequestsFromZkWhenNoDatabaseAfterHours(7); requestHistoryPersister.runActionOnPoll(); Assert.assertTrue(!requestManager.getRequestHistory(requestId).isEmpty()); requestManager.startDeletingRequest(request, Optional.absent(), user, Optional.<String> absent(), Optional.<String> absent()); requestManager.deleteHistoryParent(requestId); requestManager.activate(request, RequestHistoryType.CREATED, System.currentTimeMillis() - TimeUnit.HOURS.toMillis(3), Optional.<String> absent(), Optional.<String> absent()); requestManager.cooldown(request, System.currentTimeMillis() - TimeUnit.HOURS.toMillis(2)); requestHistoryPersister.runActionOnPoll(); Assert.assertTrue(!requestManager.getRequestHistory(requestId).isEmpty()); configuration.setDeleteStaleRequestsFromZkWhenNoDatabaseAfterHours(1); requestHistoryPersister.runActionOnPoll(); Assert.assertTrue(requestManager.getRequestHistory(requestId).isEmpty()); }
@Test public void itCorrectlyUpdatesRequestDeletingStateHistory() { initRequest(); Assert.assertEquals(RequestState.ACTIVE, requestManager.getRequest(requestId).get().getState()); Assert.assertEquals(1, requestManager.getRequestHistory(requestId).size()); requestManager.startDeletingRequest(request, Optional.absent(), Optional.<String>absent(), Optional.<String>absent(), Optional.of("the cake is a lie")); Assert.assertEquals(RequestState.DELETING, requestManager.getRequest(requestId).get().getState()); Assert.assertEquals(2, requestManager.getRequestHistory(requestId).size()); cleaner.drainCleanupQueue(); Assert.assertEquals(3, requestManager.getRequestHistory(requestId).size()); List<RequestHistoryType> historyTypes = new ArrayList<>(); for (SingularityRequestHistory request : requestManager.getRequestHistory(requestId)) { historyTypes.add(request.getEventType()); } Assert.assertTrue(historyTypes.contains(RequestHistoryType.CREATED)); Assert.assertTrue(historyTypes.contains(RequestHistoryType.DELETING)); Assert.assertTrue(historyTypes.contains(RequestHistoryType.DELETED)); }
@Test public void testPurgingDoesntApplyIfDatabasePresent() { initRequest(); initFirstDeploy(); requestManager.startDeletingRequest(request, Optional.absent(), user, Optional.<String> absent(), Optional.<String> absent()); requestManager.deleteHistoryParent(requestId); requestManager.activate(request, RequestHistoryType.CREATED, System.currentTimeMillis() - TimeUnit.HOURS.toMillis(3), Optional.<String> absent(), Optional.<String> absent()); configuration.setDatabaseConfiguration(new DataSourceFactory()); configuration.setDeleteStaleRequestsFromZkWhenNoDatabaseAfterHours(1); requestHistoryPersister.runActionOnPoll(); Assert.assertTrue(!requestManager.getRequestHistory(requestId).isEmpty()); }
@Test public void testRequestCountPurging() { final SingularityRequest requestOne = new SingularityRequestBuilder("request1", RequestType.WORKER).build(); final SingularityRequest requestTwo = new SingularityRequestBuilder("request2", RequestType.WORKER).build(); final SingularityRequest requestThree = new SingularityRequestBuilder("request3", RequestType.WORKER).build(); saveRequest(requestOne); saveRequest(requestTwo); saveRequest(requestThree); configuration.setMaxRequestsWithHistoryInZkWhenNoDatabase(Optional.of(2)); configuration.setDeleteStaleRequestsFromZkWhenNoDatabaseAfterHours(7); requestManager.startDeletingRequest(requestOne, Optional.absent(), user, Optional.<String>absent(), Optional.<String>absent()); requestManager.deleteHistoryParent(requestOne.getId()); requestManager.activate(requestOne, RequestHistoryType.CREATED, System.currentTimeMillis() - TimeUnit.HOURS.toMillis(4), Optional.<String> absent(), Optional.<String> absent()); requestManager.cooldown(requestOne, System.currentTimeMillis() - TimeUnit.HOURS.toMillis(3)); requestManager.startDeletingRequest(requestTwo, Optional.absent(), user, Optional.<String>absent(), Optional.<String>absent()); requestManager.deleteHistoryParent(requestTwo.getId()); requestManager.activate(requestTwo, RequestHistoryType.CREATED, System.currentTimeMillis() - TimeUnit.HOURS.toMillis(4), Optional.<String> absent(), Optional.<String> absent()); requestManager.cooldown(requestTwo, System.currentTimeMillis() - TimeUnit.HOURS.toMillis(3)); requestManager.startDeletingRequest(requestThree, Optional.absent(), user, Optional.<String>absent(), Optional.<String>absent()); requestManager.deleteHistoryParent(requestThree.getId()); requestManager.activate(requestThree, RequestHistoryType.CREATED, System.currentTimeMillis() - TimeUnit.HOURS.toMillis(4), Optional.<String> absent(), Optional.<String> absent()); requestManager.cooldown(requestThree, System.currentTimeMillis() - TimeUnit.HOURS.toMillis(3)); Assert.assertEquals(2, requestManager.getRequestHistory(requestOne.getId()).size()); Assert.assertEquals(2, requestManager.getRequestHistory(requestTwo.getId()).size()); Assert.assertEquals(2, requestManager.getRequestHistory(requestThree.getId()).size()); requestHistoryPersister.runActionOnPoll(); Assert.assertEquals(0, requestManager.getRequestHistory(requestOne.getId()).size()); Assert.assertEquals(2, requestManager.getRequestHistory(requestTwo.getId()).size()); Assert.assertEquals(2, requestManager.getRequestHistory(requestThree.getId()).size()); }
@Override protected List<SingularityRequestHistory> getFromZk(String requestId) { List<SingularityRequestHistory> requestHistory = requestManager.getRequestHistory(requestId); Collections.sort(requestHistory); return requestHistory; }
@Override protected Optional<Integer> getTotalCount(String requestId) { int numFromZk = requestManager.getRequestHistory(requestId).size(); int numFromHistory = historyManager.getRequestHistoryCount(requestId); return Optional.of(numFromZk + numFromHistory); }
public Optional<SingularityRequestHistory> getMostRecentHistoryFromZk(String requestId) { // Most recent history is stored in zk, don't need to check mysql List<SingularityRequestHistory> requestHistory = requestManager.getRequestHistory(requestId); return JavaUtils.getFirst(requestHistory); }
private List<SingularityTaskId> getHealthcheckedHealthyTasks(final SingularityDeploy deploy, final Collection<SingularityTaskId> matchingActiveTasks, final boolean isDeployPending) { final Map<SingularityTaskId, SingularityTaskHealthcheckResult> healthcheckResults = taskManager.getLastHealthcheck(matchingActiveTasks); final List<SingularityTaskId> healthyTaskIds = Lists.newArrayListWithCapacity(matchingActiveTasks.size()); List<SingularityRequestHistory> requestHistories = requestManager.getRequestHistory(deploy.getRequestId()); for (SingularityTaskId taskId : matchingActiveTasks) { DeployHealth individualTaskHealth; if (healthchecksSkipped(taskId, requestHistories, deploy)) { LOG.trace("Detected skipped healthchecks for {}", taskId); individualTaskHealth = DeployHealth.HEALTHY; } else { individualTaskHealth = getTaskHealth(deploy, isDeployPending, Optional.fromNullable(healthcheckResults.get(taskId)), taskId); } if (individualTaskHealth == DeployHealth.HEALTHY) { healthyTaskIds.add(taskId); } } return healthyTaskIds; }
@Override public void runActionOnPoll() { LOG.info("Attempting to grab persister lock"); persisterLock.lock(); try { LOG.info("Checking request history for persistence"); final long start = System.currentTimeMillis(); final List<SingularityRequestHistoryParent> requestHistoryParents = new ArrayList(); AtomicInteger numHistoryTransferred = new AtomicInteger(); for (String requestId : requestManager.getRequestIdsWithHistory()) { requestHistoryParents.add(new SingularityRequestHistoryParent(requestManager.getRequestHistory(requestId), requestId)); } Collections.sort(requestHistoryParents, Collections.reverseOrder()); // createdAt descending AtomicInteger i = new AtomicInteger(); for (SingularityRequestHistoryParent requestHistoryParent : requestHistoryParents) { lock.runWithRequestLock(() -> { if (moveToHistoryOrCheckForPurge(requestHistoryParent, i.getAndIncrement())) { numHistoryTransferred.getAndAdd(requestHistoryParent.history.size()); } }, requestHistoryParent.requestId, "request history purger"); } LOG.info("Transferred {} history updates for {} requests in {}", numHistoryTransferred, requestHistoryParents.size(), JavaUtils.duration(start)); } finally { persisterLock.unlock(); } }
private DeployHealth getHealthcheckDeployState(final SingularityDeploy deploy, final Collection<SingularityTaskId> matchingActiveTasks, final boolean isDeployPending) { Map<SingularityTaskId, SingularityTaskHealthcheckResult> healthcheckResults = taskManager.getLastHealthcheck(matchingActiveTasks); List<SingularityRequestHistory> requestHistories = requestManager.getRequestHistory(deploy.getRequestId()); for (SingularityTaskId taskId : matchingActiveTasks) { DeployHealth individualTaskHealth; if (healthchecksSkipped(taskId, requestHistories, deploy)) { LOG.trace("Detected skipped healthchecks for {}", taskId); individualTaskHealth = DeployHealth.HEALTHY; } else { individualTaskHealth = getTaskHealth(deploy, isDeployPending, Optional.fromNullable(healthcheckResults.get(taskId)), taskId); } if (individualTaskHealth != DeployHealth.HEALTHY) { return individualTaskHealth; } } return DeployHealth.HEALTHY; }
@Test public void testRequestAgePurging() { initRequest(); configuration.setDeleteStaleRequestsFromZkWhenNoDatabaseAfterHours(7); requestHistoryPersister.runActionOnPoll(); Assert.assertTrue(!requestManager.getRequestHistory(requestId).isEmpty()); requestManager.startDeletingRequest(request, Optional.absent(), user, Optional.<String> absent(), Optional.<String> absent()); requestManager.deleteHistoryParent(requestId); requestManager.activate(request, RequestHistoryType.CREATED, System.currentTimeMillis() - TimeUnit.HOURS.toMillis(3), Optional.<String> absent(), Optional.<String> absent()); requestManager.cooldown(request, System.currentTimeMillis() - TimeUnit.HOURS.toMillis(2)); requestHistoryPersister.runActionOnPoll(); Assert.assertTrue(!requestManager.getRequestHistory(requestId).isEmpty()); configuration.setDeleteStaleRequestsFromZkWhenNoDatabaseAfterHours(1); requestHistoryPersister.runActionOnPoll(); Assert.assertTrue(requestManager.getRequestHistory(requestId).isEmpty()); }
@Test public void itCorrectlyUpdatesRequestDeletingStateHistory() { initRequest(); Assert.assertEquals(RequestState.ACTIVE, requestManager.getRequest(requestId).get().getState()); Assert.assertEquals(1, requestManager.getRequestHistory(requestId).size()); requestManager.startDeletingRequest(request, Optional.absent(), Optional.<String>absent(), Optional.<String>absent(), Optional.of("the cake is a lie")); Assert.assertEquals(RequestState.DELETING, requestManager.getRequest(requestId).get().getState()); Assert.assertEquals(2, requestManager.getRequestHistory(requestId).size()); cleaner.drainCleanupQueue(); Assert.assertEquals(3, requestManager.getRequestHistory(requestId).size()); List<RequestHistoryType> historyTypes = new ArrayList<>(); for (SingularityRequestHistory request : requestManager.getRequestHistory(requestId)) { historyTypes.add(request.getEventType()); } Assert.assertTrue(historyTypes.contains(RequestHistoryType.CREATED)); Assert.assertTrue(historyTypes.contains(RequestHistoryType.DELETING)); Assert.assertTrue(historyTypes.contains(RequestHistoryType.DELETED)); }
@Test public void testPurgingDoesntApplyIfDatabasePresent() { initRequest(); initFirstDeploy(); requestManager.startDeletingRequest(request, Optional.absent(), user, Optional.<String> absent(), Optional.<String> absent()); requestManager.deleteHistoryParent(requestId); requestManager.activate(request, RequestHistoryType.CREATED, System.currentTimeMillis() - TimeUnit.HOURS.toMillis(3), Optional.<String> absent(), Optional.<String> absent()); configuration.setDatabaseConfiguration(new DataSourceFactory()); configuration.setDeleteStaleRequestsFromZkWhenNoDatabaseAfterHours(1); requestHistoryPersister.runActionOnPoll(); Assert.assertTrue(!requestManager.getRequestHistory(requestId).isEmpty()); }
@Test public void testRequestCountPurging() { final SingularityRequest requestOne = new SingularityRequestBuilder("request1", RequestType.WORKER).build(); final SingularityRequest requestTwo = new SingularityRequestBuilder("request2", RequestType.WORKER).build(); final SingularityRequest requestThree = new SingularityRequestBuilder("request3", RequestType.WORKER).build(); saveRequest(requestOne); saveRequest(requestTwo); saveRequest(requestThree); configuration.setMaxRequestsWithHistoryInZkWhenNoDatabase(Optional.of(2)); configuration.setDeleteStaleRequestsFromZkWhenNoDatabaseAfterHours(7); requestManager.startDeletingRequest(requestOne, Optional.absent(), user, Optional.<String>absent(), Optional.<String>absent()); requestManager.deleteHistoryParent(requestOne.getId()); requestManager.activate(requestOne, RequestHistoryType.CREATED, System.currentTimeMillis() - TimeUnit.HOURS.toMillis(4), Optional.<String> absent(), Optional.<String> absent()); requestManager.cooldown(requestOne, System.currentTimeMillis() - TimeUnit.HOURS.toMillis(3)); requestManager.startDeletingRequest(requestTwo, Optional.absent(), user, Optional.<String>absent(), Optional.<String>absent()); requestManager.deleteHistoryParent(requestTwo.getId()); requestManager.activate(requestTwo, RequestHistoryType.CREATED, System.currentTimeMillis() - TimeUnit.HOURS.toMillis(4), Optional.<String> absent(), Optional.<String> absent()); requestManager.cooldown(requestTwo, System.currentTimeMillis() - TimeUnit.HOURS.toMillis(3)); requestManager.startDeletingRequest(requestThree, Optional.absent(), user, Optional.<String>absent(), Optional.<String>absent()); requestManager.deleteHistoryParent(requestThree.getId()); requestManager.activate(requestThree, RequestHistoryType.CREATED, System.currentTimeMillis() - TimeUnit.HOURS.toMillis(4), Optional.<String> absent(), Optional.<String> absent()); requestManager.cooldown(requestThree, System.currentTimeMillis() - TimeUnit.HOURS.toMillis(3)); Assert.assertEquals(2, requestManager.getRequestHistory(requestOne.getId()).size()); Assert.assertEquals(2, requestManager.getRequestHistory(requestTwo.getId()).size()); Assert.assertEquals(2, requestManager.getRequestHistory(requestThree.getId()).size()); requestHistoryPersister.runActionOnPoll(); Assert.assertEquals(0, requestManager.getRequestHistory(requestOne.getId()).size()); Assert.assertEquals(2, requestManager.getRequestHistory(requestTwo.getId()).size()); Assert.assertEquals(2, requestManager.getRequestHistory(requestThree.getId()).size()); }