@Override public int compareTo(SingularityPendingTaskId o) { return ComparisonChain.start() .compare(this.getNextRunAt(), o.getNextRunAt()) .compare(this.getRequestId(), o.getRequestId()) .compare(this.getDeployId(), o.getDeployId()) .compare(this.getInstanceNo(), o.getInstanceNo()) .compare(this.getCreatedAt(), o.getCreatedAt()) .compare(this.getPendingType(), o.getPendingType()) .result(); }
private boolean shouldRetryImmediately(SingularityRequest request, SingularityDeployStatistics deployStatistics, Optional<SingularityTask> task) { if (!request.getNumRetriesOnFailure().isPresent()) { return false; } if (task.isPresent() && task.get().getTaskRequest().getPendingTask().getPendingTaskId().getPendingType() == PendingType.IMMEDIATE && request.getRequestType() == RequestType.SCHEDULED) { return false; // don't retry UI triggered scheduled jobs (UI triggered on-demand jobs are okay to retry though) } final int numRetriesInARow = deployStatistics.getNumSequentialRetries(); if (numRetriesInARow >= request.getNumRetriesOnFailure().get()) { LOG.debug("Request {} had {} retries in a row, not retrying again (num retries on failure: {})", request.getId(), numRetriesInARow, request.getNumRetriesOnFailure()); return false; } LOG.debug("Request {} had {} retries in a row - retrying again (num retries on failure: {})", request.getId(), numRetriesInARow, request.getNumRetriesOnFailure()); return true; }
private void createTaskAndDeletePendingTaskPrivate(SingularityTask task) throws Exception { // TODO: Should more of the below be done within a transaction? deletePendingTask(task.getTaskRequest().getPendingTask().getPendingTaskId()); final long now = System.currentTimeMillis(); String msg = String.format("Task launched because of %s", task.getTaskRequest().getPendingTask().getPendingTaskId().getPendingType().name()); if (task.getTaskRequest().getPendingTask().getUser().isPresent()) { msg = String.format("%s by %s", msg, task.getTaskRequest().getPendingTask().getUser().get()); } if (task.getTaskRequest().getPendingTask().getMessage().isPresent()) { msg = String.format("%s (%s)", msg, task.getTaskRequest().getPendingTask().getMessage().get()); } saveTaskHistoryUpdate(new SingularityTaskHistoryUpdate(task.getTaskId(), now, ExtendedTaskState.TASK_LAUNCHED, Optional.of(msg), Optional.<String>absent())); saveLastActiveTaskStatus(new SingularityTaskStatusHolder(task.getTaskId(), Optional.absent(), now, serverId, Optional.of(task.getAgentId().getValue()))); try { final String path = getTaskPath(task.getTaskId()); CuratorTransactionFinal transaction = curator.inTransaction().create().forPath(path, taskTranscoder.toBytes(task)).and(); transaction.create().forPath(getActivePath(task.getTaskId().getId())).and().commit(); leaderCache.putActiveTask(task); taskCache.set(path, task); } catch (KeeperException.NodeExistsException nee) { LOG.error("Task or active path already existed for {}", task.getTaskId()); } }
double numFromSameBounceOnSlave = 0; double numOtherDeploysOnSlave = 0; boolean taskLaunchedFromBounceWithActionId = taskRequest.getPendingTask().getPendingTaskId().getPendingType() == PendingType.BOUNCE && taskRequest.getPendingTask().getActionId().isPresent(); if (maybeTask.isPresent()) { SingularityPendingTask pendingTask = maybeTask.get().getTaskRequest().getPendingTask(); if (pendingTask.getPendingTaskId().getPendingType() == PendingType.BOUNCE) { if (pendingTask.getActionId().isPresent()) { if (pendingTask.getActionId().get().equals(taskRequest.getPendingTask().getActionId().get())) {
@Test public void testImmediateRunReplacesScheduledTask() { initScheduledRequest(); SingularityDeploy deploy = SingularityDeploy.newBuilder(requestId, firstDeployId) .setCommand(Optional.of("sleep 100")) .build(); SingularityDeployRequest singularityDeployRequest = new SingularityDeployRequest(deploy, Optional.absent(), Optional.absent(), Optional.absent()); deployResource.deploy(singularityDeployRequest, singularityUser); scheduler.drainPendingQueue(); SingularityPendingTask task1 = createAndSchedulePendingTask(firstDeployId); Assert.assertEquals(1, taskManager.getPendingTaskIds().size()); Assert.assertEquals(PendingType.NEW_DEPLOY, taskManager.getPendingTaskIds().get(0).getPendingType()); requestManager.addToPendingQueue(new SingularityPendingRequest(requestId, deploy.getId(), System.currentTimeMillis(), Optional.absent(), PendingType.IMMEDIATE, deploy.getSkipHealthchecksOnDeploy(), Optional.absent())); scheduler.drainPendingQueue(); Assert.assertEquals(1, taskManager.getPendingTaskIds().size()); Assert.assertEquals(PendingType.IMMEDIATE, taskManager.getPendingTaskIds().get(0).getPendingType()); }
@Test public void testNewlyDeployedScheduledTasksAreScheduledAfterStartup() { initScheduledRequest(); initFirstDeploy(); SingularityTask runningTask = launchTask(request, firstDeploy, 1, TaskState.TASK_RUNNING); long now = System.currentTimeMillis(); initSecondDeploy(); requestManager.addToPendingQueue(new SingularityPendingRequest(requestId, secondDeployId, now, Optional.absent(), PendingType.STARTUP, Optional.absent(), Optional.absent())); deployChecker.checkDeploys(); resourceOffers(); // There's an instance running, so we shouldn't schedule a pending task yet Assert.assertTrue(taskManager.getPendingTaskIds().isEmpty()); statusUpdate(runningTask, TaskState.TASK_FINISHED); scheduler.drainPendingQueue(); // Now a pending task should be scheduled with the new deploy Assert.assertEquals(1, taskManager.getPendingTaskIds().size()); Assert.assertEquals(PendingType.NEW_DEPLOY, taskManager.getPendingTaskIds().get(0).getPendingType()); Assert.assertEquals(secondDeployId, taskManager.getPendingTaskIds().get(0).getDeployId()); }
@Test public void testSchedulerIsolatesPendingTasksBasedOnDeploy() { initRequest(); initFirstDeploy(); initSecondDeploy(); SingularityPendingTask p1 = pendingTask(requestId, firstDeployId, PendingType.ONEOFF); SingularityPendingTask p2 = pendingTask(requestId, firstDeployId, PendingType.TASK_DONE); SingularityPendingTask p3 = pendingTask(requestId, secondDeployId, PendingType.TASK_DONE); taskManager.savePendingTask(p1); taskManager.savePendingTask(p2); taskManager.savePendingTask(p3); requestManager.addToPendingQueue(new SingularityPendingRequest(requestId, secondDeployId, System.currentTimeMillis(), Optional.<String> absent(), PendingType.NEW_DEPLOY, Optional.<Boolean> absent(), Optional.<String> absent())); scheduler.drainPendingQueue(); // we expect there to be 3 pending tasks : List<SingularityPendingTask> returnedScheduledTasks = taskManager.getPendingTasks(); Assert.assertEquals(3, returnedScheduledTasks.size()); Assert.assertTrue(returnedScheduledTasks.contains(p1)); Assert.assertTrue(returnedScheduledTasks.contains(p2)); Assert.assertTrue(!returnedScheduledTasks.contains(p3)); boolean found = false; for (SingularityPendingTask pendingTask : returnedScheduledTasks) { if (pendingTask.getPendingTaskId().getDeployId().equals(secondDeployId)) { found = true; Assert.assertEquals(PendingType.NEW_DEPLOY, pendingTask.getPendingTaskId().getPendingType()); } } Assert.assertTrue(found); }
@Test public void testScheduledTasksDontGetRescheduled() { initScheduledRequest(); deploy(firstDeployId); deployChecker.checkDeploys(); Assert.assertEquals("NEW_DEPLOY request added", 1, requestManager.getPendingRequests().size()); Assert.assertTrue("No tasks started yet", taskManager.getPendingTaskIds().isEmpty()); startup.checkSchedulerForInconsistentState(); Assert.assertEquals("NEW_DEPLOY request added", 1, requestManager.getPendingRequests().size()); Assert.assertEquals("NEW_DEPLOY is first", PendingType.NEW_DEPLOY, requestManager.getPendingRequests().get(0).getPendingType()); Assert.assertTrue("No tasks started yet", taskManager.getPendingTaskIds().isEmpty()); scheduler.drainPendingQueue(); Assert.assertTrue("Pending queue is cleared", requestManager.getPendingRequests().isEmpty()); List<SingularityPendingTask> pending = taskManager.getPendingTasks(); Assert.assertEquals("One task is started", 1, taskManager.getPendingTaskIds().size()); Assert.assertEquals("First request takes precedence", PendingType.NEW_DEPLOY, taskManager.getPendingTaskIds().get(0).getPendingType()); startup.checkSchedulerForInconsistentState(); scheduler.drainPendingQueue(); Assert.assertTrue(requestManager.getPendingRequests().isEmpty()); Assert.assertTrue(taskManager.getPendingTaskIds().size() == 1); Assert.assertTrue(taskManager.getActiveTaskIds().isEmpty()); Assert.assertTrue(pending.equals(taskManager.getPendingTasks())); taskManager.deletePendingTask(pending.get(0).getPendingTaskId()); startup.checkSchedulerForInconsistentState(); Assert.assertTrue(requestManager.getPendingRequests().size() == 1); Assert.assertTrue(taskManager.getPendingTaskIds().isEmpty()); }
@Test public void testRunNowScheduledJobDoesNotRetry() { initScheduledRequest(); SingularityRequest request = requestResource.getRequest(requestId, singularityUser).getRequest(); SingularityRequest newRequest = request.toBuilder().setNumRetriesOnFailure(Optional.of(2)).build(); requestResource.postRequest(newRequest, singularityUser); initFirstDeploy(); requestResource.scheduleImmediately(singularityUser, requestId, new SingularityRunNowRequestBuilder().build()); resourceOffers(); SingularityTask task = taskManager.getActiveTasks().get(0); statusUpdate(task, TaskState.TASK_FAILED); scheduler.drainPendingQueue(); SingularityDeployStatistics deployStatistics = deployManager.getDeployStatistics(task.getTaskId().getRequestId(), task.getTaskId().getDeployId()).get(); Assert.assertEquals(MesosTaskState.TASK_FAILED, deployStatistics.getLastTaskState().get().toTaskState().get()); Assert.assertEquals(PendingType.TASK_DONE, taskManager.getPendingTaskIds().get(0).getPendingType()); Assert.assertEquals(1, deployStatistics.getNumFailures()); Assert.assertEquals(0, deployStatistics.getNumSequentialRetries()); Assert.assertEquals(Optional.<Long>absent(), deployStatistics.getAverageRuntimeMillis()); }
@Test public void testRunNowOnDemandJobMayRetryOnFailure() { initRequestWithType(RequestType.ON_DEMAND, false); SingularityRequest request = requestResource.getRequest(requestId, singularityUser).getRequest(); SingularityRequest newRequest = request.toBuilder().setNumRetriesOnFailure(Optional.of(2)).build(); requestResource.postRequest(newRequest, singularityUser); initFirstDeploy(); requestResource.scheduleImmediately(singularityUser, requestId, new SingularityRunNowRequestBuilder().build()); resourceOffers(); SingularityTask task = taskManager.getActiveTasks().get(0); statusUpdate(task, TaskState.TASK_FAILED); scheduler.drainPendingQueue(); SingularityDeployStatistics deployStatistics = deployManager.getDeployStatistics(task.getTaskId().getRequestId(), task.getTaskId().getDeployId()).get(); Assert.assertEquals(MesosTaskState.TASK_FAILED, deployStatistics.getLastTaskState().get().toTaskState().get()); Assert.assertEquals(PendingType.RETRY, taskManager.getPendingTaskIds().get(0).getPendingType()); Assert.assertEquals(1, deployStatistics.getNumFailures()); Assert.assertEquals(1, deployStatistics.getNumSequentialRetries()); }
Assert.assertEquals(PendingType.TASK_DONE, pendingTaskId.getPendingType()); Assert.assertEquals(lowPriorityRequest.getId(), pendingTaskId.getRequestId());
private boolean shouldRetryImmediately(SingularityRequest request, SingularityDeployStatistics deployStatistics, Optional<SingularityTask> task) { if (!request.getNumRetriesOnFailure().isPresent()) { return false; } if (task.isPresent() && task.get().getTaskRequest().getPendingTask().getPendingTaskId().getPendingType() == PendingType.IMMEDIATE && request.getRequestType() == RequestType.SCHEDULED) { return false; // don't retry UI triggered scheduled jobs (UI triggered on-demand jobs are okay to retry though) } final int numRetriesInARow = deployStatistics.getNumSequentialRetries(); if (numRetriesInARow >= request.getNumRetriesOnFailure().get()) { LOG.debug("Request {} had {} retries in a row, not retrying again (num retries on failure: {})", request.getId(), numRetriesInARow, request.getNumRetriesOnFailure()); return false; } LOG.debug("Request {} had {} retries in a row - retrying again (num retries on failure: {})", request.getId(), numRetriesInARow, request.getNumRetriesOnFailure()); return true; }
private void createTaskAndDeletePendingTaskPrivate(SingularityTask task) throws Exception { // TODO: Should more of the below be done within a transaction? deletePendingTask(task.getTaskRequest().getPendingTask().getPendingTaskId()); final long now = System.currentTimeMillis(); String msg = String.format("Task launched because of %s", task.getTaskRequest().getPendingTask().getPendingTaskId().getPendingType().name()); if (task.getTaskRequest().getPendingTask().getUser().isPresent()) { msg = String.format("%s by %s", msg, task.getTaskRequest().getPendingTask().getUser().get()); } if (task.getTaskRequest().getPendingTask().getMessage().isPresent()) { msg = String.format("%s (%s)", msg, task.getTaskRequest().getPendingTask().getMessage().get()); } saveTaskHistoryUpdate(new SingularityTaskHistoryUpdate(task.getTaskId(), now, ExtendedTaskState.TASK_LAUNCHED, Optional.of(msg), Optional.<String>absent())); saveLastActiveTaskStatus(new SingularityTaskStatusHolder(task.getTaskId(), Optional.absent(), now, serverId, Optional.of(task.getAgentId().getValue()))); try { final String path = getTaskPath(task.getTaskId()); CuratorTransactionFinal transaction = curator.inTransaction().create().forPath(path, taskTranscoder.toBytes(task)).and(); transaction.create().forPath(getActivePath(task.getTaskId().getId())).and().commit(); leaderCache.putActiveTask(task); taskCache.set(path, task); } catch (KeeperException.NodeExistsException nee) { LOG.error("Task or active path already existed for {}", task.getTaskId()); } }
@Test public void testImmediateRunReplacesScheduledTask() { initScheduledRequest(); SingularityDeploy deploy = SingularityDeploy.newBuilder(requestId, firstDeployId) .setCommand(Optional.of("sleep 100")) .build(); SingularityDeployRequest singularityDeployRequest = new SingularityDeployRequest(deploy, Optional.absent(), Optional.absent(), Optional.absent()); deployResource.deploy(singularityDeployRequest, singularityUser); scheduler.drainPendingQueue(); SingularityPendingTask task1 = createAndSchedulePendingTask(firstDeployId); Assert.assertEquals(1, taskManager.getPendingTaskIds().size()); Assert.assertEquals(PendingType.NEW_DEPLOY, taskManager.getPendingTaskIds().get(0).getPendingType()); requestManager.addToPendingQueue(new SingularityPendingRequest(requestId, deploy.getId(), System.currentTimeMillis(), Optional.absent(), PendingType.IMMEDIATE, deploy.getSkipHealthchecksOnDeploy(), Optional.absent())); scheduler.drainPendingQueue(); Assert.assertEquals(1, taskManager.getPendingTaskIds().size()); Assert.assertEquals(PendingType.IMMEDIATE, taskManager.getPendingTaskIds().get(0).getPendingType()); }
@Test public void testNewlyDeployedScheduledTasksAreScheduledAfterStartup() { initScheduledRequest(); initFirstDeploy(); SingularityTask runningTask = launchTask(request, firstDeploy, 1, TaskState.TASK_RUNNING); long now = System.currentTimeMillis(); initSecondDeploy(); requestManager.addToPendingQueue(new SingularityPendingRequest(requestId, secondDeployId, now, Optional.absent(), PendingType.STARTUP, Optional.absent(), Optional.absent())); deployChecker.checkDeploys(); resourceOffers(); // There's an instance running, so we shouldn't schedule a pending task yet Assert.assertTrue(taskManager.getPendingTaskIds().isEmpty()); statusUpdate(runningTask, TaskState.TASK_FINISHED); scheduler.drainPendingQueue(); // Now a pending task should be scheduled with the new deploy Assert.assertEquals(1, taskManager.getPendingTaskIds().size()); Assert.assertEquals(PendingType.NEW_DEPLOY, taskManager.getPendingTaskIds().get(0).getPendingType()); Assert.assertEquals(secondDeployId, taskManager.getPendingTaskIds().get(0).getDeployId()); }
@Test public void testSchedulerIsolatesPendingTasksBasedOnDeploy() { initRequest(); initFirstDeploy(); initSecondDeploy(); SingularityPendingTask p1 = pendingTask(requestId, firstDeployId, PendingType.ONEOFF); SingularityPendingTask p2 = pendingTask(requestId, firstDeployId, PendingType.TASK_DONE); SingularityPendingTask p3 = pendingTask(requestId, secondDeployId, PendingType.TASK_DONE); taskManager.savePendingTask(p1); taskManager.savePendingTask(p2); taskManager.savePendingTask(p3); requestManager.addToPendingQueue(new SingularityPendingRequest(requestId, secondDeployId, System.currentTimeMillis(), Optional.<String> absent(), PendingType.NEW_DEPLOY, Optional.<Boolean> absent(), Optional.<String> absent())); scheduler.drainPendingQueue(); // we expect there to be 3 pending tasks : List<SingularityPendingTask> returnedScheduledTasks = taskManager.getPendingTasks(); Assert.assertEquals(3, returnedScheduledTasks.size()); Assert.assertTrue(returnedScheduledTasks.contains(p1)); Assert.assertTrue(returnedScheduledTasks.contains(p2)); Assert.assertTrue(!returnedScheduledTasks.contains(p3)); boolean found = false; for (SingularityPendingTask pendingTask : returnedScheduledTasks) { if (pendingTask.getPendingTaskId().getDeployId().equals(secondDeployId)) { found = true; Assert.assertEquals(PendingType.NEW_DEPLOY, pendingTask.getPendingTaskId().getPendingType()); } } Assert.assertTrue(found); }
@Test public void testScheduledTasksDontGetRescheduled() { initScheduledRequest(); deploy(firstDeployId); deployChecker.checkDeploys(); Assert.assertEquals("NEW_DEPLOY request added", 1, requestManager.getPendingRequests().size()); Assert.assertTrue("No tasks started yet", taskManager.getPendingTaskIds().isEmpty()); startup.checkSchedulerForInconsistentState(); Assert.assertEquals("NEW_DEPLOY request added", 1, requestManager.getPendingRequests().size()); Assert.assertEquals("NEW_DEPLOY is first", PendingType.NEW_DEPLOY, requestManager.getPendingRequests().get(0).getPendingType()); Assert.assertTrue("No tasks started yet", taskManager.getPendingTaskIds().isEmpty()); scheduler.drainPendingQueue(); Assert.assertTrue("Pending queue is cleared", requestManager.getPendingRequests().isEmpty()); List<SingularityPendingTask> pending = taskManager.getPendingTasks(); Assert.assertEquals("One task is started", 1, taskManager.getPendingTaskIds().size()); Assert.assertEquals("First request takes precedence", PendingType.NEW_DEPLOY, taskManager.getPendingTaskIds().get(0).getPendingType()); startup.checkSchedulerForInconsistentState(); scheduler.drainPendingQueue(); Assert.assertTrue(requestManager.getPendingRequests().isEmpty()); Assert.assertTrue(taskManager.getPendingTaskIds().size() == 1); Assert.assertTrue(taskManager.getActiveTaskIds().isEmpty()); Assert.assertTrue(pending.equals(taskManager.getPendingTasks())); taskManager.deletePendingTask(pending.get(0).getPendingTaskId()); startup.checkSchedulerForInconsistentState(); Assert.assertTrue(requestManager.getPendingRequests().size() == 1); Assert.assertTrue(taskManager.getPendingTaskIds().isEmpty()); }
@Test public void testRunNowScheduledJobDoesNotRetry() { initScheduledRequest(); SingularityRequest request = requestResource.getRequest(requestId, singularityUser).getRequest(); SingularityRequest newRequest = request.toBuilder().setNumRetriesOnFailure(Optional.of(2)).build(); requestResource.postRequest(newRequest, singularityUser); initFirstDeploy(); requestResource.scheduleImmediately(singularityUser, requestId, new SingularityRunNowRequestBuilder().build()); resourceOffers(); SingularityTask task = taskManager.getActiveTasks().get(0); statusUpdate(task, TaskState.TASK_FAILED); scheduler.drainPendingQueue(); SingularityDeployStatistics deployStatistics = deployManager.getDeployStatistics(task.getTaskId().getRequestId(), task.getTaskId().getDeployId()).get(); Assert.assertEquals(MesosTaskState.TASK_FAILED, deployStatistics.getLastTaskState().get().toTaskState().get()); Assert.assertEquals(PendingType.TASK_DONE, taskManager.getPendingTaskIds().get(0).getPendingType()); Assert.assertEquals(1, deployStatistics.getNumFailures()); Assert.assertEquals(0, deployStatistics.getNumSequentialRetries()); Assert.assertEquals(Optional.<Long>absent(), deployStatistics.getAverageRuntimeMillis()); }
@Test public void testRunNowOnDemandJobMayRetryOnFailure() { initRequestWithType(RequestType.ON_DEMAND, false); SingularityRequest request = requestResource.getRequest(requestId, singularityUser).getRequest(); SingularityRequest newRequest = request.toBuilder().setNumRetriesOnFailure(Optional.of(2)).build(); requestResource.postRequest(newRequest, singularityUser); initFirstDeploy(); requestResource.scheduleImmediately(singularityUser, requestId, new SingularityRunNowRequestBuilder().build()); resourceOffers(); SingularityTask task = taskManager.getActiveTasks().get(0); statusUpdate(task, TaskState.TASK_FAILED); scheduler.drainPendingQueue(); SingularityDeployStatistics deployStatistics = deployManager.getDeployStatistics(task.getTaskId().getRequestId(), task.getTaskId().getDeployId()).get(); Assert.assertEquals(MesosTaskState.TASK_FAILED, deployStatistics.getLastTaskState().get().toTaskState().get()); Assert.assertEquals(PendingType.RETRY, taskManager.getPendingTaskIds().get(0).getPendingType()); Assert.assertEquals(1, deployStatistics.getNumFailures()); Assert.assertEquals(1, deployStatistics.getNumSequentialRetries()); }
Assert.assertEquals(PendingType.TASK_DONE, pendingTaskId.getPendingType()); Assert.assertEquals(lowPriorityRequest.getId(), pendingTaskId.getRequestId());