private TaskCleanupType getCleanupType(SingularityPendingDeploy pendingDeploy, SingularityRequest request, SingularityDeployResult deployResult) { if (pendingDeploy.getDeployProgress().isPresent() && pendingDeploy.getDeployProgress().get().getDeployInstanceCountPerStep() != request.getInstancesSafe()) { // For incremental deploys, return a special cleanup type if (deployResult.getDeployState() == DeployState.FAILED) { return TaskCleanupType.INCREMENTAL_DEPLOY_FAILED; } else if (deployResult.getDeployState() == DeployState.CANCELED) { return TaskCleanupType.INCREMENTAL_DEPLOY_CANCELLED; } } return deployResult.getDeployState().getCleanupType(); }
if (deployResult.getDeployState() == DeployState.SUCCEEDED) { if (saveNewDeployState(pendingDeployMarker, Optional.of(pendingDeployMarker))) { if (!(request.getRequestType() == RequestType.RUN_ONCE)) { LOG.warn("Failing deploy {} because it failed to save deploy state", pendingDeployMarker); deployResult = new SingularityDeployResult(DeployState.FAILED_INTERNAL_STATE, Optional.of(String.format("Deploy had state %s but failed to persist it correctly", deployResult.getDeployState())), deployResult.getLbUpdate(), SingularityDeployFailure.failedToSave(), deployResult.getTimestamp()); } else if (!deployResult.getDeployState().isDeployFinished()) { return;
@Test public void testRemovedRequestData() { long now = System.currentTimeMillis(); initRequest(); SingularityDeployBuilder db = new SingularityDeployBuilder(requestId, firstDeployId); db.setMaxTaskRetries(Optional.of(1)); initDeploy(db, now); deployChecker.checkDeploys(); Assert.assertEquals(DeployState.WAITING, deployManager.getPendingDeploys().get(0).getCurrentDeployState()); requestManager.startDeletingRequest(request, Optional.absent(), Optional.<String>absent(), Optional.<String>absent(), Optional.<String>absent()); requestManager.markDeleted(request, now, Optional.<String>absent(), Optional.<String>absent()); deployChecker.checkDeploys(); SingularityDeployResult deployResult = deployManager.getDeployResult(requestId, firstDeployId).get(); Assert.assertEquals(DeployState.FAILED, deployResult.getDeployState()); Assert.assertTrue(deployResult.getMessage().get().contains("MISSING")); }
private void cleanupTasks(SingularityPendingDeploy pendingDeploy, SingularityRequest request, SingularityDeployResult deployResult, Iterable<SingularityTaskId> tasksToKill) { for (SingularityTaskId matchingTask : tasksToKill) { taskManager.saveTaskCleanup(new SingularityTaskCleanup(pendingDeploy.getDeployMarker().getUser(), getCleanupType(pendingDeploy, request, deployResult), deployResult.getTimestamp(), matchingTask, Optional.of(String.format("Deploy %s - %s", pendingDeploy.getDeployMarker().getDeployId(), deployResult.getDeployState().name())), Optional.<String> absent(), Optional.<SingularityTaskShellCommandRequestId>absent())); } }
@Override public SingularityDeployHistory map(int index, ResultSet r, StatementContext ctx) throws SQLException { SingularityDeployMarker marker = new SingularityDeployMarker(r.getString("requestId"), r.getString("deployId"), r.getTimestamp("createdAt").getTime(), Optional.fromNullable(r.getString(userColumn)), Optional.fromNullable(r.getString("message"))); SingularityDeployResult deployState = new SingularityDeployResult(DeployState.valueOf(r.getString("deployState")), Optional.<String>absent(), Optional.<SingularityLoadBalancerUpdate>absent(), Collections.<SingularityDeployFailure>emptyList(), r.getTimestamp("deployStateAt") .getTime()); return new SingularityDeployHistory(Optional.of(deployState), marker, Optional.<SingularityDeploy>absent(), Optional.<SingularityDeployStatistics>absent()); } }
@Override public void saveDeployHistory(SingularityDeployHistory deployHistory) { if (LOG.isTraceEnabled()) { LOG.trace("saveDeployHistory {}", deployHistory); } history.insertDeployHistory(deployHistory.getDeployMarker().getRequestId(), deployHistory.getDeployMarker().getDeployId(), new Date(deployHistory.getDeployMarker().getTimestamp()), getUserField(deployHistory.getDeployMarker().getUser()), getMessageField(deployHistory.getDeployMarker().getMessage()), deployHistory.getDeployResult().isPresent() ? new Date(deployHistory.getDeployResult().get().getTimestamp()) : new Date(deployHistory.getDeployMarker().getTimestamp()), deployHistory.getDeployResult().isPresent() ? deployHistory.getDeployResult().get().getDeployState().name() : DeployState.CANCELED.name(), deployHistoryTranscoder.toBytes(deployHistory)); }
private SingularityDeployResult getDeployResultWithFailures(SingularityRequest request, Optional<SingularityDeploy> deploy, SingularityPendingDeploy pendingDeploy, DeployState state, String message, Collection<SingularityTaskId> matchingTasks) { List<SingularityDeployFailure> deployFailures = getDeployFailures(request, deploy, pendingDeploy, state, matchingTasks); if (deployFailures.size() == 1 && !deployFailures.get(0).getTaskId().isPresent()) { // Single non-task-specific failure should become the deploy result message (e.g. not enough resources to launch all tasks) return new SingularityDeployResult(state, deployFailures.get(0).getMessage() , pendingDeploy.getLastLoadBalancerUpdate(), Collections.<SingularityDeployFailure>emptyList(), System.currentTimeMillis()); } else { return new SingularityDeployResult(state, Optional.of(message), pendingDeploy.getLastLoadBalancerUpdate(), deployFailures, System.currentTimeMillis()); } }
@Test public void testDeployTimesOut() { initRequest(); final long hourAgo = System.currentTimeMillis() - TimeUnit.HOURS.toMillis(1); final String deployId = "timeout_test"; SingularityDeployBuilder db = new SingularityDeployBuilder(requestId, deployId); db.setDeployHealthTimeoutSeconds(Optional.of(TimeUnit.MINUTES.toSeconds(1))); initDeploy(db, hourAgo); deployChecker.checkDeploys(); Assert.assertEquals(DeployState.OVERDUE, deployManager.getDeployResult(requestId, deployId).get().getDeployState()); }
if (deployResult.getDeployState() == DeployState.SUCCEEDED) { if (saveNewDeployState(pendingDeployMarker, Optional.of(pendingDeployMarker))) { if (!(request.getRequestType() == RequestType.RUN_ONCE)) { LOG.warn("Failing deploy {} because it failed to save deploy state", pendingDeployMarker); deployResult = new SingularityDeployResult(DeployState.FAILED_INTERNAL_STATE, Optional.of(String.format("Deploy had state %s but failed to persist it correctly", deployResult.getDeployState())), deployResult.getLbUpdate(), SingularityDeployFailure.failedToSave(), deployResult.getTimestamp()); } else if (!deployResult.getDeployState().isDeployFinished()) { return;
.setRequestId(requestId) .setDeployId(deployId) .setTimestamp(deployResult.getTimestamp()) .setUser(pendingDeploy.getDeployMarker().getUser()) .setCmdLineArgsList(runNowRequest.getCommandLineArgs()) builder.setPendingType(canceledOr(deployResult.getDeployState(), pendingType)); requestManager.addToPendingQueue(builder.build()); } else { PendingType pendingType = canceledOr(deployResult.getDeployState(), PendingType.NEW_DEPLOY); requestManager.addToPendingQueue(new SingularityPendingRequest(request.getId(), pendingDeploy.getDeployMarker().getDeployId(), deployResult.getTimestamp(), pendingDeploy.getDeployMarker().getUser(), pendingType, deploy.isPresent() ? deploy.get().getSkipHealthchecksOnDeploy() : Optional.absent(), pendingDeploy.getDeployMarker().getMessage())); if (deployResult.getDeployState() == DeployState.SUCCEEDED) { if (request.isDeployable() && !request.isOneOff()) { if (request.isDeployable() && (deployResult.getDeployState() == DeployState.CANCELED || deployResult.getDeployState() == DeployState.FAILED || deployResult.getDeployState() == DeployState.OVERDUE)) { Optional<SingularityRequestDeployState> maybeRequestDeployState = deployManager.getRequestDeployState(request.getId()); if (maybeRequestDeployState.isPresent() request.getId(), maybeRequestDeployState.get().getActiveDeploy().get().getDeployId(), deployResult.getTimestamp(), pendingDeploy.getDeployMarker().getUser(), deployResult.getDeployState() == DeployState.CANCELED ? PendingType.DEPLOY_CANCELLED : PendingType.DEPLOY_FAILED, request.getSkipHealthchecks(), pendingDeploy.getDeployMarker().getMessage()));
private SingularityDeployResult checkOverdue(SingularityRequest request, Optional<SingularityDeploy> deploy, SingularityPendingDeploy pendingDeploy, Collection<SingularityTaskId> deployActiveTasks, boolean isOverdue) { String message = null; if (deploy.isPresent()) { message = String.format("Deploy was able to launch %s tasks, but not all of them became healthy within %s", deployActiveTasks.size(), JavaUtils.durationFromMillis(getAllowedMillis(deploy.get()))); } if (deploy.isPresent() && isOverdue) { return getDeployResultWithFailures(request, deploy, pendingDeploy, DeployState.OVERDUE, message, deployActiveTasks); } else { return new SingularityDeployResult(DeployState.WAITING); } }
@Test public void testRemovedRequestData() { long now = System.currentTimeMillis(); initRequest(); SingularityDeployBuilder db = new SingularityDeployBuilder(requestId, firstDeployId); db.setMaxTaskRetries(Optional.of(1)); initDeploy(db, now); deployChecker.checkDeploys(); Assert.assertEquals(DeployState.WAITING, deployManager.getPendingDeploys().get(0).getCurrentDeployState()); requestManager.startDeletingRequest(request, Optional.absent(), Optional.<String>absent(), Optional.<String>absent(), Optional.<String>absent()); requestManager.markDeleted(request, now, Optional.<String>absent(), Optional.<String>absent()); deployChecker.checkDeploys(); SingularityDeployResult deployResult = deployManager.getDeployResult(requestId, firstDeployId).get(); Assert.assertEquals(DeployState.FAILED, deployResult.getDeployState()); Assert.assertTrue(deployResult.getMessage().get().contains("MISSING")); }
@Test public void testDeployFailsForInvalidRequestState() { initRequest(); SingularityRequest request = requestResource.getRequest(requestId, singularityUser).getRequest(); initFirstDeploy(); deploy(secondDeployId, Optional.<Boolean>absent(), Optional.of(1), Optional.of(false), false); requestManager.pause(request, System.currentTimeMillis(), Optional.<String>absent(), Optional.<String>absent()); deployChecker.checkDeploys(); Assert.assertEquals(DeployState.FAILED, deployManager.getDeployResult(requestId, secondDeployId).get().getDeployState()); }
private void cleanupTasks(SingularityPendingDeploy pendingDeploy, SingularityRequest request, SingularityDeployResult deployResult, Iterable<SingularityTaskId> tasksToKill) { for (SingularityTaskId matchingTask : tasksToKill) { taskManager.saveTaskCleanup(new SingularityTaskCleanup(pendingDeploy.getDeployMarker().getUser(), getCleanupType(pendingDeploy, request, deployResult), deployResult.getTimestamp(), matchingTask, Optional.of(String.format("Deploy %s - %s", pendingDeploy.getDeployMarker().getDeployId(), deployResult.getDeployState().name())), Optional.<String> absent(), Optional.<SingularityTaskShellCommandRequestId>absent())); } }
private void failPendingDeployDueToState(SingularityPendingDeploy pendingDeploy, Optional<SingularityRequestWithState> maybeRequestWithState, Optional<SingularityDeploy> deploy) { SingularityDeployResult deployResult = new SingularityDeployResult(DeployState.FAILED, Optional.of(String.format("Request in state %s is not deployable", SingularityRequestWithState.getRequestState(maybeRequestWithState))), Optional.<SingularityLoadBalancerUpdate>absent()); if (!maybeRequestWithState.isPresent()) { deployManager.saveDeployResult(pendingDeploy.getDeployMarker(), deploy, deployResult); removePendingDeploy(pendingDeploy); return; } saveNewDeployState(pendingDeploy.getDeployMarker(), Optional.<SingularityDeployMarker> absent()); finishDeploy(maybeRequestWithState.get(), deploy, pendingDeploy, Collections.<SingularityTaskId>emptyList(), deployResult); }
@Test public void testMaxHealthcheckRetries() { initRequest(); final String deployId = "retry_test"; HealthcheckOptions options = new HealthcheckOptionsBuilder("http://uri").setMaxRetries(Optional.of(2)).build(); SingularityDeployBuilder db = new SingularityDeployBuilder(requestId, deployId).setHealthcheck(Optional.of(options)); SingularityDeploy deploy = initDeploy(db, System.currentTimeMillis()); deployChecker.checkDeploys(); Assert.assertTrue(!deployManager.getDeployResult(requestId, deployId).isPresent()); SingularityTask task = launchTask(request, deploy, System.currentTimeMillis(), 1, TaskState.TASK_RUNNING); deployChecker.checkDeploys(); Assert.assertTrue(!deployManager.getDeployResult(requestId, deployId).isPresent()); taskManager.saveHealthcheckResult(new SingularityTaskHealthcheckResult(Optional.of(503), Optional.of(1000L), System.currentTimeMillis(), Optional.<String> absent(), Optional.<String> absent(), task.getTaskId(), Optional.<Boolean>absent())); taskManager.saveHealthcheckResult(new SingularityTaskHealthcheckResult(Optional.of(503), Optional.of(1000L), System.currentTimeMillis() + 1, Optional.<String> absent(), Optional.<String> absent(), task.getTaskId(), Optional.<Boolean>absent())); deployChecker.checkDeploys(); Assert.assertTrue(!deployManager.getDeployResult(requestId, deployId).isPresent()); taskManager.saveHealthcheckResult(new SingularityTaskHealthcheckResult(Optional.of(503), Optional.of(1000L), System.currentTimeMillis() + 1, Optional.<String> absent(), Optional.<String> absent(), task.getTaskId(), Optional.<Boolean>absent())); deployChecker.checkDeploys(); Assert.assertEquals(DeployState.FAILED, deployManager.getDeployResult(requestId, deployId).get().getDeployState()); }
@Override public void saveDeployHistory(SingularityDeployHistory deployHistory) { if (LOG.isTraceEnabled()) { LOG.trace("saveDeployHistory {}", deployHistory); } history.insertDeployHistory(deployHistory.getDeployMarker().getRequestId(), deployHistory.getDeployMarker().getDeployId(), new Date(deployHistory.getDeployMarker().getTimestamp()), getUserField(deployHistory.getDeployMarker().getUser()), getMessageField(deployHistory.getDeployMarker().getMessage()), deployHistory.getDeployResult().isPresent() ? new Date(deployHistory.getDeployResult().get().getTimestamp()) : new Date(deployHistory.getDeployMarker().getTimestamp()), deployHistory.getDeployResult().isPresent() ? deployHistory.getDeployResult().get().getDeployState().name() : DeployState.CANCELED.name(), deployHistoryTranscoder.toBytes(deployHistory)); }
private SingularityDeployResult cancelLoadBalancer(SingularityPendingDeploy pendingDeploy, List<SingularityDeployFailure> deployFailures) { final SingularityLoadBalancerUpdate lbUpdate = sendCancelToLoadBalancer(pendingDeploy); final DeployState deployState = interpretLoadBalancerState(lbUpdate, DeployState.CANCELING); updatePendingDeploy(pendingDeploy, Optional.of(lbUpdate), deployState); return new SingularityDeployResult(deployState, lbUpdate, deployFailures); }
final Optional<SingularityDeployHistory> deployHistory = singularityClient.getHistoryForRequestDeploy(REQUEST_ID, deployId); if (deployHistory.isPresent() && deployHistory.get().getDeployResult().isPresent()) { deployState = Optional.fromNullable(deployHistory.get().getDeployResult().get().getDeployState());
.setRequestId(requestId) .setDeployId(deployId) .setTimestamp(deployResult.getTimestamp()) .setUser(pendingDeploy.getDeployMarker().getUser()) .setCmdLineArgsList(runNowRequest.getCommandLineArgs()) builder.setPendingType(canceledOr(deployResult.getDeployState(), pendingType)); requestManager.addToPendingQueue(builder.build()); } else { PendingType pendingType = canceledOr(deployResult.getDeployState(), PendingType.NEW_DEPLOY); requestManager.addToPendingQueue(new SingularityPendingRequest(request.getId(), pendingDeploy.getDeployMarker().getDeployId(), deployResult.getTimestamp(), pendingDeploy.getDeployMarker().getUser(), pendingType, deploy.isPresent() ? deploy.get().getSkipHealthchecksOnDeploy() : Optional.absent(), pendingDeploy.getDeployMarker().getMessage())); if (deployResult.getDeployState() == DeployState.SUCCEEDED) { if (request.isDeployable() && !request.isOneOff()) { if (request.isDeployable() && (deployResult.getDeployState() == DeployState.CANCELED || deployResult.getDeployState() == DeployState.FAILED || deployResult.getDeployState() == DeployState.OVERDUE)) { Optional<SingularityRequestDeployState> maybeRequestDeployState = deployManager.getRequestDeployState(request.getId()); if (maybeRequestDeployState.isPresent() request.getId(), maybeRequestDeployState.get().getActiveDeploy().get().getDeployId(), deployResult.getTimestamp(), pendingDeploy.getDeployMarker().getUser(), deployResult.getDeployState() == DeployState.CANCELED ? PendingType.DEPLOY_CANCELLED : PendingType.DEPLOY_FAILED, request.getSkipHealthchecks(), pendingDeploy.getDeployMarker().getMessage()));