private boolean ownedByDeploymentGroup(final Deployment deployment, final DeploymentGroup deploymentGroup) { return Objects.equals(deployment.getDeploymentGroupName(), deploymentGroup.getName()); }
public List<String> getMatchingHosts(final DeploymentGroup deploymentGroup) { final List<HostSelector> selectors = deploymentGroup.getHostSelectors(); if (selectors == null || selectors.isEmpty()) { log.error("skipping deployment group with no host selectors: " + deploymentGroup.getName()); return emptyList(); } return getMatchingHosts(selectors); }
@POST @Produces(APPLICATION_JSON) @Timed @ExceptionMetered public Response createDeploymentGroup(@Valid final DeploymentGroup deploymentGroup) { try { model.addDeploymentGroup(deploymentGroup); return Response.ok(CREATED_RESPONSE).build(); } catch (DeploymentGroupExistsException ignored) { final DeploymentGroup existing; try { existing = model.getDeploymentGroup(deploymentGroup.getName()); } catch (DeploymentGroupDoesNotExistException e) { // Edge condition: There's a race where someone can potentially remove the deployment-group // while this operation is in progress. This should be very rare. If it does happen, // return 500. return Response.status(Response.Status.INTERNAL_SERVER_ERROR).build(); } if (!Objects.equals(existing.getHostSelectors(), deploymentGroup.getHostSelectors())) { return Response.ok(DEPLOYMENT_GROUP_ALREADY_EXISTS_RESPONSE).build(); } return Response.ok(NOT_MODIFIED_RESPONSE).build(); } }
@Override public void run(final boolean timeout) throws InterruptedException { final List<String> allHosts = masterModel.listHosts(); final Map<String, Map<String, String>> hostsToLabels = Maps.newHashMap(); // determine all hosts and their labels for (final String host : allHosts) { hostsToLabels.put(host, masterModel.getHostLabels(host)); } final HostMatcher hostMatcher = new HostMatcher(hostsToLabels); for (final DeploymentGroup dg : masterModel.getDeploymentGroups().values()) { final List<String> matchingHosts = hostMatcher.getMatchingHosts(dg); try { masterModel.updateDeploymentGroupHosts(dg.getName(), matchingHosts); } catch (Exception e) { log.warn("error processing hosts update for deployment group: {} - {}", dg.getName(), e); } } } }
"would have set state=FAILED for deploymentGroup={} but ignoreFailures is set to true " + "for this group/rollout. errorCode={} message={}", deploymentGroup.getName(), errorCode, errMsg); operations.add(delete(Paths.statusDeploymentGroupTasks(deploymentGroup.getName()))); operations.add(set(Paths.statusDeploymentGroup(deploymentGroup.getName()), status)); events.add(eventFactory.rollingUpdateFailed(deploymentGroup, taskEv)); log.info("rolling-update step on deployment-group name={} failed; message={}", deploymentGroup.getName(), errMsg);
deploymentGroup.getName(), updateHosts, undeployHosts, rolloutTasks);
deploymentGroup.getName(), jobId, rolloutOptionsWithFallback); final ZooKeeperClient client = provider.get("rollingUpdate"); operations.add(set(Paths.configDeploymentGroup(updated.getName()), updated)); deploymentGroup.getName(), jobId, operations); deploymentGroup.getName(), jobId); } catch (final NoNodeException e) { throw new DeploymentGroupDoesNotExistException(deploymentGroup.getName()); } catch (final KeeperException e) { throw new HeliosRuntimeException( "rolling-update on deployment-group " + deploymentGroup.getName() + " failed", e);
public RollingUpdateOp nextTask(final List<ZooKeeperOperation> operations) { final List<ZooKeeperOperation> ops = Lists.newArrayList(operations); final List<Map<String, Object>> events = Lists.newArrayList(); final RolloutTask task = tasks.getRolloutTasks().get(tasks.getTaskIndex()); // Update the task index, delete tasks if done if (tasks.getTaskIndex() + 1 == tasks.getRolloutTasks().size()) { final DeploymentGroupStatus status = DeploymentGroupStatus.newBuilder() .setState(DONE) .build(); // We are done -> delete tasks & update status ops.add(delete(Paths.statusDeploymentGroupTasks(deploymentGroup.getName()))); ops.add(set(Paths.statusDeploymentGroup(deploymentGroup.getName()), status)); // Emit an event signalling that we're DONE! events.add(eventFactory.rollingUpdateDone(deploymentGroup)); } else { ops.add( set(Paths.statusDeploymentGroupTasks(deploymentGroup.getName()), tasks.toBuilder() .setTaskIndex(tasks.getTaskIndex() + 1) .build())); // Only emit an event if the task resulted in taking in action. If there are no ZK operations // the task was effectively a no-op. if (!operations.isEmpty()) { events.add(eventFactory.rollingUpdateTaskSucceeded(deploymentGroup, task)); } } return new RollingUpdateOp(ImmutableList.copyOf(ops), ImmutableList.copyOf(events)); }
final byte[] data = client.getData(path); final DeploymentGroup descriptor = parse(data, DeploymentGroup.class); descriptors.put(descriptor.getName(), descriptor); } catch (NoNodeException e) {
out.println(Json.asPrettyStringUnchecked(deploymentGroup)); } else { out.printf("Name: %s%n", deploymentGroup.getName()); out.printf("Host selectors:%n"); for (final HostSelector hostSelector : deploymentGroup.getHostSelectors()) {
client.ensurePath(Paths.statusDeploymentGroups()); client.transaction( create(Paths.configDeploymentGroup(deploymentGroup.getName()), deploymentGroup), create(Paths.statusDeploymentGroup(deploymentGroup.getName())), create(Paths.statusDeploymentGroupHosts(deploymentGroup.getName()), Json.asBytesUnchecked(emptyList())), create(Paths.statusDeploymentGroupRemovedHosts(deploymentGroup.getName()), Json.asBytesUnchecked(emptyList())) ); } catch (final NodeExistsException e) { throw new DeploymentGroupExistsException(deploymentGroup.getName());
Paths.statusDeploymentGroupTasks(deploymentGroup.getName())); if (tasksStat == null) { ops.add(create(Paths.statusDeploymentGroupTasks(deploymentGroup.getName()))); .setState(DONE) .build(); ops.add(delete(Paths.statusDeploymentGroupTasks(deploymentGroup.getName()))); events.add(eventFactory.rollingUpdateDone(deploymentGroup)); } else { .setState(ROLLING_OUT) .build(); ops.add(set(Paths.statusDeploymentGroupTasks(deploymentGroup.getName()), tasks)); ops.add(set(Paths.statusDeploymentGroup(deploymentGroup.getName()), status));
try { final Node node = client.getNode( Paths.statusDeploymentGroupRemovedHosts(deploymentGroup.getName())); final int version = node.getStat().getVersion(); final List<String> hostsToUndeploy = Json.read(node.getBytes(), STRING_LIST_TYPE); check(Paths.statusDeploymentGroupRemovedHosts(deploymentGroup.getName()), version), set(Paths.statusDeploymentGroupRemovedHosts(deploymentGroup.getName()), Json.asBytes(hostsToUndeploy)))); } catch (KeeperException | IOException e) {
private boolean isRolloutTimedOut(final ZooKeeperClient client, final DeploymentGroup deploymentGroup) { final String groupName = deploymentGroup.getName(); final RolloutOptions defaultOptions = RolloutOptions.getDefault(); final long groupTimeoutSetting = deploymentGroup.getRolloutOptions() == null ? defaultOptions.getTimeout() : deploymentGroup.getRolloutOptions().withFallback(defaultOptions).getTimeout(); final long secondsSinceDeploy; try { final String statusPath = Paths.statusDeploymentGroupTasks(groupName); secondsSinceDeploy = MILLISECONDS.toSeconds( System.currentTimeMillis() - client.getNode(statusPath).getStat().getMtime()); } catch (KeeperException e) { // statusPath doesn't exist or some other ZK issue. probably this deployment group // was removed. log.warn("error determining deployment group modification time: name={}", groupName, e); return false; } if (secondsSinceDeploy > groupTimeoutSetting) { log.info("rolling-update on deployment-group name={} has timed out after " + "{} seconds (rolloutOptions.timeout={})", groupName, secondsSinceDeploy, groupTimeoutSetting); return true; } return false; }
final DeploymentGroup deploymentGroup) { if (!Objects.equals(deployment.getDeploymentGroupName(), deploymentGroup.getName())) { return false;
private RollingUpdateOp processRollingUpdateTask(final ZooKeeperClient client, final RollingUpdateOpFactory opFactory, final RolloutTask task, final DeploymentGroup deploymentGroup) { final RolloutTask.Action action = task.getAction(); final String host = task.getTarget(); switch (action) { case UNDEPLOY_OLD_JOBS: // add undeploy ops for jobs previously deployed by this deployment group return rollingUpdateUndeploy(client, opFactory, deploymentGroup, host); case DEPLOY_NEW_JOB: // add deploy ops for the new job return rollingUpdateDeploy(client, opFactory, deploymentGroup, host); case AWAIT_RUNNING: return rollingUpdateAwaitRunning(client, opFactory, deploymentGroup, host); case FORCE_UNDEPLOY_JOBS: return forceRollingUpdateUndeploy(client, opFactory, deploymentGroup, host); case AWAIT_UNDEPLOYED: return rollingUpdateAwaitUndeployed(client, opFactory, deploymentGroup, host); case MARK_UNDEPLOYED: return rollingUpdateMarkUndeployed(client, opFactory, deploymentGroup, host); default: throw new HeliosRuntimeException(String.format( "unknown rollout task type %s for deployment group %s.", action, deploymentGroup.getName())); } }
private RollingUpdateOp rollingUpdateDeploy(final ZooKeeperClient client, final RollingUpdateOpFactory opFactory, final DeploymentGroup deploymentGroup, final String host) { final Deployment deployment = Deployment.of(deploymentGroup.getJobId(), Goal.START, Deployment.EMTPY_DEPLOYER_USER, this.name, deploymentGroup.getName()); try { final String token = MoreObjects.firstNonNull( deploymentGroup.getRolloutOptions().getToken(), Job.EMPTY_TOKEN); return opFactory.nextTask(getDeployOperations(client, host, deployment, token)); } catch (JobDoesNotExistException e) { return opFactory.error(e, host, RollingUpdateError.JOB_NOT_FOUND); } catch (TokenVerificationException e) { return opFactory.error(e, host, RollingUpdateError.TOKEN_VERIFICATION_ERROR); } catch (HostNotFoundException e) { return opFactory.error(e, host, RollingUpdateError.HOST_NOT_FOUND); } catch (JobPortAllocationConflictException e) { return opFactory.error(e, host, RollingUpdateError.PORT_CONFLICT); } catch (JobAlreadyDeployedException e) { // Nothing to do return opFactory.nextTask(); } }
ops.add(set(Paths.configDeploymentGroup(deploymentGroup.getName()), deploymentGroup));
+ "Perhaps it was manually undeployed?", host, RollingUpdateError.JOB_UNEXPECTEDLY_UNDEPLOYED); } else if (!Objects.equals(deployment.getDeploymentGroupName(), deploymentGroup.getName())) { return opFactory.error( "job was already deployed, either manually or by a different deployment group", host,
@Override public void run(final boolean timeout) throws InterruptedException { for (final DeploymentGroup dg : masterModel.getDeploymentGroups().values()) { try { masterModel.rollingUpdateStep(dg, DefaultRolloutPlanner.of(dg)); } catch (Exception e) { log.warn("error processing rolling update step for deployment group: {} - {}", dg.getName(), e); } } } }