@Nonnull @Override public TaskResult execute(@Nonnull Stage stage) { Map<String, Object> context = stage.getContext(); Map judge1Result = (Map)context.get("judge1Result"); Map judge2Result = (Map)context.get("judge2Result"); // TODO: Now that the plumbing works, perform some kind of actual comparison. Map<String, Map> comparisonResult = ImmutableMap.<String, Map>builder() .put("judge1Result", judge1Result) .put("judge2Result", judge2Result) .build(); Map<String, Map> outputs = Collections.singletonMap("comparisonResult", comparisonResult); return new TaskResult(ExecutionStatus.SUCCEEDED, Collections.emptyMap(), outputs); } }
@Override public Result cancel(Stage stage) { Map<String, Object> context = stage.getContext(); String canaryPipelineExecutionId = (String) context.getOrDefault("canaryPipelineExecutionId", null); stage.getId(), stage.getExecution().getId(), canaryPipelineExecutionId, stage.getContext()); } catch (Exception e) { log.error("Failed to cancel stage (stageId: {}, executionId: {}), e: {}", stage.getId(), stage.getExecution().getId(), e.getMessage(), e); stage.getId(), stage.getExecution().getId(), stage.getContext());
.stageStatus(pipeline.getStages() .stream() .map(stage -> new StageMetadata(stage.getType(), stage.getName(), stage.getStatus())) .collect(Collectors.toList())) .complete(isComplete) .filter(stage -> stage.getType().equals(SetupAndExecuteCanariesStage.STAGE_TYPE)) .findFirst() .ifPresent(stage -> Optional .ofNullable(stage.getContext().getOrDefault(CANARY_ANALYSIS_CONFIG_CONTEXT_KEY, null)) .ifPresent(data -> { CanaryAnalysisConfig canaryAnalysisConfig = kayentaObjectMapper.convertValue(data, CanaryAnalysisConfig.class); .filter(stage -> stage.getType().equals(GenerateCanaryAnalysisResultStage.STAGE_TYPE)) .findFirst() .ifPresent(generateCanaryAnalysisResultStage -> Optional .ofNullable(generateCanaryAnalysisResultStage.getOutputs() .getOrDefault(CANARY_ANALYSIS_EXECUTION_RESULT, null)) .ifPresent(data -> responseBuilder.canaryAnalysisExecutionResult(kayentaObjectMapper.convertValue(data, .filter(stage -> stage.getContext().containsKey("exception")) .findFirst() .ifPresent(stage -> responseBuilder.exception(stage.getContext().get("exception")));
/** * Always run the GenerateCanaryAnalysisResultStage. */ private void addAlwaysRunResultStage(@Nonnull Stage parent, @Nonnull StageGraphBuilder graph) { graph.append(stage -> { stage.setType(GenerateCanaryAnalysisResultStage.STAGE_TYPE); stage.setName(GenerateCanaryAnalysisResultStage.STAGE_DESCRIPTION); stage.setContext(parent.getContext()); }); }
/** * Gets the run canary stages that contain the results */ @NotNull protected List<Stage> getRunCanaryStages(@Nonnull Stage stage) { // Collect the Run Canary Stages where the parent id is itself // Sorting by number after the # in the name return stage.getExecution().getStages().stream() .filter(s -> s.getType().equals(RunCanaryStage.STAGE_TYPE)) .sorted(Comparator.comparing(s -> Integer.valueOf(StringUtils.substringAfterLast(s.getName(), "#")))) .collect(Collectors.toList()); }
public String getCanaryExecutionRequestFromJudgeContext(Execution pipeline) { Stage contextStage = pipeline.getStages().stream() .filter(stage -> stage.getRefId().equals(CanaryStageNames.REFID_JUDGE)) .findFirst() .orElseThrow(() -> new IllegalArgumentException("Unable to find stage '" + CanaryStageNames.REFID_JUDGE + "' in pipeline ID '" + pipeline.getId() + "'")); Map<String, Object> context = contextStage.getContext(); return (String) context.get("canaryExecutionRequest"); }
protected Map<String, String> serializeStage(Stage stage) { String prefix = format("stage.%s.", stage.getId()); Map<String, String> map = new HashMap<>(); map.put(prefix + "refId", stage.getRefId()); map.put(prefix + "type", stage.getType()); map.put(prefix + "name", stage.getName()); map.put(prefix + "startTime", stage.getStartTime() != null ? stage.getStartTime().toString() : null); map.put(prefix + "endTime", stage.getEndTime() != null ? stage.getEndTime().toString() : null); map.put(prefix + "startTimeExpiry", stage.getStartTimeExpiry() != null ? String.valueOf(stage.getStartTimeExpiry()) : null); map.put(prefix + "status", stage.getStatus().name()); map.put(prefix + "syntheticStageOwner", stage.getSyntheticStageOwner() != null ? stage.getSyntheticStageOwner().name() : null); map.put(prefix + "parentStageId", stage.getParentStageId()); if (!stage.getRequisiteStageRefIds().isEmpty()) { map.put(prefix + "requisiteStageRefIds", stage.getRequisiteStageRefIds().stream() .collect(Collectors.joining(",")) ); map.put(prefix + "scheduledTime", (stage.getScheduledTime() != null ? stage.getScheduledTime().toString() : null)); try { map.put(prefix + "context", mapper.writeValueAsString(stage.getContext())); map.put(prefix + "outputs", mapper.writeValueAsString(stage.getOutputs())); map.put(prefix + "tasks", mapper.writeValueAsString(stage.getTasks())); map.put(prefix + "lastModified", (stage.getLastModified() != null ? mapper.writeValueAsString(stage.getLastModified()) : null)); } catch (JsonProcessingException e) { throw new StageSerializationException( format("Failed converting stage to json, executionId: %s, stageId: %s", stage.getExecution().getId(), stage.getId()),
@Nonnull @Override public TaskResult execute(@Nonnull Stage stage) { Map<String, Object> context = stage.getContext(); String storageAccountName = (String)context.get("storageAccountName"); List<String> controlMetricSetListIds = getMetricSetListIds(stage.getExecution(), (String)context.get("controlRefidPrefix")); List<String> experimentMetricSetListIds = getMetricSetListIds(stage.getExecution(), (String)context.get("experimentRefidPrefix")); String resolvedAccountName = CredentialsHelper.resolveAccountByNameOrType(storageAccountName, AccountCredentials.Type.OBJECT_STORE,
public StageComplete( @Nonnull Object source, @Nonnull Stage stage ) { this(source, stage.getExecution().getType(), stage.getExecution().getId(), stage.getId(), stage.getType(), stage.getName(), stage.getStatus()); }
String canaryExecutionId = pipeline.getId(); Stage compareJudgeResultsStage = pipeline.getStages().stream() .filter(stage -> stage.getRefId().equals("compareJudgeResults")) .findFirst() .orElseThrow(() -> new IllegalArgumentException("Unable to find stage 'compareJudgeResults' in pipeline ID '" + canaryExecutionId + "'")); Map<String, Object> compareJudgeResultsOutputs = compareJudgeResultsStage.getOutputs(); Boolean isComplete = pipeline.getStatus().isComplete(); String pipelineStatus = pipeline.getStatus().toString().toLowerCase(); .filter(stage -> stage.getContext().containsKey("exception")) .collect(Collectors.toList()); .stream() .collect( Collectors.toMap(s -> s.getName(), s -> s.getContext().get("exception")) ));
.filter(stage -> stage.getRefId().equals(CanaryStageNames.REFID_JUDGE)) .findFirst() .orElseThrow(() -> new IllegalArgumentException("Unable to find stage '" + CanaryStageNames.REFID_JUDGE + "' in pipeline ID '" + canaryExecutionId + "'")); Map<String, Object> judgeOutputs = judgeStage.getOutputs(); .filter(stage -> stage.getRefId().equals(CanaryStageNames.REFID_SET_CONTEXT)) .findFirst() .orElseThrow(() -> new IllegalArgumentException("Unable to find stage '" + CanaryStageNames.REFID_SET_CONTEXT + "' in pipeline ID '" + canaryExecutionId + "'")); Map<String, Object> contextContext = contextStage.getContext(); .filter(stage -> stage.getRefId().equals(CanaryStageNames.REFID_MIX_METRICS)) .findFirst() .orElseThrow(() -> new IllegalArgumentException("Unable to find stage '" + CanaryStageNames.REFID_MIX_METRICS + "' in pipeline ID '" + canaryExecutionId + "'")); Map<String, Object> mixerContext = mixerStage.getContext(); Map<String, Object> mixerOutputs = mixerStage.getOutputs(); .collect(Collectors.toMap(Stage::getRefId, s -> s.getStatus().toString().toLowerCase())); .filter(stage -> stage.getContext().containsKey("exception")) .findFirst() .orElse(null); canaryExecutionStatusResponseBuilder.exception(stageWithException.getContext().get("exception"));
/** * A Stage is optional if it has an {@link OptionalStageEvaluator} in its * context that evaluates {@code false}. */ public static boolean isOptional(Stage stage, ContextParameterProcessor contextParameterProcessor) { Map stageEnabled = (Map) stage.getContext().get("stageEnabled"); String type = stageEnabled == null ? null : (String) stageEnabled.get("type"); String optionalType = type == null ? null : type.toLowerCase(); if (!OPTIONAL_STAGE_TYPES.containsKey(optionalType)) { if (stage.getSyntheticStageOwner() != null || stage.getParentStageId() != null) { Stage parentStage = stage .getExecution() .getStages() .stream() .filter(it -> it.getId().equals(stage.getParentStageId())) .findFirst() .orElseThrow(() -> new IllegalStateException(format("stage %s not found", stage.getParentStageId()))); return isOptional(parentStage, contextParameterProcessor); } return false; } try { return !stage.mapTo("/stageEnabled", OPTIONAL_STAGE_TYPES.get(optionalType)).evaluate(stage, contextParameterProcessor); } catch (InvalidExpression e) { log.warn("Unable to determine stage optionality, reason: {} (executionId: {}, stageId: {})", e.getMessage(), stage.getExecution().getId(), stage.getId()); return false; } }
static public Moniker monikerFromStage(Stage stage) { if (stage.getContext().containsKey("moniker")) { return (Moniker) stage.mapTo("/moniker", Moniker.class); } else { return null; } }
private static Predicate<Stage> matchesDeployedStage(String ...id) { List<String> idsOrNames = Arrays.asList(id); if (!idsOrNames.isEmpty()){ return stage -> DEPLOY_STAGE_NAMES.contains(stage.getType()) && stage.getContext().containsKey("deploy.server.groups") && stage.getStatus() == ExecutionStatus.SUCCEEDED && (idsOrNames.contains(stage.getName()) || idsOrNames.contains(stage.getId())); } else { return stage -> DEPLOY_STAGE_NAMES.contains(stage.getType()) && stage.getContext().containsKey("deploy.server.groups") && stage.getStatus() == ExecutionStatus.SUCCEEDED; } } }
@Override public void afterStages( @Nonnull Stage parent, @Nonnull StageGraphBuilder graph) { StageData stageData = parent.mapTo(StageData.class); Map<String, Object> parentOutputs = parent.getOutputs(); Map<String, String> rollbackTypes = (Map<String, String>) parentOutputs.get("rollbackTypes"); Map<String, Map<String, Object>> rollbackContexts = (Map<String, Map<String, Object>>) parentOutputs.get("rollbackContexts"); if (addLocking) { graph.append(stage -> { stage.setType(AcquireLockStage.PIPELINE_TYPE); stage.getContext().put("lock", Collections.singletonMap("lockName", lockName)); }); context.put( "rollbackType", ((Map) parent.getOutputs().get("rollbackTypes")).get(region) ); context.putAll(propagateParentStageContext(parent.getParent())); it.setType(RollbackServerGroupStage.PIPELINE_CONFIG_TYPE); it.setName("Rollback " + region); it.setContext(context); }); stage.setType(ReleaseLockStage.PIPELINE_TYPE); stage.getContext().put("lock", Collections.singletonMap("lockName", lockName)); });
@Override public final void beforeStages(@Nonnull Stage parent, @Nonnull StageGraphBuilder graph) { if (lockingConfigurationProperties.isEnabled()) { List<Location> locations = locationsFromStage(parent.getContext()); ClusterSelection clusterSelection = parent.mapTo(ClusterSelection.class); for (Location location : locations) { String lockName = ClusterLockHelper.clusterLockName( clusterSelection.getMoniker(), clusterSelection.getCredentials(), location); if (trafficGuard.hasDisableLock(clusterSelection.getMoniker(), clusterSelection.getCredentials(), location)) { graph.add(stage -> { stage.setType(AcquireLockStage.PIPELINE_TYPE); stage.getContext().put("lock", Collections.singletonMap("lockName", lockName)); }); } } } addAdditionalBeforeStages(parent, graph); }
@Override public void taskGraph( @Nonnull Stage stage, @Nonnull TaskNode.Builder builder) { builder.withTask("suspendExecutionDuringTimeWindow", SuspendExecutionDuringTimeWindowTask.class); try { JitterConfig jitter = stage.mapTo("/restrictedExecutionWindow/jitter", JitterConfig.class); if (jitter.enabled && jitter.maxDelay > 0) { if (jitter.skipManual && stage.getExecution().getTrigger().getType().equals("manual")) { return; } long waitTime = ThreadLocalRandom.current().nextLong(jitter.minDelay, jitter.maxDelay + 1); stage.setContext(contextWithWait(stage.getContext(), waitTime)); builder.withTask("waitForJitter", WaitTask.class); } } catch (IllegalArgumentException e) { // Do nothing } }
@Test public void test_that_getRunCanaryStages_returns_the_expected_sorted_list_of_stages_sorted_by_the_number_in_the_stage_name() { Stage stage = mock(Stage.class); Execution execution = mock(Execution.class); when(stage.getExecution()).thenReturn(execution); when(execution.getStages()).thenReturn(ImmutableList.of( new Stage(null, STAGE_TYPE, "foo #1", Maps.newHashMap(ImmutableMap.of("index", "0"))), new Stage(null, STAGE_TYPE, "foo #3", Maps.newHashMap(ImmutableMap.of("index", "2"))), new Stage(null, STAGE_TYPE, "foo #2", Maps.newHashMap(ImmutableMap.of("index", "1"))), new Stage(null, STAGE_TYPE, "foo #4", Maps.newHashMap(ImmutableMap.of("index", "3"))) )); List<Stage> actual = task.getRunCanaryStages(stage); for (int i = 0; i < 4; i++) { assertEquals(String.valueOf(i), actual.get(i).getContext().get("index")); } }
default Map<String, List<String>> manifestNamesByNamespace(Stage stage) { Map<String, List<String>> result = (Map<String, List<String>>) stage.getContext().get("outputs.manifestNamesByNamespace"); if (result != null) { return result; } result = new HashMap<>(); String name = (String) stage.getContext().get("manifest.name"); String location = (String) stage.getContext().get("manifest.location"); if (name != null && location != null) { result.put(location, Collections.singletonList(name)); } else { Logger.getLogger(this.getClass().getName()).warning("No manifests found in stage " + stage.getId()); } return result; } }
private static Predicate<Stage> isManualStageWithManualInput(String id) { return i -> (id != null && id.equals(i.getName())) && (i.getContext() != null && i.getType().equals("manualJudgment") && i.getContext().get("judgmentInput") != null); }