@Override public Map<WorkflowInstance, RunState> readActiveStates(String componentId) throws IOException { return activeStatesMap.entrySet().stream() .filter((entry) -> componentId.equals(entry.getKey().workflowId().componentId())) .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)); }
static SortedMap<WorkflowId, SortedSet<RunStateData>> groupStates(List<RunStateData> runStateDataList) { return runStateDataList.stream() .collect(groupingBy( state -> state.workflowInstance().workflowId(), CliUtil::newSortedWorkflowIdSet, toCollection(CliUtil::newSortedStateSet) )); }
private Map<WorkflowId, Workflow> getWorkflows(final List<InstanceState> activeStates) { final Set<WorkflowId> workflowIds = activeStates.stream() .map(activeState -> activeState.workflowInstance().workflowId()) .collect(toSet()); return storage.workflows(workflowIds); }
private void dequeueInstances( StyxConfig config, Map<String, Resource> resources, Map<WorkflowId, Set<String>> workflowResourceReferences, Map<WorkflowId, Workflow> workflows, List<InstanceState> eligibleInstances, AtomicLongMap<String> currentResourceDemand) { final ConcurrentMap<String, Boolean> resourceExhaustedCache = new ConcurrentHashMap<>(); final Map<WorkflowInstance, CompletableFuture<Void>> futures = eligibleInstances.stream() .collect(toMap( InstanceState::workflowInstance, instanceState -> CompletableFuture.runAsync(() -> dequeueInstance(config, resources, workflowResourceReferences, Optional.ofNullable(workflows.get(instanceState.workflowInstance().workflowId())), instanceState, resourceExhaustedCache, currentResourceDemand), executor))); futures.forEach((instance, future) -> { try { future.get(); } catch (InterruptedException e) { LOG.warn("Interrupted", e); Thread.currentThread().interrupt(); throw new RuntimeException(e); } catch (ExecutionException e) { LOG.error("Failed to process instance for dequeue: " + instance, e); } }); }
public String toKey() { // Used as ID in storage etc. Do not change. return workflowId().toKey() + "#" + parameter(); }
public static Set<WorkflowInstance> getTimedOutInstances(Map<WorkflowId, Workflow> workflows, List<InstanceState> activeStates, Instant instant, TimeoutConfig ttl) { return activeStates.parallelStream() .filter(entry -> { final Optional<Workflow> workflowOpt = Optional.ofNullable(workflows.get(entry.workflowInstance().workflowId())); return hasTimedOut(workflowOpt, entry.runState(), instant, ttl.ttlOf(entry.runState().state())); }) .map(InstanceState::workflowInstance) .collect(toSet()); }
private static List<EnvVar> buildEnv(WorkflowInstance workflowInstance, RunSpec runSpec, String styxEnvironment) { // store user provided env first to prevent accidentally/intentionally overwriting system ones final Map<String, String> env = new HashMap<>(runSpec.env()); env.put(COMPONENT_ID, workflowInstance.workflowId().componentId()); env.put(WORKFLOW_ID, workflowInstance.workflowId().id()); env.put(PARAMETER, workflowInstance.parameter()); env.put(COMMIT_SHA, runSpec.commitSha().orElse("")); env.put(SERVICE_ACCOUNT, runSpec.serviceAccount().orElse("")); env.put(DOCKER_ARGS, String.join(" ", runSpec.args())); env.put(DOCKER_IMAGE, runSpec.imageName()); env.put(EXECUTION_ID, runSpec.executionId()); env.put(TERMINATION_LOG, "/dev/termination-log"); env.put(TRIGGER_ID, runSpec.trigger().map(TriggerUtil::triggerId).orElse(null)); env.put(TRIGGER_TYPE, runSpec.trigger().map(TriggerUtil::triggerType).orElse(null)); env.put(ENVIRONMENT, styxEnvironment); env.put(LOGGING, "structured"); return env.entrySet().stream() .map(entry -> envVar(entry.getKey(), entry.getValue())) .collect(Collectors.toList()); }
private static Stream<ResourceWithInstance> pairWithResources(boolean globalConcurrencyEnabled, InstanceState instanceState, Map<WorkflowId, Workflow> workflows, WorkflowResourceDecorator resourceDecorator) { final Optional<Workflow> workflowOpt = Optional.ofNullable(workflows.get(instanceState.workflowInstance().workflowId())); final Set<String> workflowResources = workflowResources(globalConcurrencyEnabled, workflowOpt); return workflowOpt .map(workflow -> resourceDecorator.decorateResources( instanceState.runState(), workflow.configuration(), workflowResources)) .orElse(workflowResources).stream() .map(resource -> ResourceWithInstance.create(resource, instanceState)); }
try { storage.runInTransaction(tx -> { final Optional<Workflow> workflow = tx.workflow(workflowInstance.workflowId()); if (!workflow.isPresent()) { throw new IllegalArgumentException( "Workflow not found: " + workflowInstance.workflowId().toKey());
private KubernetesSecretSpec ensureSecrets(WorkflowInstance workflowInstance, RunSpec runSpec) { return KubernetesSecretSpec.builder() .customSecret(ensureCustomSecret(workflowInstance, runSpec)) .serviceAccountSecret(runSpec.serviceAccount().map( serviceAccount -> serviceAccountSecretManager.ensureServiceAccountKeySecret( workflowInstance.workflowId().toString(), serviceAccount))) .build(); }
private Optional<WorkflowConfiguration.Secret> ensureCustomSecret( WorkflowInstance workflowInstance, RunSpec runSpec) { return runSpec.secret().map(specSecret -> { if (specSecret.name().startsWith(STYX_WORKFLOW_SA_SECRET_NAME)) { LOG.warn("[AUDIT] Workflow {} refers to secret {} with managed service account key secret name prefix, " + "denying execution", workflowInstance.workflowId(), specSecret.name()); throw new InvalidExecutionException( "Referenced secret '" + specSecret.name() + "' has the managed service account key secret name prefix"); } // if it ever happens, that feels more like a hack than pure luck so let's be paranoid if (STYX_WORKFLOW_SA_SECRET_MOUNT_PATH.equals(specSecret.mountPath())) { LOG.warn("[AUDIT] Workflow {} tries to mount secret {} to the reserved path", workflowInstance.workflowId(), specSecret.name()); throw new InvalidExecutionException( "Referenced secret '" + specSecret.name() + "' has the mount path " + STYX_WORKFLOW_SA_SECRET_MOUNT_PATH + " defined that is reserved"); } final Secret secret = client.secrets().withName(specSecret.name()).get(); if (secret == null) { LOG.warn("[AUDIT] Workflow {} refers to a non-existent secret {}", workflowInstance.workflowId(), specSecret.name()); throw new InvalidExecutionException( "Referenced secret '" + specSecret.name() + "' was not found"); } else { LOG.info("[AUDIT] Workflow {} refers to secret {}", workflowInstance.workflowId(), specSecret.name()); } return specSecret; }); }
private Response<WorkflowInstance> haltWorkflowInstance(AuthContext ac, WorkflowInstance workflowInstance) { workflowActionAuthorizer.authorizeWorkflowAction(ac, workflowInstance.workflowId()); final Event event = Event.halt(workflowInstance); return Response.forStatus(eventInjectorHelper(event)).withPayload(workflowInstance); }
private Response<WorkflowInstance> retryWorkflowInstanceAfter(AuthContext ac, RequestContext rc, WorkflowInstance workflowInstance) { workflowActionAuthorizer.authorizeWorkflowAction(ac, workflowInstance.workflowId()); final long delay; try { delay = Long.parseLong(rc.request().parameter("delay").orElse("0")); } catch (NumberFormatException e) { return Response.forStatus(BAD_REQUEST.withReasonPhrase( "Delay parameter could not be parsed")); } final Event event = Event.retryAfter(workflowInstance, delay); return Response.forStatus(eventInjectorHelper(event)).withPayload(workflowInstance); }
workflowResourceReferences.getOrDefault(instanceState.workflowInstance().workflowId(), emptySet());
private Response<Event> injectEvent(AuthContext ac, Event event) { workflowActionAuthorizer.authorizeWorkflowAction(ac, event.workflowInstance().workflowId()); if ("dequeue".equals(EventUtil.name(event))) { // For backwards compatibility return Response.forStatus(eventInjectorHelper( Event.retryAfter(event.workflowInstance(), 0L))).withPayload(event); } else if ("halt".equals(EventUtil.name(event))) { // For backwards compatibility return Response.forStatus(eventInjectorHelper(event)); } else if ("timeout".equals(EventUtil.name(event))) { // This is for manually getting out of a stale state return Response.forStatus(eventInjectorHelper(event)); } else { return Response.forStatus(BAD_REQUEST.withReasonPhrase( "This API for injecting generic events is deprecated, refer to the specific API for the " + "event you want to send to the scheduler")); } }
private ExecutionDescription getExecDescription(WorkflowInstance workflowInstance, StateData data) throws IOException, MissingRequiredPropertyException { final WorkflowId workflowId = workflowInstance.workflowId();
final Optional<Workflow> workflowResult = storage.workflow(workflowInstance.workflowId()); if (workflowResult.isPresent()) { workflow = workflowResult.get();
final Key key = activeWorkflowInstanceKey(keyFactory, wfi); final Entity.Builder entity = Entity.newBuilder(key) .set(PROPERTY_COMPONENT, wfi.workflowId().componentId()) .set(PROPERTY_WORKFLOW, wfi.workflowId().id()) .set(PROPERTY_PARAMETER, wfi.parameter()) .set(PROPERTY_COUNTER, state.counter());