public static WorkflowInstance parseKey(String key) { final int lastHashPos = key.lastIndexOf('#'); if (lastHashPos < 1) { throw new IllegalArgumentException("Key must contain a hash '#' sign on position > 0"); } final WorkflowId workflowId = WorkflowId.parseKey(key.substring(0, lastHashPos)); return create(workflowId, key.substring(lastHashPos + 1)); }
public String toKey() { // Used as ID in storage etc. Do not change. return workflowId().toKey() + "#" + parameter(); }
@Override public Event toEvent() { return Event.info(WorkflowInstance.parseKey(workflowInstance), message); } }
private Response<TriggerRequest> triggerWorkflowInstance( AuthContext ac, RequestContext rc, TriggerRequest triggerRequest) { final WorkflowInstance workflowInstance = WorkflowInstance.create( triggerRequest.workflowId(), triggerRequest.parameter()); final Workflow workflow; final Optional<Workflow> workflowResult = storage.workflow(workflowInstance.workflowId()); if (workflowResult.isPresent()) { workflow = workflowResult.get(); workflowInstance.parameter(), workflow.configuration().schedule()); } catch (IllegalArgumentException e) {
List<WorkflowInstanceExecutionData> executionData(WorkflowId workflowId, String start, String stop) throws IOException { try (final Table eventsTable = connection.getTable(EVENTS_TABLE_NAME)) { final Scan scan = new Scan() .setRowPrefixFilter(Bytes.toBytes(workflowId.toKey() + '#')) .setFilter(new FirstKeyOnlyFilter()); final WorkflowInstance startRow = WorkflowInstance.create(workflowId, start); scan.setStartRow(Bytes.toBytes(startRow.toKey() + '#')); if (!Strings.isNullOrEmpty(stop)) { final WorkflowInstance stopRow = WorkflowInstance.create(workflowId, stop); scan.setStopRow(Bytes.toBytes(stopRow.toKey() + '#')); } final Set<WorkflowInstance> workflowInstancesSet = Sets.newHashSet(); try (ResultScanner scanner = eventsTable.getScanner(scan)) { Result result = scanner.next(); while (result != null) { final String key = new String(result.getRow()); final int lastHash = key.lastIndexOf('#'); final WorkflowInstance wfi = WorkflowInstance.parseKey(key.substring(0, lastHash)); workflowInstancesSet.add(wfi); result = scanner.next(); } } return executionData(workflowInstancesSet); } }
@Override public Map<WorkflowInstance, RunState> readActiveStates(String componentId) throws IOException { return activeStatesMap.entrySet().stream() .filter((entry) -> componentId.equals(entry.getKey().workflowId().componentId())) .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)); }
@Override public String toString() { // Ensure that the key gets printed when containing objects are logged return toKey(); } }
@Override public void printStates(RunStateDataPayload runStateDataPayload) { System.out.println(String.format(" %-20s %-12s %-47s %-7s %s", "WORKFLOW INSTANCE", "STATE", "EXECUTION ID", "TRIES", "PREVIOUS EXECUTION MESSAGE")); CliUtil.groupStates(runStateDataPayload.activeStates()).entrySet().forEach(entry -> { System.out.println(); System.out.println(String.format("%s %s", colored(CYAN, entry.getKey().componentId()), colored(BLUE, entry.getKey().id()))); entry.getValue().forEach(runStateData -> { final StateData stateData = runStateData.stateData(); final Ansi ansiState = getAnsiForState(runStateData); final Message lastMessage = stateData.message().orElse(Message.create(Message.MessageLevel.UNKNOWN, "No info")); final Ansi ansiMessage = colored(messageColor(lastMessage.level()), lastMessage.line()); System.out.println(String.format(" %-20s %-20s %-47s %-7d %s", runStateData.workflowInstance().parameter(), ansiState, stateData.executionId().orElse("<no-execution-id>"), stateData.tries(), ansiMessage)); }); }); }
List<WorkflowInstanceExecutionData> executionData(WorkflowId workflowId, String offset, int limit) throws IOException { try (final Table eventsTable = connection.getTable(EVENTS_TABLE_NAME)) { final Scan scan = new Scan() .setRowPrefixFilter(Bytes.toBytes(workflowId.toKey() + '#')) .setFilter(new FirstKeyOnlyFilter()); if (!Strings.isNullOrEmpty(offset)) { final WorkflowInstance offsetInstance = WorkflowInstance.create(workflowId, offset); scan.setStartRow(Bytes.toBytes(offsetInstance.toKey() + '#')); } final Set<WorkflowInstance> workflowInstancesSet = Sets.newHashSet(); try (ResultScanner scanner = eventsTable.getScanner(scan)) { Result result = scanner.next(); while (result != null) { final String key = new String(result.getRow()); final int lastHash = key.lastIndexOf('#'); final WorkflowInstance wfi = WorkflowInstance.parseKey(key.substring(0, lastHash)); workflowInstancesSet.add(wfi); if (workflowInstancesSet.size() == limit) { break; } result = scanner.next(); } } return executionData(workflowInstancesSet); } }
static SortedMap<WorkflowId, SortedSet<RunStateData>> groupStates(List<RunStateData> runStateDataList) { return runStateDataList.stream() .collect(groupingBy( state -> state.workflowInstance().workflowId(), CliUtil::newSortedWorkflowIdSet, toCollection(CliUtil::newSortedStateSet) )); }
@Override public PersistentEvent submit(WorkflowInstance workflowInstance, ExecutionDescription executionDescription, String executionId) { return new Submit(workflowInstance.toKey(), executionDescription, executionId); }
@Override public void printStates(RunStateDataPayload runStateDataPayload) { SortedMap<WorkflowId, SortedSet<RunStateDataPayload.RunStateData>> groupedStates = CliUtil.groupStates(runStateDataPayload.activeStates()); groupedStates.forEach((workflowId, value) -> value.forEach(RunStateData -> { final StateData stateData = RunStateData.stateData(); System.out.println(String.format( "%s %s %s %s %s %d %s", workflowId.componentId(), workflowId.id(), RunStateData.workflowInstance().parameter(), RunStateData.state(), stateData.executionId().orElse("<no-execution-id>"), stateData.tries(), stateData.message().map(Message::line).orElse("No info") )); })); }
private EventsPayload eventsForWorkflowInstance(String cid, String eid, String iid) { final WorkflowId workflowId = WorkflowId.create(cid, eid); final WorkflowInstance workflowInstance = WorkflowInstance.create(workflowId, iid); try { final Set<SequenceEvent> sequenceEvents = storage.readEvents(workflowInstance); final List<EventsPayload.TimestampedEvent> timestampedEvents = sequenceEvents.stream() .map(sequenceEvent -> EventsPayload.TimestampedEvent.create( sequenceEvent.event(), sequenceEvent.timestamp())) .collect(toList()); return EventsPayload.create(timestampedEvents); } catch (IOException e) { throw new RuntimeException(e); } } }
private static List<EnvVar> buildEnv(WorkflowInstance workflowInstance, RunSpec runSpec, String styxEnvironment) { // store user provided env first to prevent accidentally/intentionally overwriting system ones final Map<String, String> env = new HashMap<>(runSpec.env()); env.put(COMPONENT_ID, workflowInstance.workflowId().componentId()); env.put(WORKFLOW_ID, workflowInstance.workflowId().id()); env.put(PARAMETER, workflowInstance.parameter()); env.put(COMMIT_SHA, runSpec.commitSha().orElse("")); env.put(SERVICE_ACCOUNT, runSpec.serviceAccount().orElse("")); env.put(DOCKER_ARGS, String.join(" ", runSpec.args())); env.put(DOCKER_IMAGE, runSpec.imageName()); env.put(EXECUTION_ID, runSpec.executionId()); env.put(TERMINATION_LOG, "/dev/termination-log"); env.put(TRIGGER_ID, runSpec.trigger().map(TriggerUtil::triggerId).orElse(null)); env.put(TRIGGER_TYPE, runSpec.trigger().map(TriggerUtil::triggerType).orElse(null)); env.put(ENVIRONMENT, styxEnvironment); env.put(LOGGING, "structured"); return env.entrySet().stream() .map(entry -> envVar(entry.getKey(), entry.getValue())) .collect(Collectors.toList()); }
private Map<WorkflowId, Workflow> getWorkflows(final List<InstanceState> activeStates) { final Set<WorkflowId> workflowIds = activeStates.stream() .map(activeState -> activeState.workflowInstance().workflowId()) .collect(toSet()); return storage.workflows(workflowIds); }
@Override public PersistentEvent triggerExecution(WorkflowInstance workflowInstance, Trigger trigger, TriggerParameters parameters) { return new TriggerExecution(workflowInstance.toKey(), Optional.of(trigger), Optional.of(parameters)); }
@Override public Event toEvent() { return Event.terminate(WorkflowInstance.parseKey(workflowInstance), exitCode); } }
private RunSpec createRunSpec(RunState state) throws ResourceNotFoundException { final Optional<ExecutionDescription> executionDescriptionOpt = state.data().executionDescription(); final ExecutionDescription executionDescription = executionDescriptionOpt.orElseThrow( () -> new ResourceNotFoundException("Missing execution description for " + state.workflowInstance())); final String executionId = state.data().executionId().orElseThrow( () -> new ResourceNotFoundException("Missing execution id for " + state.workflowInstance())); final String dockerImage = executionDescription.dockerImage(); final List<String> dockerArgs = executionDescription.dockerArgs(); final String parameter = state.workflowInstance().parameter(); final List<String> command = argsReplace(dockerArgs, parameter); return RunSpec.builder() .executionId(executionId) .imageName(dockerImage) .args(command) .terminationLogging(executionDescription.dockerTerminationLogging()) .secret(executionDescription.secret()) .serviceAccount(executionDescription.serviceAccount()) .trigger(state.data().trigger()) .commitSha(state.data().executionDescription().flatMap(ExecutionDescription::commitSha)) .env(executionDescription.env()) .build(); } }
private WorkflowInstance parseWorkflowInstance(Entity activeWorkflowInstance) { final String componentId = activeWorkflowInstance.getString(PROPERTY_COMPONENT); final String workflowId = activeWorkflowInstance.getString(PROPERTY_WORKFLOW); final String parameter = activeWorkflowInstance.getString(PROPERTY_PARAMETER); return WorkflowInstance.create(WorkflowId.create(componentId, workflowId), parameter); }
private ExecutionDescription getExecDescription(WorkflowInstance workflowInstance, StateData data) throws IOException, MissingRequiredPropertyException { final WorkflowId workflowId = workflowInstance.workflowId(); final List<String> command = argsReplace(dockerArgs, workflowInstance.parameter());