public String toKey() { // Used as ID in storage etc. Do not change. return componentId() + "#" + id(); }
public static WorkflowId parseKey(String key) { final int hashPos = key.indexOf('#'); if (hashPos < 1) { throw new IllegalArgumentException("Key must contain a hash '#' sign on position > 0"); } return create(key.substring(0, hashPos), key.substring(hashPos + 1)); } }
private void logAuthorization(WorkflowId workflowId, String serviceAccount, boolean enforce, String accessMessage, boolean cached) { log.info("[AUDIT] {}, authorizing use of service account {} in workflow {} (Enforcing: {}, Cache: {})", accessMessage, serviceAccount, workflowId.toKey(), enforce, cached ? CACHE_HIT : CACHE_MISS); }
@Override public List<Backfill> backfillsForWorkflow(boolean showAll, String workflow) throws IOException { Stream<Backfill> backfillStream = backfillStore.values().stream() .filter(backfill -> backfill.workflowId().id().equals(workflow)); if (!showAll) { backfillStream = backfillStream .filter(backfill -> backfill.halted() && backfill.allTriggered()); } return ImmutableList.copyOf(backfillStream.collect(Collectors.toList())); }
@Override public Map<WorkflowInstance, RunState> readActiveStates(String componentId) throws IOException { return activeStatesMap.entrySet().stream() .filter((entry) -> componentId.equals(entry.getKey().workflowId().componentId())) .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)); }
public WorkflowId id() { return WorkflowId.ofWorkflow(this); }
public static WorkflowInstance parseKey(String key) { final int lastHashPos = key.lastIndexOf('#'); if (lastHashPos < 1) { throw new IllegalArgumentException("Key must contain a hash '#' sign on position > 0"); } final WorkflowId workflowId = WorkflowId.parseKey(key.substring(0, lastHashPos)); return create(workflowId, key.substring(lastHashPos + 1)); }
@Override public void printWorkflow(Workflow wf, WorkflowState state) { System.out.println("Component: " + wf.componentId()); System.out.println(" Workflow: " + wf.id().id()); System.out.println(" Schedule: " + wf.configuration().schedule()); System.out.println(" Offset: " + wf.configuration().offset().orElse("")); System.out.println(" Image: " + wf.configuration().dockerImage().orElse("")); System.out.println(" Args: " + wf.configuration().dockerArgs().orElse(Collections.emptyList())); System.out.println(" TermLog: " + wf.configuration().dockerTerminationLogging()); System.out.println(" Secret: " + wf.configuration().secret().map(s -> s.name() + ':' + s.mountPath()).orElse("")); System.out.println(" Svc Acct: " + wf.configuration().serviceAccount().orElse("")); System.out.println("Resources: " + wf.configuration().resources()); System.out.println(" Env: " + Joiner.on(' ').withKeyValueSeparator('=').join(wf.configuration().env())); System.out.println(" Timeout: " + wf.configuration().runningTimeout().map(Duration::toString).orElse("")); System.out.println(" Commit: " + wf.configuration().commitSha().orElse("")); System.out.println(" Enabled: " + state.enabled().map(Object::toString).orElse("")); System.out.println(" Trig: " + state.nextNaturalTrigger().map(Object::toString).orElse("")); System.out.println("Ofst Trig: " + state.nextNaturalOffsetTrigger().map(Object::toString).orElse("")); }
@Override public List<Backfill> backfillsForComponent(boolean showAll, String component) throws IOException { Stream<Backfill> backfillStream = backfillStore.values().stream() .filter(backfill -> backfill.workflowId().componentId().equals(component)); if (!showAll) { backfillStream = backfillStream .filter(backfill -> backfill.halted() && backfill.allTriggered()); } return ImmutableList.copyOf(backfillStream.collect(Collectors.toList())); }
.stream() .filter(workflow1 -> workflow1.configuration().dockerImage().isPresent()) .filter(workflow1 -> enabledWorkflowCache.get().contains(WorkflowId.ofWorkflow(workflow1))) .count());
static Workflow parseWorkflowJson(Entity entity, WorkflowId workflowId) throws IOException { try { return OBJECT_MAPPER .readValue(entity.getString(PROPERTY_WORKFLOW_JSON), Workflow.class); } catch (IOException e) { LOG.error("Failed to read workflow for {}, {}", workflowId.componentId(), workflowId.id(), e); throw e; } }
private Response<Workflow> workflow(String componentId, String id) { try { return storage.workflow(WorkflowId.create(componentId, id)) .map(Response::forPayload) .orElse(Response.forStatus(Status.NOT_FOUND)); } catch (IOException e) { throw Throwables.propagate(e); } }
private void logDenial(WorkflowId workflowId, String serviceAccount, boolean enforce, String principalEmail, boolean cached) { log.info("[AUDIT] Principal {} denied use of service account {} in workflow {} (Enforcing: {}, Cache: {})", principalEmail, serviceAccount, workflowId.toKey(), enforce, cached ? CACHE_HIT : CACHE_MISS); }
@Override public void printWorkflow(Workflow wf, WorkflowState state) { System.out.println(Joiner.on(' ').join( wf.componentId(), wf.id().id(), wf.configuration().schedule(), wf.configuration().offset().orElse(""), wf.configuration().dockerImage().orElse(""), wf.configuration().dockerArgs().orElse(Collections.emptyList()), wf.configuration().dockerTerminationLogging(), wf.configuration().secret().map(s -> s.name() + ':' + s.mountPath()).orElse(""), wf.configuration().serviceAccount().map(Object::toString).orElse(""), wf.configuration().resources(), wf.configuration().env(), wf.configuration().runningTimeout().map(Duration::toString).orElse(""), wf.configuration().commitSha().orElse(""), state.enabled().map(Object::toString).orElse(""), state.nextNaturalTrigger().map(Object::toString).orElse(""), state.nextNaturalOffsetTrigger().map(Object::toString).orElse(""))); }
static Key workflowKey(KeyFactory keyFactory, WorkflowId workflowId) { return keyFactory.addAncestor(PathElement.of(KIND_COMPONENT, workflowId.componentId())) .setKind(KIND_WORKFLOW) .newKey(workflowId.id()); }
private EventsPayload eventsForWorkflowInstance(String cid, String eid, String iid) { final WorkflowId workflowId = WorkflowId.create(cid, eid); final WorkflowInstance workflowInstance = WorkflowInstance.create(workflowId, iid); try { final Set<SequenceEvent> sequenceEvents = storage.readEvents(workflowInstance); final List<EventsPayload.TimestampedEvent> timestampedEvents = sequenceEvents.stream() .map(sequenceEvent -> EventsPayload.TimestampedEvent.create( sequenceEvent.event(), sequenceEvent.timestamp())) .collect(toList()); return EventsPayload.create(timestampedEvents); } catch (IOException e) { throw new RuntimeException(e); } } }
@Override public void printWorkflows(List<Workflow> workflows) { workflows.forEach(wf -> System.out.println(wf.id().toKey())); }
private static List<EnvVar> buildEnv(WorkflowInstance workflowInstance, RunSpec runSpec, String styxEnvironment) { // store user provided env first to prevent accidentally/intentionally overwriting system ones final Map<String, String> env = new HashMap<>(runSpec.env()); env.put(COMPONENT_ID, workflowInstance.workflowId().componentId()); env.put(WORKFLOW_ID, workflowInstance.workflowId().id()); env.put(PARAMETER, workflowInstance.parameter()); env.put(COMMIT_SHA, runSpec.commitSha().orElse("")); env.put(SERVICE_ACCOUNT, runSpec.serviceAccount().orElse("")); env.put(DOCKER_ARGS, String.join(" ", runSpec.args())); env.put(DOCKER_IMAGE, runSpec.imageName()); env.put(EXECUTION_ID, runSpec.executionId()); env.put(TERMINATION_LOG, "/dev/termination-log"); env.put(TRIGGER_ID, runSpec.trigger().map(TriggerUtil::triggerId).orElse(null)); env.put(TRIGGER_TYPE, runSpec.trigger().map(TriggerUtil::triggerType).orElse(null)); env.put(ENVIRONMENT, styxEnvironment); env.put(LOGGING, "structured"); return env.entrySet().stream() .map(entry -> envVar(entry.getKey(), entry.getValue())) .collect(Collectors.toList()); }
try { if (componentOpt.isPresent() && workflowOpt.isPresent()) { final WorkflowId workflowId = WorkflowId.create(componentOpt.get(), workflowOpt.get()); backfills = storage.backfillsForWorkflowId(showAll, workflowId).stream(); } else if (componentOpt.isPresent()) {
@Override public void printStates(RunStateDataPayload runStateDataPayload) { printJson(CliUtil.groupStates(runStateDataPayload.activeStates()).entrySet().stream() .collect(toMap(e -> e.getKey().toKey(), Entry::getValue))); }