@Override public void printWorkflow(Workflow wf, WorkflowState state) { System.out.println("Component: " + wf.componentId()); System.out.println(" Workflow: " + wf.id().id()); System.out.println(" Schedule: " + wf.configuration().schedule()); System.out.println(" Offset: " + wf.configuration().offset().orElse("")); System.out.println(" Image: " + wf.configuration().dockerImage().orElse("")); System.out.println(" Args: " + wf.configuration().dockerArgs().orElse(Collections.emptyList())); System.out.println(" TermLog: " + wf.configuration().dockerTerminationLogging()); System.out.println(" Secret: " + wf.configuration().secret().map(s -> s.name() + ':' + s.mountPath()).orElse("")); System.out.println(" Svc Acct: " + wf.configuration().serviceAccount().orElse("")); System.out.println("Resources: " + wf.configuration().resources()); System.out.println(" Env: " + Joiner.on(' ').withKeyValueSeparator('=').join(wf.configuration().env())); System.out.println(" Timeout: " + wf.configuration().runningTimeout().map(Duration::toString).orElse("")); System.out.println(" Commit: " + wf.configuration().commitSha().orElse("")); System.out.println(" Enabled: " + state.enabled().map(Object::toString).orElse("")); System.out.println(" Trig: " + state.nextNaturalTrigger().map(Object::toString).orElse("")); System.out.println("Ofst Trig: " + state.nextNaturalOffsetTrigger().map(Object::toString).orElse("")); }
private TriggerInstantSpec initializeNaturalTrigger(Workflow workflow) { final Instant now = time.get(); final Instant offsetNow = workflow.configuration().subtractOffset(now); final Schedule schedule = workflow.configuration().schedule(); final Instant nextTrigger = nextInstant(offsetNow, schedule); final Instant nextWithOffset = workflow.configuration().addOffset(nextTrigger); return TriggerInstantSpec.create(nextTrigger, nextWithOffset); } }
default String defaultOffset() { return defaultOffset(schedule()); }
workflowId, workflowInstance))); final String dockerImage = workflow.configuration().dockerImage().orElseThrow( () -> new MissingRequiredPropertyException(format("%s has no docker image, halting %s", workflowId, final List<String> dockerArgs = workflow.configuration().dockerArgs() .orElse(Collections.emptyList()); final List<String> command = argsReplace(dockerArgs, workflowInstance.parameter()); final Map<String, String> env = new HashMap<>(workflow.configuration().env()); data.triggerParameters().ifPresent(p -> env.putAll(p.env())); .dockerImage(dockerImage) .dockerArgs(command) .dockerTerminationLogging(workflow.configuration().dockerTerminationLogging()) .secret(workflow.configuration().secret()) .serviceAccount(workflow.configuration().serviceAccount()) .commitSha(workflow.configuration().commitSha()) .env(env) .build();
@Override public CompletionStage<Void> event(Workflow workflow, Trigger trigger, Instant instant, TriggerParameters parameters) { if (!workflow.configuration().dockerImage().isPresent()) { LOG.warn("{} has no docker image, skipping", workflow.id()); return CompletableFuture.completedFuture(null); } final String parameter = toParameter(workflow.configuration().schedule(), instant); final WorkflowInstance workflowInstance = WorkflowInstance.create(workflow.id(), parameter); try { return stateManager.trigger(workflowInstance, trigger, parameters); } catch (IsClosedException isClosedException) { LOG.warn("State receiver is closed when processing workflow {} for trigger {} at {}", workflow, trigger, instant, isClosedException); return exceptionallyCompletedFuture(isClosedException); } } }
} catch (Exception e) { final WorkflowInstance workflowInstance = WorkflowInstance.create(workflow.id(), toParameter(workflow.configuration().schedule(), instantSpec.instant())); final Schedule schedule = workflow.configuration().schedule(); final Instant nextTrigger = nextInstant(instantSpec.instant(), schedule); final Instant nextWithOffset = workflow.configuration().addOffset(nextTrigger); final TriggerInstantSpec nextSpec = TriggerInstantSpec.create(nextTrigger, nextWithOffset);
final Schedule newSchedule = workflow.configuration().schedule(); final Optional<String> newOffset = workflow.configuration().offset(); if (!previous.isPresent() || !previous.get().configuration().schedule().equals(newSchedule) || !previous.get().configuration().offset().equals(newOffset)) { try { nextSpec = Optional.of(initializeNaturalTrigger(workflow));
.filter(workflow -> workflow.configuration().dockerImage().isPresent()) .count()); .filter(workflow1 -> workflow1.configuration().dockerImage().isPresent()) .filter(workflow1 -> enabledWorkflowCache.get().contains(WorkflowId.ofWorkflow(workflow1))) .count()); workflowCache.get().values() .stream() .filter(workflow -> workflow.configuration().dockerImage().isPresent()) .filter(workflow -> workflow.configuration().dockerTerminationLogging()) .count());
return Response.forStatus(Status.NOT_FOUND.withReasonPhrase("No next natural trigger for workflow.")); final Schedule schedule = workflow.get().configuration().schedule(); final Instant nextNaturalTrigger = workflowState.nextNaturalTrigger().get(); final Instant startInstant = TimeUtil.offsetInstant(nextNaturalTrigger, schedule, -limit);
default Instant subtractOffset(Instant next) { final String offset = offset().orElseGet(this::defaultOffset); return TimeUtil.subtractOffset(next.atZone(ZoneOffset.UTC), offset).toInstant(); }
public static Set<String> workflowResources(boolean globalConcurrenyEnabled, Optional<Workflow> workflowOpt) { final ImmutableSet.Builder<String> builder = ImmutableSet.builder(); if (globalConcurrenyEnabled) { builder.add(GLOBAL_RESOURCE_ID); } workflowOpt.ifPresent(wf -> builder.addAll(wf.configuration().resources())); return builder.build(); }
private static boolean hasTimedOut(Optional<Workflow> workflowOpt, RunState runState, Instant instant, Duration timeout) { if (runState.state().isTerminal()) { return false; } final Duration effectiveTimeout = runState.state() == RunState.State.RUNNING ? workflowOpt .flatMap(workflow -> workflow.configuration().runningTimeout()) .orElse(timeout) : timeout; final Duration sanitizedTimeout = effectiveTimeout.compareTo(timeout) < 0 ? effectiveTimeout : timeout; final Instant deadline = Instant .ofEpochMilli(runState.timestamp()) .plus(sanitizedTimeout); return !deadline.isAfter(instant); }
private Optional<String> validate(RequestContext rc, BackfillInput input, Workflow workflow) { if (!workflow.configuration().dockerImage().isPresent()) { return Optional.of("Workflow is missing docker image"); } final Collection<String> errors = workflowValidator.validateWorkflow(workflow); if (!errors.isEmpty()) { return Optional.of("Invalid workflow configuration: " + String.join(", ", errors)); } final Schedule schedule = workflow.configuration().schedule(); if (!input.start().isBefore(input.end())) { return Optional.of("start must be before end"); } if (!TimeUtil.isAligned(input.start(), schedule)) { return Optional.of("start parameter not aligned with schedule"); } if (!TimeUtil.isAligned(input.end(), schedule)) { return Optional.of("end parameter not aligned with schedule"); } final boolean allowFuture = Boolean.parseBoolean(rc.request().parameter("allowFuture").orElse("false")); if (!allowFuture && (input.start().isAfter(time.get()) || TimeUtil.previousInstant(input.end(), schedule).isAfter(time.get()))) { return Optional.of("Cannot backfill future partitions"); } return Optional.empty(); }
final Schedule schedule = workflow.configuration().schedule(); if (TimeUtil.isAligned(instant, schedule)) { instant = TimeUtil.previousInstant(instant, schedule); triggerInstant = workflow.configuration().addOffset(instant); } else { triggerInstant = timestampToInstant(entity.getTimestamp(PROPERTY_NEXT_NATURAL_OFFSET_TRIGGER));
final Schedule schedule = workflow.configuration().schedule();
default Instant addOffset(Instant next) { final String offset = offset().orElseGet(this::defaultOffset); return TimeUtil.addOffset(next.atZone(ZoneOffset.UTC), offset).toInstant(); }
@Override public void printWorkflow(Workflow wf, WorkflowState state) { System.out.println(Joiner.on(' ').join( wf.componentId(), wf.id().id(), wf.configuration().schedule(), wf.configuration().offset().orElse(""), wf.configuration().dockerImage().orElse(""), wf.configuration().dockerArgs().orElse(Collections.emptyList()), wf.configuration().dockerTerminationLogging(), wf.configuration().secret().map(s -> s.name() + ':' + s.mountPath()).orElse(""), wf.configuration().serviceAccount().map(Object::toString).orElse(""), wf.configuration().resources(), wf.configuration().env(), wf.configuration().runningTimeout().map(Duration::toString).orElse(""), wf.configuration().commitSha().orElse(""), state.enabled().map(Object::toString).orElse(""), state.nextNaturalTrigger().map(Object::toString).orElse(""), state.nextNaturalOffsetTrigger().map(Object::toString).orElse(""))); }
if (!workflow.configuration().dockerImage().isPresent()) { return Response.forStatus(BAD_REQUEST.withReasonPhrase("Workflow is missing docker image")); instant = parseAlignedInstant( workflowInstance.parameter(), workflow.configuration().schedule()); } catch (IllegalArgumentException e) { return Response.forStatus(BAD_REQUEST.withReasonPhrase(e.getMessage()));
upperLimit(e, cfg.id().length(), MAX_ID_LENGTH, "id too long"); upperLimit(e, cfg.commitSha().map(String::length).orElse(0), MAX_COMMIT_SHA_LENGTH, "commitSha too long"); upperLimit(e, cfg.secret().map(s -> s.name().length()).orElse(0), MAX_SECRET_NAME_LENGTH, "secret name too long"); upperLimit(e, cfg.secret().map(s -> s.mountPath().length()).orElse(0), MAX_SECRET_MOUNT_PATH_LENGTH, "secret mount path too long"); upperLimit(e, cfg.serviceAccount().map(String::length).orElse(0), MAX_SERVICE_ACCOUNT_LENGTH, "service account too long"); upperLimit(e, cfg.resources().size(), MAX_RESOURCES, "too many resources"); upperLimit(e, cfg.env().size(), MAX_ENV_VARS, "too many env vars"); upperLimit(e, cfg.env().entrySet().stream() .mapToInt(entry -> entry.getKey().length() + entry.getValue().length()).sum(), MAX_ENV_SIZE, "env too big"); cfg.dockerImage().ifPresent(image -> dockerImageValidator.validateImageReference(image).stream() .map(s -> "invalid image: " + s) .forEach(e::add)); cfg.resources().stream().map(String::length).forEach(v -> upperLimit(e, v, MAX_RESOURCE_LENGTH, "resource name too long")); cfg.dockerArgs().ifPresent(args -> { final int dockerArgs = args.size() + args.stream().mapToInt(String::length).sum(); upperLimit(e, dockerArgs, MAX_DOCKER_ARGS_TOTAL, "docker args is too large");