public Event toEvent() { final WorkflowInstance workflowInstance = WorkflowInstance.parseKey(this.workflowInstance); switch (type) { case "timeTrigger": return Event.timeTrigger(workflowInstance); case "success": return Event.success(workflowInstance); case "retry": return Event.retry(workflowInstance); case "stop": return Event.stop(workflowInstance); case "timeout": return Event.timeout(workflowInstance); case "halt": return Event.halt(workflowInstance); default: throw new IllegalStateException("Event type " + type + " not covered by base PersistentEvent class"); } }
public static String info(Event event) { return event.accept(EventInfoVisitor.INSTANCE); }
@Override public Event toEvent() { return Event.dequeue(WorkflowInstance.parseKey(workflowInstance), resourceIds); } }
case PREPARE: try { final Event submitEvent = Event.submit( state.workflowInstance(), getExecDescription(workflowInstance, state.data()), createExecutionId()); try { stateManager.receiveIgnoreClosed(Event.halt(workflowInstance)); } catch (MissingRequiredPropertyException e) { LOG.warn("Failed to prepare execution description for " + state.workflowInstance(), e); stateManager.receiveIgnoreClosed(Event.halt(workflowInstance)); } catch (IOException e) { try { LOG.error("Failed to retrieve execution description for " + state.workflowInstance(), e); stateManager.receive(Event.runError(state.workflowInstance(), e.getMessage())); } catch (IsClosedException isClosedException) { LOG.warn("Failed to send 'runError' event", isClosedException);
} catch (ResourceNotFoundException e) { LOG.error("Unable to start docker procedure.", e); stateManager.receiveIgnoreClosed(Event.halt(state.workflowInstance())); return; final Event submitted = Event.submitted(state.workflowInstance(), runSpec.executionId()); try { stateManager.receive(submitted, state.counter()); LOG.error(msg, e); stateManager.receive(Event.runError(state.workflowInstance(), e.getMessage()), state.counter() + 1); } catch (IsClosedException isClosedException) {
private static List<Event> handleExited(WorkflowInstance workflowInstance, RunState state, Pod pod, Optional<ContainerStatus> mainContainerStatusOpt, Stats stats) { final List<Event> generatedEvents = Lists.newArrayList(); switch (state.state()) { case PREPARE: case SUBMITTED: generatedEvents.add(Event.started(workflowInstance)); // intentional fall-through case RUNNING: final Optional<Integer> exitCode = mainContainerStatusOpt.flatMap(cs -> getExitCodeIfValid(workflowInstance.toKey(), pod, cs, stats)); generatedEvents.add(Event.terminate(workflowInstance, exitCode)); break; default: // no event break; } return ImmutableList.copyOf(generatedEvents); }
private void checkRetry(RunState state) { final WorkflowInstance workflowInstance = state.workflowInstance(); if (state.data().retryCost() < MAX_RETRY_COST) { final Optional<Integer> exitCode = state.data().lastExit(); if (shouldFailFast(exitCode)) { stateManager.receiveIgnoreClosed(Event.stop(workflowInstance)); } else { final long delayMillis; if (isMissingDependency(exitCode)) { delayMillis = Duration.ofMinutes(MISSING_DEPS_RETRY_DELAY_MINUTES).toMillis(); } else { delayMillis = retryUtil.calculateDelay(state.data().consecutiveFailures()).toMillis(); } stateManager.receiveIgnoreClosed(Event.retryAfter(workflowInstance, delayMillis)); } } else { stateManager.receiveIgnoreClosed(Event.stop(workflowInstance)); } }
private Response<Event> injectEvent(AuthContext ac, Event event) { workflowActionAuthorizer.authorizeWorkflowAction(ac, event.workflowInstance().workflowId()); if ("dequeue".equals(EventUtil.name(event))) { // For backwards compatibility return Response.forStatus(eventInjectorHelper( Event.retryAfter(event.workflowInstance(), 0L))).withPayload(event); } else if ("halt".equals(EventUtil.name(event))) { // For backwards compatibility return Response.forStatus(eventInjectorHelper(event)); } else if ("timeout".equals(EventUtil.name(event))) { // This is for manually getting out of a stale state return Response.forStatus(eventInjectorHelper(event)); } else { return Response.forStatus(BAD_REQUEST.withReasonPhrase( "This API for injecting generic events is deprecated, refer to the specific API for the " + "event you want to send to the scheduler")); } }
Event.runError(instanceState.workflowInstance(), String.format("Referenced resources not found: %s", unknownResources)), instanceState.runState().counter()); stateManager.receiveIgnoreClosed(Event.retryAfter(instanceState.workflowInstance(), blocker.get().delay().toMillis()), instanceState.runState().counter());
private Response<WorkflowInstance> haltWorkflowInstance(AuthContext ac, WorkflowInstance workflowInstance) { workflowActionAuthorizer.authorizeWorkflowAction(ac, workflowInstance.workflowId()); final Event event = Event.halt(workflowInstance); return Response.forStatus(eventInjectorHelper(event)).withPayload(workflowInstance); }
@Override public Event toEvent() { return Event.runError(WorkflowInstance.parseKey(workflowInstance), message); } }
@Override public Event toEvent() { return Event.started(WorkflowInstance.parseKey(workflowInstance)); } }
@Override public Event toEvent() { return Event.retryAfter(WorkflowInstance.parseKey(workflowInstance), delayMillis); } }
@Override public Event toEvent() { return Event.info(WorkflowInstance.parseKey(workflowInstance), message); } }
@Override public Event toEvent() { return Event.submitted(WorkflowInstance.parseKey(workflowInstance), executionId); } }
@Override public Event toEvent() { return Event.submit(WorkflowInstance.parseKey(workflowInstance), executionDescription, executionId); } }
private void checkStatuses() { LOG.debug("Checking running statuses, {} statuses to check", inFlight.size()); for (String containerId : inFlight.keySet()) { final ContainerInfo containerInfo; try { containerInfo = client.inspectContainer(containerId); } catch (DockerException | InterruptedException e) { LOG.error("Error while reading status from docker", e); continue; } if (containerInfo.state().running() && !started.contains(containerId)) { final WorkflowInstance workflowInstance = inFlight.get(containerId); stateManager.receiveIgnoreClosed(Event.started(workflowInstance)); started.add(containerId); } if (!containerInfo.state().running()) { // Unlike in KubernetesDockerRunner case, where docker_termination_logging is supported, // here we are susceptible to Docker exit code bug, https://github.com/kubernetes/kubernetes/issues/41516. final Optional<Integer> exitCode = Optional.ofNullable(containerInfo.state().exitCode()); final WorkflowInstance workflowInstance = inFlight.remove(containerId); // trigger started event if we didn't see the container in running before if (!started.contains(containerId)) { stateManager.receiveIgnoreClosed(Event.started(workflowInstance)); } else { started.remove(containerId); } stateManager.receiveIgnoreClosed(Event.terminate(workflowInstance, exitCode)); } } }
private CompletionStage<Boolean> haltActiveBackfillInstance(WorkflowInstance workflowInstance, Client client) { try { final ByteString payload = serialize(Event.halt(workflowInstance)); final Request request = Request.forUri(schedulerApiUrl("events"), "POST") .withPayload(payload); return client.send(request) .thenApply(response -> response.status().family().equals(SUCCESSFUL)); } catch (JsonProcessingException e) { return CompletableFuture.completedFuture(false); } }
private static Optional<Event> isInErrorState(WorkflowInstance workflowInstance, Pod pod, Optional<ContainerStatus> mainContainerStatusOpt) { final PodStatus status = pod.getStatus(); final String phase = status.getPhase(); if ("NodeLost".equals(pod.getStatus().getReason())) { return Optional.of(Event.runError(workflowInstance, "Lost node running pod")); } switch (phase) { case "Pending": // check if one or more docker contains failed to pull their image, a possible silent error return mainContainerStatusOpt .filter(KubernetesPodEventTranslator::hasPullImageError) .map(x -> Event.runError(workflowInstance, "One or more containers failed to pull their image")); case "Succeeded": case "Failed": if (!mainContainerStatusOpt.isPresent()) { return Optional.of(Event.runError(workflowInstance, "Could not find our container in pod")); } final ContainerStatus containerStatus = mainContainerStatusOpt.get(); final ContainerStateTerminated terminated = containerStatus.getState().getTerminated(); if (terminated == null) { return Optional.of(Event.runError(workflowInstance, "Unexpected null terminated status")); } return Optional.empty(); case "Unknown": return Optional.of(Event.runError(workflowInstance, "Pod entered Unknown phase")); default: return Optional.empty(); } }
private static List<Event> handleStarted(WorkflowInstance workflowInstance, RunState state) { switch (state.state()) { case PREPARE: case SUBMITTED: return ImmutableList.of(Event.started(workflowInstance)); default: return ImmutableList.of(); } }