public static ExecutionOptions parseFlowOptions(final HttpServletRequest req) throws ServletException { final ExecutionOptions execOptions = new ExecutionOptions(); execOptions.setFailureAction(FailureAction.FINISH_CURRENTLY_RUNNING); } else if (option.equals("cancelImmediately")) { execOptions.setFailureAction(FailureAction.CANCEL_ALL); } else if (option.equals("finishPossible")) { execOptions.setFailureAction(FailureAction.FINISH_ALL_POSSIBLE); execOptions.setFailureEmailsOverridden(override); execOptions.setSuccessEmailsOverridden(override); if (!emails.isEmpty()) { final String[] emailSplit = emails.split("\\s*,\\s*|\\s*;\\s*|\\s+"); execOptions.setFailureEmails(Arrays.asList(emailSplit)); if (!emails.isEmpty()) { final String[] emailSplit = emails.split("\\s*,\\s*|\\s*;\\s*|\\s+"); execOptions.setSuccessEmails(Arrays.asList(emailSplit)); execOptions.setNotifyOnFirstFailure(Boolean.parseBoolean(getParam(req, "notifyFailureFirst"))); execOptions.setNotifyOnLastFailure(Boolean.parseBoolean(getParam(req, "notifyFailureLast"))); execOptions.setConcurrentOption(concurrentOption); if (concurrentOption.equals("pipeline")) {
ret.put("successEmails", options.getSuccessEmails()); ret.put("failureEmails", options.getFailureEmails()); ret.put("flowParam", options.getFlowParameters()); final FailureAction action = options.getFailureAction(); String failureAction = null; switch (action) { ret.put("notifyFailureFirst", options.getNotifyOnFirstFailure()); ret.put("notifyFailureLast", options.getNotifyOnLastFailure()); ret.put("failureEmailsOverride", options.isFailureEmailsOverridden()); ret.put("successEmailsOverride", options.isSuccessEmailsOverridden()); ret.put("concurrentOptions", options.getConcurrentOption()); ret.put("pipelineLevel", options.getPipelineLevel()); ret.put("pipelineExecution", options.getPipelineExecutionId()); ret.put("queueLevel", options.getQueueLevel()); ret.put("disabled", options.getDisabledJobs());
public void createAndSetScheduleOptions(final Object obj) { final HashMap<String, Object> schedObj = (HashMap<String, Object>) obj; if (schedObj.containsKey("executionOptions")) { final ExecutionOptions execOptions = ExecutionOptions.createFromObject(schedObj.get("executionOptions")); this.executionOptions = execOptions; } else if (schedObj.containsKey("flowOptions")) { final ExecutionOptions execOptions = ExecutionOptions.createFromObject(schedObj.get("flowOptions")); this.executionOptions = execOptions; execOptions.setConcurrentOption(ExecutionOptions.CONCURRENT_OPTION_SKIP); } else { this.executionOptions = new ExecutionOptions(); this.executionOptions .setConcurrentOption(ExecutionOptions.CONCURRENT_OPTION_SKIP); } if (schedObj.containsKey("slaOptions")) { final List<Object> slaOptionsObject = (List<Object>) schedObj.get("slaOptions"); final List<SlaOption> slaOptions = new ArrayList<>(); for (final Object slaObj : slaOptionsObject) { slaOptions.add(SlaOption.fromObject(slaObj)); } this.slaOptions = slaOptions; } }
@Override protected void setFlow(final Project project, final Flow flow) { super.setFlow(project, flow); this.executionOptions = new ExecutionOptions(); this.executionOptions.setMailCreator(flow.getMailCreator()); if (flow.getSuccessEmails() != null) { this.executionOptions.setSuccessEmails(flow.getSuccessEmails()); } if (flow.getFailureEmails() != null) { this.executionOptions.setFailureEmails(flow.getFailureEmails()); } }
exFlow.setDelayedExecution(1000); final ExecutionOptions options = new ExecutionOptions(); options.setConcurrentOption("blah"); options.setDisabledJobs(Arrays.asList(new Object[]{"bee", null, "boo"})); options.setFailureAction(FailureAction.CANCEL_ALL); options .setFailureEmails(Arrays.asList(new String[]{"doo", null, "daa"})); options .setSuccessEmails(Arrays.asList(new String[]{"dee", null, "dae"})); options.setPipelineLevel(2); options.setPipelineExecutionId(3); options.setNotifyOnFirstFailure(true); options.setNotifyOnLastFailure(true); options.addAllFlowParameters(flowProps); exFlow.setExecutionOptions(options);
options = new ExecutionOptions(); if (options.getDisabledJobs() != null) { FlowUtils.applyDisabledJobs(options.getDisabledJobs(), exflow); + " has more than " + this.maxConcurrentRunsOneFlow + " concurrent runs. Skipping", ExecutorManagerException.Reason.SkippedExecution); } else if (options.getConcurrentOption().equals( ExecutionOptions.CONCURRENT_OPTION_PIPELINE)) { Collections.sort(running); final Integer runningExecId = running.get(running.size() - 1); options.setPipelineExecutionId(runningExecId); message = "Flow " + flowId + " is already running with exec id " + runningExecId + ". Pipelining level " + options.getPipelineLevel() + ". \n"; } else if (options.getConcurrentOption().equals( ExecutionOptions.CONCURRENT_OPTION_SKIP)) { throw new ExecutorManagerException("Flow " + flowId !ProjectWhitelist.isProjectWhitelisted(exflow.getProjectId(), ProjectWhitelist.WhitelistType.MemoryCheck); options.setMemoryCheck(memoryCheck);
@Override public void doAction() throws Exception { if (projectManager == null || executorManagerAdapter == null) { throw new Exception("ExecuteFlowAction not properly initialized!"); } final Project project = FlowUtils.getProject(projectManager, this.projectId); final Flow flow = FlowUtils.getFlow(project, this.flowName); final ExecutableFlow exflow = FlowUtils.createExecutableFlow(project, flow); exflow.setSubmitUser(this.submitUser); if (this.executionOptions == null) { this.executionOptions = new ExecutionOptions(); } if (!this.executionOptions.isFailureEmailsOverridden()) { this.executionOptions.setFailureEmails(flow.getFailureEmails()); } if (!this.executionOptions.isSuccessEmailsOverridden()) { this.executionOptions.setSuccessEmails(flow.getSuccessEmails()); } exflow.setExecutionOptions(this.executionOptions); if (this.slaOptions != null && this.slaOptions.size() > 0) { exflow.setSlaOptions(this.slaOptions); } logger.info("Invoking flow " + project.getName() + "." + this.flowName); executorManagerAdapter.submitExecutableFlow(exflow, this.submitUser); logger.info("Invoked flow " + project.getName() + "." + this.flowName); }
new TypedMapWrapper<>(optionsMap); final ExecutionOptions options = new ExecutionOptions(); if (optionsMap.containsKey(FLOW_PARAMETERS)) { options.flowParameters = new HashMap<>(); options.setSuccessEmails(wrapper.<String>getList(SUCCESS_EMAILS, Collections.<String>emptyList())); options.setFailureEmails(wrapper.<String>getList(FAILURE_EMAILS, Collections.<String>emptyList())); options.setSuccessEmailsOverridden(wrapper.getBool(SUCCESS_EMAILS_OVERRIDE, false)); options.setFailureEmailsOverridden(wrapper.getBool(FAILURE_EMAILS_OVERRIDE, false)); options.setMemoryCheck(wrapper.getBool(MEMORY_CHECK, true));
if (!options.isFailureEmailsOverridden()) { options.setFailureEmails(flow.getFailureEmails()); if (!options.isSuccessEmailsOverridden()) { options.setSuccessEmails(flow.getSuccessEmails()); options.setMailCreator(flow.getMailCreator());
final Alerter mailAlerter = alerterHolder.get("email"); if (flow.getStatus() != Status.SUCCEEDED) { if (options.getFailureEmails() != null && !options.getFailureEmails().isEmpty()) { try { mailAlerter.alertOnError(flow, extraReasons); if (options.getFlowParameters().containsKey("alert.type")) { final String alertType = options.getFlowParameters().get("alert.type"); final Alerter alerter = alerterHolder.get(alertType); if (alerter != null) { if (options.getSuccessEmails() != null && !options.getSuccessEmails().isEmpty()) { try { mailAlerter.alertOnSuccess(flow); if (options.getFlowParameters().containsKey("alert.type")) { final String alertType = options.getFlowParameters().get("alert.type"); final Alerter alerter = alerterHolder.get(alertType); if (alerter != null) {
@Before public void setUp() throws Exception { this.defaultTz = TimeZone.getDefault(); assertNotNull(this.defaultTz); // EEST TimeZone.setDefault(TimeZone.getTimeZone("Europe/Helsinki")); DateTimeUtils.setCurrentMillisFixed(FIXED_CURRENT_TIME_MILLIS); this.mailCreator = new DefaultMailCreator(); this.executor = new Executor(1, "executor1-host", 1234, true); this.flow = new Flow("mail-creator-test"); this.project = new Project(1, "test-project"); this.options = new ExecutionOptions(); this.message = new EmailMessage("localhost", EmailMessageCreator.DEFAULT_SMTP_PORT, "", "", null); this.azkabanName = "unit-tests"; this.scheme = "http"; this.clientHostname = "localhost"; this.clientPortNumber = "8081"; final Node failedNode = new Node("test-job"); failedNode.setType("noop"); this.flow.addNode(failedNode); this.executableFlow = new ExecutableFlow(this.project, this.flow); this.executableFlow.setExecutionOptions(this.options); this.executableFlow.setStartTime(START_TIME_MILLIS); this.options.setFailureEmails(ImmutableList.of("test@example.com")); this.options.setSuccessEmails(ImmutableList.of("test@example.com")); }
private int getPriority(final ExecutableFlow exflow) { final ExecutionOptions options = exflow.getExecutionOptions(); int priority = ExecutionOptions.DEFAULT_FLOW_PRIORITY; if (options != null && options.getFlowParameters() != null && options.getFlowParameters() .containsKey(ExecutionOptions.FLOW_PRIORITY)) { try { priority = Integer.valueOf(options.getFlowParameters().get( ExecutionOptions.FLOW_PRIORITY)); } catch (final NumberFormatException ex) { priority = ExecutionOptions.DEFAULT_FLOW_PRIORITY; logger.error( "Failed to parse flow priority for exec_id = " + exflow.getExecutionId(), ex); } } return priority; } }
final AlerterHolder alerterHolder) { final ExecutionOptions options = flow.getExecutionOptions(); if (options.getNotifyOnFirstFailure()) { logger.info("Alert on first error of execution " + flow.getExecutionId()); final Alerter mailAlerter = alerterHolder.get("email"); if (options.getFlowParameters().containsKey("alert.type")) { final String alertType = options.getFlowParameters().get("alert.type"); final Alerter alerter = alerterHolder.get(alertType); if (alerter != null) {
final List<String> emailList = option.getFailureEmails(); final int execId = flow.getExecutionId(); + flow.getProjectName() + "' has encountered a failure on " + azkabanName + "</h2>"); if (option.getFailureAction() == FailureAction.CANCEL_ALL) { message .println("This flow is set to cancel all currently running jobs."); } else if (option.getFailureAction() == FailureAction.FINISH_ALL_POSSIBLE) { message .println("This flow is set to complete all jobs that aren't blocked by the failure.");
if (flowOptions.getFailureEmails() != null) { final List<String> emails = flowOptions.getFailureEmails(); if (emails.size() > 0) { ret.put("slaEmails", emails);
final Schedule schedule = this.scheduleManager.getSchedule(project.getId(), flowName); final ExecutionOptions executionOptions = schedule != null ? schedule.getExecutionOptions() : new ExecutionOptions(); final Flow flow = project.getFlow(flowName); if (flow == null) { ret.put("projectId", exFlow.getProjectId()); ret.put("project", project.getName()); FlowUtils.applyDisabledJobs(executionOptions.getDisabledJobs(), exFlow); final Map<String, Object> flowObj = getExecutableNodeInfo(exFlow); ret.putAll(flowObj);
/** * Constructor. * If executorService is null, then it will create it's own for thread pools. * * @param flow * @param executorLoader * @param projectLoader * @param jobtypeManager * @param executorService * @throws ExecutorManagerException */ public FlowRunner(ExecutableFlow flow, ExecutorLoader executorLoader, ProjectLoader projectLoader, JobTypeManager jobtypeManager, ExecutorService executorService) throws ExecutorManagerException { this.execId = flow.getExecutionId(); this.flow = flow; this.executorLoader = executorLoader; this.projectLoader = projectLoader; this.execDir = new File(flow.getExecutionPath()); this.jobtypeManager = jobtypeManager; ExecutionOptions options = flow.getExecutionOptions(); this.pipelineLevel = options.getPipelineLevel(); this.pipelineExecId = options.getPipelineExecutionId(); this.failureAction = options.getFailureAction(); this.proxyUsers = flow.getProxyUsers(); this.executorService = executorService; this.finishedNodes = new SwapQueue<ExecutableNode>(); }
/** * Sends as many emails as there are unique combinations of: * * [mail creator] x [failure email address list] * * Executions with the same combo are grouped into a single message. */ @Override public void alertOnFailedUpdate(final Executor executor, List<ExecutableFlow> flows, final ExecutorManagerException updateException) { flows = flows.stream() .filter(flow -> flow.getExecutionOptions() != null) .filter(flow -> CollectionUtils.isNotEmpty(flow.getExecutionOptions().getFailureEmails())) .collect(Collectors.toList()); // group by mail creator in case some flows use different creators final ImmutableListMultimap<String, ExecutableFlow> creatorsToFlows = Multimaps .index(flows, flow -> flow.getExecutionOptions().getMailCreator()); for (final String mailCreatorName : creatorsToFlows.keySet()) { final ImmutableList<ExecutableFlow> creatorFlows = creatorsToFlows.get(mailCreatorName); final MailCreator mailCreator = getMailCreator(mailCreatorName); // group by recipients in case some flows have different failure email addresses final ImmutableListMultimap<List<String>, ExecutableFlow> emailsToFlows = Multimaps .index(creatorFlows, flow -> flow.getExecutionOptions().getFailureEmails()); for (final List<String> emailList : emailsToFlows.keySet()) { sendFailedUpdateEmail(executor, updateException, mailCreator, emailsToFlows.get(emailList)); } } }
@Ignore @Test public void jsonConversionTest() throws Exception { final ActionTypeLoader loader = new ActionTypeLoader(); loader.init(new Props()); final ExecutionOptions options = new ExecutionOptions(); final List<Object> disabledJobs = new ArrayList<>(); options.setDisabledJobs(disabledJobs); final ExecuteFlowAction executeFlowAction = new ExecuteFlowAction("ExecuteFlowAction", 1, "testproject", "testflow", "azkaban", options, null); final Object obj = executeFlowAction.toJson(); final ExecuteFlowAction action = (ExecuteFlowAction) loader.createActionFromJson(ExecuteFlowAction.type, obj); assertTrue(executeFlowAction.getProjectId() == action.getProjectId()); assertTrue(executeFlowAction.getFlowName().equals(action.getFlowName())); assertTrue(executeFlowAction.getSubmitUser().equals(action.getSubmitUser())); }
private List<TriggerAction> getTriggerActions() { final List<TriggerAction> actions = new ArrayList<>(); final TriggerAction act = new ExecuteFlowAction("fuu", 1, "test-project", "test-flow", "test-user", new ExecutionOptions(), Collections.emptyList()); actions.add(act); return actions; }