private MailCreator getMailCreator(final ExecutableFlow flow) { final String name = flow.getExecutionOptions().getMailCreator(); return getMailCreator(name); }
/** * Sends as many emails as there are unique combinations of: * * [mail creator] x [failure email address list] * * Executions with the same combo are grouped into a single message. */ @Override public void alertOnFailedUpdate(final Executor executor, List<ExecutableFlow> flows, final ExecutorManagerException updateException) { flows = flows.stream() .filter(flow -> flow.getExecutionOptions() != null) .filter(flow -> CollectionUtils.isNotEmpty(flow.getExecutionOptions().getFailureEmails())) .collect(Collectors.toList()); // group by mail creator in case some flows use different creators final ImmutableListMultimap<String, ExecutableFlow> creatorsToFlows = Multimaps .index(flows, flow -> flow.getExecutionOptions().getMailCreator()); for (final String mailCreatorName : creatorsToFlows.keySet()) { final ImmutableList<ExecutableFlow> creatorFlows = creatorsToFlows.get(mailCreatorName); final MailCreator mailCreator = getMailCreator(mailCreatorName); // group by recipients in case some flows have different failure email addresses final ImmutableListMultimap<List<String>, ExecutableFlow> emailsToFlows = Multimaps .index(creatorFlows, flow -> flow.getExecutionOptions().getFailureEmails()); for (final List<String> emailList : emailsToFlows.keySet()) { sendFailedUpdateEmail(executor, updateException, mailCreator, emailsToFlows.get(emailList)); } } }
private int getPriority(final ExecutableFlow exflow) { final ExecutionOptions options = exflow.getExecutionOptions(); int priority = ExecutionOptions.DEFAULT_FLOW_PRIORITY; if (options != null && options.getFlowParameters() != null && options.getFlowParameters() .containsKey(ExecutionOptions.FLOW_PRIORITY)) { try { priority = Integer.valueOf(options.getFlowParameters().get( ExecutionOptions.FLOW_PRIORITY)); } catch (final NumberFormatException ex) { priority = ExecutionOptions.DEFAULT_FLOW_PRIORITY; logger.error( "Failed to parse flow priority for exec_id = " + exflow.getExecutionId(), ex); } } return priority; } }
/** * Increments executor failure count. If it reaches max failure count, sends alert emails to AZ * admin. * * @param entry executor to list of flows map entry * @param executor the executor * @param e Exception thrown when the executor is not alive */ private void handleExecutorNotAliveCase( final Entry<Optional<Executor>, List<ExecutableFlow>> entry, final Executor executor, final ExecutorManagerException e) { logger.error("Failed to get update from executor " + executor.getId(), e); this.executorFailureCount.put(executor.getId(), this.executorFailureCount.getOrDefault (executor.getId(), 0) + 1); if (this.executorFailureCount.get(executor.getId()) % this.executorMaxFailureCount == 0 && !this.alertEmails.isEmpty()) { entry.getValue().stream().forEach(flow -> flow .getExecutionOptions().setFailureEmails(this.alertEmails)); logger.info(String.format("Executor failure count is %d. Sending alert emails to %s.", this.executorFailureCount.get(executor.getId()), this.alertEmails)); this.alerterHolder.get("email").alertOnFailedUpdate(executor, entry.getValue(), e); } } }
public static ExecutableFlow createExecutableFlow() throws IOException { final ExecutableFlow flow = TestUtils.createTestExecutableFlow("exectest1", "exec1"); flow.getExecutionOptions().getFlowParameters() .put(ExecutionOptions.FLOW_PRIORITY, "1"); flow.getExecutionOptions().getFlowParameters() .put(ExecutionOptions.USE_EXECUTOR, "2"); return flow; }
private Executor selectExecutor(final ExecutableFlow exflow, final Set<Executor> availableExecutors) { Executor choosenExecutor = getUserSpecifiedExecutor(exflow.getExecutionOptions(), exflow.getExecutionId()); // If no executor was specified by admin if (choosenExecutor == null) { ExecutorManager.logger.info("Using dispatcher for execution id :" + exflow.getExecutionId()); final ExecutorSelector selector = new ExecutorSelector(ExecutorManager.this.filterList, ExecutorManager.this.comparatorWeightsMap); choosenExecutor = selector.getBest(availableExecutors, exflow); } return choosenExecutor; }
final ExecutionOptions options = flow.getExecutionOptions(); if (options.getNotifyOnFirstFailure()) { logger.info("Alert on first error of execution " + flow.getExecutionId());
final ExecutionOptions options = exflow.getExecutionOptions();
private ExecutableFlow createExecutableFlow(final String flowName, final int priority, final long updateTime, final int executionId) throws IOException { final ExecutableFlow execFlow = TestUtils.createTestExecutableFlow("exectest1", flowName); execFlow.setUpdateTime(updateTime); execFlow.setExecutionId(executionId); if (priority > 0) { execFlow.getExecutionOptions().getFlowParameters() .put(ExecutionOptions.FLOW_PRIORITY, String.valueOf(priority)); } return execFlow; }
@Test public void TestFilterNonAdminOnlyFlowParams() throws IOException, ExecutorManagerException, UserManagerException { final ExecutableFlow flow = createExecutableFlow(); final UserManager manager = TestUtils.createTestXmlUserManager(); final User user = manager.getUser("testUser", "testUser"); HttpRequestUtils.filterAdminOnlyFlowParams(manager, flow.getExecutionOptions(), user); Assert.assertFalse(flow.getExecutionOptions().getFlowParameters() .containsKey(ExecutionOptions.FLOW_PRIORITY)); Assert.assertFalse(flow.getExecutionOptions().getFlowParameters() .containsKey(ExecutionOptions.USE_EXECUTOR)); }
@Test public void TestFilterAdminOnlyFlowParams() throws IOException, ExecutorManagerException, UserManagerException { final ExecutableFlow flow = createExecutableFlow(); final UserManager manager = TestUtils.createTestXmlUserManager(); final User user = manager.getUser("testAdmin", "testAdmin"); HttpRequestUtils.filterAdminOnlyFlowParams(manager, flow.getExecutionOptions(), user); Assert.assertTrue(flow.getExecutionOptions().getFlowParameters() .containsKey(ExecutionOptions.FLOW_PRIORITY)); Assert.assertTrue(flow.getExecutionOptions().getFlowParameters() .containsKey(ExecutionOptions.USE_EXECUTOR)); }
private void assertTwoFlowSame(final ExecutableFlow flow1, final ExecutableFlow flow2, final boolean compareFlowData) { assertThat(flow1.getExecutionId()).isEqualTo(flow2.getExecutionId()); assertThat(flow1.getStatus()).isEqualTo(flow2.getStatus()); assertThat(flow1.getEndTime()).isEqualTo(flow2.getEndTime()); assertThat(flow1.getStartTime()).isEqualTo(flow2.getStartTime()); assertThat(flow1.getSubmitTime()).isEqualTo(flow2.getSubmitTime()); assertThat(flow1.getFlowId()).isEqualTo(flow2.getFlowId()); assertThat(flow1.getProjectId()).isEqualTo(flow2.getProjectId()); assertThat(flow1.getVersion()).isEqualTo(flow2.getVersion()); assertThat(flow1.getSubmitUser()).isEqualTo(flow2.getSubmitUser()); if (compareFlowData) { assertThat(flow1.getExecutionOptions().getFailureAction()) .isEqualTo(flow2.getExecutionOptions().getFailureAction()); assertThat(new HashSet<>(flow1.getEndNodes())).isEqualTo(new HashSet<>(flow2.getEndNodes())); } }
@Test public void testSubmitFlowsWithSkipOption() throws Exception { submitFlow(this.flow2, this.ref2); this.flow3.getExecutionOptions().setConcurrentOption(ExecutionOptions.CONCURRENT_OPTION_SKIP); assertThatThrownBy( () -> this.controller.submitExecutableFlow(this.flow3, this.user.getUserId())) .isInstanceOf(ExecutorManagerException.class).hasMessageContaining( "Flow " + this.flow3.getId() + " is already running. Skipping execution."); }
/** * ExecutorManager should try to dispatch to all executors & when both fail it should remove the * execution from queue and finalize it. */ @Ignore @Test public void testDispatchFailed() throws Exception { testSetUpForRunningFlows(); this.manager.start(); final ExecutableFlow flow1 = TestUtils.createTestExecutableFlow("exectest1", "exec1"); flow1.getExecutionOptions().setFailureEmails(Arrays.asList("test@example.com")); when(this.loader.fetchExecutableFlow(-1)).thenReturn(flow1); when(this.apiGateway.callWithExecutable(any(), any(), eq(ConnectorParams.EXECUTE_ACTION))) .thenThrow(new ExecutorManagerException("Mocked dispatch exception")); this.manager.submitExecutableFlow(flow1, this.user.getUserId()); waitFlowFinished(flow1); verify(this.apiGateway) .callWithExecutable(flow1, this.manager.fetchExecutor(1), ConnectorParams.EXECUTE_ACTION); verify(this.apiGateway) .callWithExecutable(flow1, this.manager.fetchExecutor(2), ConnectorParams.EXECUTE_ACTION); verify(this.loader, Mockito.times(2)).unassignExecutor(-1); verify(this.mailAlerter).alertOnError(eq(flow1), eq("Failed to dispatch queued execution derived-member-data because reached " + "azkaban.maxDispatchingErrors (tried 2 executors)"), contains("Mocked dispatch exception")); }
/** * ExecutorManager should try to dispatch to all executors until it succeeds. */ @Test public void testDispatchMultipleRetries() throws Exception { this.props.put(Constants.ConfigurationKeys.MAX_DISPATCHING_ERRORS_PERMITTED, 4); testSetUpForRunningFlows(); this.manager.start(); final ExecutableFlow flow1 = TestUtils.createTestExecutableFlow("exectest1", "exec1"); flow1.getExecutionOptions().setFailureEmails(Arrays.asList("test@example.com")); when(this.loader.fetchExecutableFlow(-1)).thenReturn(flow1); // fail 2 first dispatch attempts, then succeed when(this.apiGateway.callWithExecutable(any(), any(), eq(ConnectorParams.EXECUTE_ACTION))) .thenThrow(new ExecutorManagerException("Mocked dispatch exception 1")) .thenThrow(new ExecutorManagerException("Mocked dispatch exception 2")) .thenReturn(null); // this is just to clean up the execution as FAILED after it has been submitted mockFlowDoesNotExist(); this.manager.submitExecutableFlow(flow1, this.user.getUserId()); waitFlowFinished(flow1); // it's random which executor is chosen each time, but both should have been tried at least once verify(this.apiGateway, Mockito.atLeast(1)) .callWithExecutable(flow1, this.manager.fetchExecutor(1), ConnectorParams.EXECUTE_ACTION); verify(this.apiGateway, Mockito.atLeast(1)) .callWithExecutable(flow1, this.manager.fetchExecutor(2), ConnectorParams.EXECUTE_ACTION); // verify that there was a 3rd (successful) dispatch call verify(this.apiGateway, Mockito.times(3)) .callWithExecutable(eq(flow1), any(), eq(ConnectorParams.EXECUTE_ACTION)); verify(this.loader, Mockito.times(2)).unassignExecutor(-1); }
.alertOnFailedUpdate(eq(this.executor1), eq(Arrays.asList(this.flow1)), any(ExecutorManagerException.class)); assertThat(this.flow1.getExecutionOptions().getFailureEmails()).isEqualTo (Arrays.asList(AZ_ADMIN_ALERT_EMAIL.split(",")));
final String scheme, final String clientHostname, final String clientPortNumber) { final ExecutionOptions option = flows.get(0).getExecutionOptions(); final List<String> emailList = option.getFailureEmails();
@Test(expected = ExecutorManagerException.class) public void testDuplicateQueuedFlows() throws Exception { final ExecutorManager manager = createMultiExecutorManagerInstance(); final ExecutableFlow flow1 = TestUtils.createTestExecutableFlow("exectest1", "exec1"); flow1.getExecutionOptions().setConcurrentOption( ExecutionOptions.CONCURRENT_OPTION_SKIP); final User testUser = TestUtils.getTestUser(); manager.submitExecutableFlow(flow1, testUser.getUserId()); manager.submitExecutableFlow(flow1, testUser.getUserId()); }
final String clientPortNumber) { final ExecutionOptions option = flow.getExecutionOptions(); final List<String> emailList = option.getSuccessEmails();
final String clientHostname, final String clientPortNumber) { final ExecutionOptions option = flow.getExecutionOptions(); final List<String> emailList = option.getFailureEmails(); final int execId = flow.getExecutionId();