@Override public void doAction() throws Exception { logger.info("Alerting on sla failure."); final Map<String, Object> alert = this.slaOption.getInfo(); if (alert.containsKey(SlaOption.ALERT_TYPE)) { final String alertType = (String) alert.get(SlaOption.ALERT_TYPE); final Alerter alerter = this.alerters.get(alertType); if (alerter != null) { try { final ExecutableFlow flow = this.executorLoader.fetchExecutableFlow(this.execId); alerter.alertOnSla(this.slaOption, SlaOption.createSlaMessage(this.slaOption, flow)); } catch (final Exception e) { e.printStackTrace(); logger.error("Failed to alert by " + alertType); } } else { logger.error("Alerter type " + alertType + " doesn't exist. Failed to alert."); } } }
@Inject public AlerterHolder(final Props props, final Emailer mailAlerter) { try { this.alerters = loadAlerters(props, mailAlerter); } catch (final Exception ex) { logger.error(ex); this.alerters = new HashMap<>(); } }
private Map<String, Alerter> loadAlerters(final Props props, final Emailer mailAlerter) { final Map<String, Alerter> allAlerters = new HashMap<>(); // load built-in alerters allAlerters.put("email", mailAlerter); // load all plugin alerters final String pluginDir = props.getString("alerter.plugin.dir", "plugins/alerter"); allAlerters.putAll(loadPluginAlerters(pluginDir)); return allAlerters; }
/** * Increments executor failure count. If it reaches max failure count, sends alert emails to AZ * admin. * * @param entry executor to list of flows map entry * @param executor the executor * @param e Exception thrown when the executor is not alive */ private void handleExecutorNotAliveCase( final Entry<Optional<Executor>, List<ExecutableFlow>> entry, final Executor executor, final ExecutorManagerException e) { logger.error("Failed to get update from executor " + executor.getId(), e); this.executorFailureCount.put(executor.getId(), this.executorFailureCount.getOrDefault (executor.getId(), 0) + 1); if (this.executorFailureCount.get(executor.getId()) % this.executorMaxFailureCount == 0 && !this.alertEmails.isEmpty()) { entry.getValue().stream().forEach(flow -> flow .getExecutionOptions().setFailureEmails(this.alertEmails)); logger.info(String.format("Executor failure count is %d. Sending alert emails to %s.", this.executorFailureCount.get(executor.getId()), this.alertEmails)); this.alerterHolder.get("email").alertOnFailedUpdate(executor, entry.getValue(), e); } } }
private Map<String, Alerter> loadAlerters(final Props props, final Emailer mailAlerter) { final Map<String, Alerter> allAlerters = new HashMap<>(); // load built-in alerters allAlerters.put("email", mailAlerter); // load all plugin alerters final String pluginDir = props.getString("alerter.plugin.dir", "plugins/alerter"); allAlerters.putAll(loadPluginAlerters(pluginDir)); return allAlerters; }
@Inject public AlerterHolder(final Props props, final Emailer mailAlerter) { try { this.alerters = loadAlerters(props, mailAlerter); } catch (final Exception ex) { logger.error(ex); this.alerters = new HashMap<>(); } }
alerterHolder, final String[] extraReasons) { final ExecutionOptions options = flow.getExecutionOptions(); final Alerter mailAlerter = alerterHolder.get("email"); if (flow.getStatus() != Status.SUCCEEDED) { if (options.getFailureEmails() != null && !options.getFailureEmails().isEmpty()) { final Alerter alerter = alerterHolder.get(alertType); if (alerter != null) { try { final Alerter alerter = alerterHolder.get(alertType); if (alerter != null) { try {
@Before public void setup() { this.props = new Props(); this.mailAlerter = mock(Alerter.class); this.alertHolder = mock(AlerterHolder.class); when(this.alertHolder.get("email")).thenReturn(this.mailAlerter); this.loader = new MockExecutorLoader(); this.runningExecutions = new RunningExecutions(); this.updaterStage = new ExecutorManagerUpdaterStage(); }
final Alerter mailAlerter = this.alerterHolder.get("email"); mailAlerter.alertOnFailedUpdate(executor, entry.getValue(), e);
@Before public void setUp() throws Exception { this.props = new Props(); this.props.put(ConfigurationKeys.AZKABAN_EXECUTOR_MAX_FAILURE_COUNT, 2); this.props.put(ConfigurationKeys.AZKABAN_ADMIN_ALERT_EMAIL, AZ_ADMIN_ALERT_EMAIL); this.loader = mock(ExecutorLoader.class); this.mailAlerter = mock(Alerter.class); this.alerterHolder = mock(AlerterHolder.class); this.apiGateway = mock(ExecutorApiGateway.class); this.executorHealthChecker = new ExecutorHealthChecker(this.props, this.loader, this .apiGateway, this.alerterHolder); this.flow1 = TestUtils.createTestExecutableFlow("exectest1", "exec1"); this.flow1.setExecutionId(EXECUTION_ID_11); this.flow1.setStatus(Status.RUNNING); this.executor1 = new Executor(1, "localhost", 12345, true); when(this.loader.fetchActiveFlows()).thenReturn(this.activeFlows); when(this.alerterHolder.get("email")).thenReturn(this.mailAlerter); }
@Before public void setUp() throws Exception { MockitoAnnotations.initMocks(this); this.execution = new ExecutableFlow(); this.execution.setExecutionId(EXECUTION_ID_77); this.activeExecutor = new Executor(1, "activeExecutor-1", 9999, true); this.runningExecutions = new RunningExecutions(); this.runningExecutions.get().put(EXECUTION_ID_77, new Pair<>( new ExecutionReference(EXECUTION_ID_77, this.activeExecutor), this.execution)); this.updater = new RunningExecutionsUpdater(this.updaterStage, this.alerterHolder, this.commonMetrics, this.apiGateway, this.runningExecutions, this.executionFinalizer, this.executorLoader); when(this.alerterHolder.get("email")).thenReturn(this.mailAlerter); }
verify((this.alerterHolder).get("email")) .alertOnFailedUpdate(eq(this.executor1), eq(Arrays.asList(this.flow1)), any(ExecutorManagerException.class));
@Override public void doAction() throws Exception { logger.info("Alerting on sla failure."); final Map<String, Object> alert = this.slaOption.getInfo(); if (alert.containsKey(SlaOption.ALERT_TYPE)) { final String alertType = (String) alert.get(SlaOption.ALERT_TYPE); final Alerter alerter = this.alerters.get(alertType); if (alerter != null) { try { final ExecutableFlow flow = this.executorLoader.fetchExecutableFlow(this.execId); alerter.alertOnSla(this.slaOption, SlaOption.createSlaMessage(this.slaOption, flow)); } catch (final Exception e) { e.printStackTrace(); logger.error("Failed to alert by " + alertType); } } else { logger.error("Alerter type " + alertType + " doesn't exist. Failed to alert."); } } }
final Alerter mailAlerter = ExecutorManager.this.alerterHolder.get("email"); mailAlerter.alertOnFailedUpdate(executor, entry.getValue(), e);
final ExecutionOptions options = flow.getExecutionOptions(); final Alerter mailAlerter = this.alerterHolder.get("email"); if (flow.getStatus() == Status.FAILED || flow.getStatus() == Status.KILLED) { if (options.getFailureEmails() != null && !options.getFailureEmails().isEmpty()) { final Alerter alerter = this.alerterHolder.get(alertType); if (alerter != null) { try { final Alerter alerter = this.alerterHolder.get(alertType); if (alerter != null) { try {