private Map<Integer, Executor> getActiveExecutors() { final Map<Integer, Executor> executorMap = new HashMap<>(); try { final List<Executor> executors = this.executorLoader.fetchActiveExecutors(); for (final Executor executor : executors) { executorMap.put(executor.getId(), executor); } } catch (final ExecutorManagerException e) { log.error("Fetching executors failed!", e); } return executorMap; }
@Override public Executor fetchExecutor(final int executorId) throws ExecutorManagerException { for (final Executor executor : this.executors) { if (executor.getId() == executorId) { return executor; } } return null; }
public List<ExecutorLogEvent> getExecutorEvents(final Executor executor, final int num, final int offset) throws ExecutorManagerException { try { return this.dbOperator.query(ExecutorLogsResultHandler.SELECT_EXECUTOR_EVENTS_ORDER, new ExecutorLogsResultHandler(), executor.getId(), num, offset); } catch (final SQLException e) { throw new ExecutorManagerException( "Failed to fetch events for executor id : " + executor.getId(), e); } }
/** * Increments executor failure count. If it reaches max failure count, sends alert emails to AZ * admin. * * @param entry executor to list of flows map entry * @param executor the executor * @param e Exception thrown when the executor is not alive */ private void handleExecutorNotAliveCase( final Entry<Optional<Executor>, List<ExecutableFlow>> entry, final Executor executor, final ExecutorManagerException e) { logger.error("Failed to get update from executor " + executor.getId(), e); this.executorFailureCount.put(executor.getId(), this.executorFailureCount.getOrDefault (executor.getId(), 0) + 1); if (this.executorFailureCount.get(executor.getId()) % this.executorMaxFailureCount == 0 && !this.alertEmails.isEmpty()) { entry.getValue().stream().forEach(flow -> flow .getExecutionOptions().setFailureEmails(this.alertEmails)); logger.info(String.format("Executor failure count is %d. Sending alert emails to %s.", this.executorFailureCount.get(executor.getId()), this.alertEmails)); this.alerterHolder.get("email").alertOnFailedUpdate(executor, entry.getValue(), e); } } }
@Override public List<ExecutorLogEvent> getExecutorEvents(final Executor executor, final int num, final int skip) throws ExecutorManagerException { if (!this.executorEvents.containsKey(executor.getId())) { final List<ExecutorLogEvent> events = this.executorEvents.get(executor.getId()); return events.subList(skip, Math.min(num + skip - 1, events.size() - 1)); } return null; }
@Override public void postExecutorEvent(final Executor executor, final EventType type, final String user, final String message) throws ExecutorManagerException { final ExecutorLogEvent event = new ExecutorLogEvent(executor.getId(), user, new Date(), type, message); if (!this.executorEvents.containsKey(executor.getId())) { this.executorEvents.put(executor.getId(), new ArrayList<>()); } this.executorEvents.get(executor.getId()).add(event); }
public void updateExecutor(final Executor executor) throws ExecutorManagerException { final String UPDATE = "UPDATE executors SET host=?, port=?, active=? where id=?"; try { final int rows = this.dbOperator.update(UPDATE, executor.getHost(), executor.getPort(), executor.isActive(), executor.getId()); if (rows == 0) { throw new ExecutorManagerException("No executor with id :" + executor.getId()); } } catch (final SQLException e) { throw new ExecutorManagerException("Error inactivating executor " + executor.getId(), e); } }
/** * {@inheritDoc} * * @see azkaban.executor.ExecutorManagerAdapter#fetchExecutor(int) */ @Override public Executor fetchExecutor(final int executorId) throws ExecutorManagerException { for (final Executor executor : this.activeExecutors.getAll()) { if (executor.getId() == executorId) { return executor; } } return this.executorLoader.fetchExecutor(executorId); }
public void postExecutorEvent(final Executor executor, final EventType type, final String user, final String message) throws ExecutorManagerException { final String INSERT_PROJECT_EVENTS = "INSERT INTO executor_events (executor_id, event_type, event_time, username, message) values (?,?,?,?,?)"; try { this.dbOperator.update(INSERT_PROJECT_EVENTS, executor.getId(), type.getNumVal(), new Date(), user, message); } catch (final SQLException e) { throw new ExecutorManagerException("Failed to post executor event", e); } }
@Override public void updateExecutor(final Executor executor) throws ExecutorManagerException { final Executor oldExecutor = fetchExecutor(executor.getId()); this.executors.remove(oldExecutor); this.executors.add(executor); }
/** * @throws ExecutorManagerException * */ private void handleStatePageLoad(final HttpServletRequest req, final HttpServletResponse resp, final Session session) throws ServletException { final Page page = newPage(req, resp, session, "azkaban/webapp/servlet/velocity/statsPage.vm"); try { final Collection<Executor> executors = this.execManagerAdapter.getAllActiveExecutors(); page.add("executorList", executors); if (executors.isEmpty()) { throw new ExecutorManagerException("Executor list is empty."); } final Map<String, Object> result = this.execManagerAdapter.callExecutorStats(executors.iterator().next().getId(), ConnectorParams.STATS_GET_ALLMETRICSNAME, (Pair<String, String>[]) null); if (result.containsKey(ConnectorParams.RESPONSE_ERROR)) { page.add("errorMsg", result.get(ConnectorParams.RESPONSE_ERROR) .toString()); } else { page.add("metricList", result.get("data")); } } catch (final Exception e) { logger.error(e.getMessage(), e); page.add("errorMsg", "Failed to get a response from Azkaban exec server"); } page.render(); }
@Test public void testSingleExecutorFetchById() throws Exception { final List<Executor> executors = addTestExecutors(); for (final Executor executor : executors) { final Executor fetchedExecutor = this.executorDao.fetchExecutor(executor.getId()); assertThat(executor).isEqualTo(fetchedExecutor); } }
@Test public void testAssignExecutorInvalidExecution() throws Exception { final String host = "localhost"; final int port = 12345; final Executor executor = this.executorDao.addExecutor(host, port); // Make 99 a random non-existent execution id. assertThatThrownBy( () -> this.assignExecutor.assignExecutor(executor.getId(), 99)) .isInstanceOf(ExecutorManagerException.class) .hasMessageContaining("non-existent execution"); }
logger.error("Failed to get update from executor " + executor.getHost(), e); boolean sendUnresponsiveEmail = false; final boolean executorRemoved = isExecutorRemoved(executor.getId()); for (final ExecutableFlow flow : entry.getValue()) { final Pair<ExecutionReference, ExecutableFlow> pair =
@Test public void testExecutorActivation() throws Exception { final Executor executor = this.executorDao.addExecutor("localhost1", 12345); assertThat(executor.isActive()).isFalse(); executor.setActive(true); this.executorDao.updateExecutor(executor); final Executor fetchedExecutor = this.executorDao.fetchExecutor(executor.getId()); assertThat(fetchedExecutor.isActive()).isTrue(); } }
private ExecutableFlow createExecutionAndAssign(final Status status, final Executor executor) throws Exception { final ExecutableFlow flow = createExecution(status); this.assignExecutor.assignExecutor(executor.getId(), flow.getExecutionId()); return flow; }
exflow.setUpdateTime(System.currentTimeMillis()); this.executorLoader.assignExecutor(choosenExecutor.getId(), exflow.getExecutionId()); try {
@Test public void testSelectAndUpdateExecution() throws Exception { final ExecutableFlow flow = TestUtils.createTestExecutableFlow("exectest1", "exec1"); flow.setExecutionId(1); this.executionFlowDao.uploadExecutableFlow(flow); final Executor executor = this.executorDao.addExecutor("localhost", 12345); assertThat(this.executionFlowDao.selectAndUpdateExecution(executor.getId())).isEqualTo(flow .getExecutionId()); assertThat(this.executorDao.fetchExecutorByExecutionId(flow.getExecutionId())).isEqualTo (executor); }
@Test public void testFetchActiveFlowsStatusChanged() throws Exception { final ExecutableFlow flow1 = TestUtils.createTestExecutableFlow("exectest1", "exec1"); this.executionFlowDao.uploadExecutableFlow(flow1); final Executor executor = this.executorDao.addExecutor("test", 1); this.assignExecutor.assignExecutor(executor.getId(), flow1.getExecutionId()); Map<Integer, Pair<ExecutionReference, ExecutableFlow>> activeFlows1 = this.fetchActiveFlowDao.fetchActiveFlows(); assertThat(activeFlows1.containsKey(flow1.getExecutionId())).isTrue(); // When flow status becomes SUCCEEDED/KILLED/FAILED, it should not be in active state flow1.setStatus(Status.SUCCEEDED); this.executionFlowDao.updateExecutableFlow(flow1); activeFlows1 = this.fetchActiveFlowDao.fetchActiveFlows(); assertThat(activeFlows1.containsKey(flow1.getExecutionId())).isFalse(); flow1.setStatus(Status.KILLED); this.executionFlowDao.updateExecutableFlow(flow1); activeFlows1 = this.fetchActiveFlowDao.fetchActiveFlows(); assertThat(activeFlows1.containsKey(flow1.getExecutionId())).isFalse(); flow1.setStatus(Status.FAILED); this.executionFlowDao.updateExecutableFlow(flow1); activeFlows1 = this.fetchActiveFlowDao.fetchActiveFlows(); assertThat(activeFlows1.containsKey(flow1.getExecutionId())).isFalse(); }
@Test public void testAssignAndUnassignExecutor() throws Exception { final String host = "localhost"; final int port = 12345; final Executor executor = this.executorDao.addExecutor(host, port); final ExecutableFlow flow = TestUtils.createTestExecutableFlow("exectest1", "exec1"); this.executionFlowDao.uploadExecutableFlow(flow); this.assignExecutor.assignExecutor(executor.getId(), flow.getExecutionId()); final Executor fetchExecutor = this.executorDao .fetchExecutorByExecutionId(flow.getExecutionId()); assertThat(fetchExecutor).isEqualTo(executor); this.assignExecutor.unassignExecutor(flow.getExecutionId()); assertThat(this.executorDao.fetchExecutorByExecutionId(flow.getExecutionId())).isNull(); }