@Override public List<Executor> handle(final ResultSet rs) throws SQLException { if (!rs.next()) { return Collections.emptyList(); } final List<Executor> executors = new ArrayList<>(); do { final int id = rs.getInt(1); final String host = rs.getString(2); final int port = rs.getInt(3); final boolean active = rs.getBoolean(4); final Executor executor = new Executor(id, host, port, active); executors.add(executor); } while (rs.next()); return executors; } }
private static Pair<ExecutionReference, ExecutableFlow> getPairWithExecutorInfo(final ResultSet rs, final ExecutableFlow exFlow) throws SQLException { final int executorId = rs.getInt("executorId"); final String host = rs.getString("host"); final int port = rs.getInt("port"); final Executor executor; if (host == null) { logger.warn("Executor id " + executorId + " (on execution " + exFlow.getExecutionId() + ") wasn't found"); executor = null; } else { final boolean executorStatus = rs.getBoolean("executorStatus"); executor = new Executor(executorId, host, port, executorStatus); } final ExecutionReference ref = new ExecutionReference(exFlow.getExecutionId(), executor); return new Pair<>(ref, exFlow); }
@Override public Executor addExecutor(final String host, final int port) throws ExecutorManagerException { Executor executor = null; if (fetchExecutor(host, port) == null) { this.executorIdCounter++; executor = new Executor(this.executorIdCounter, host, port, true); this.executors.add(executor); } return executor; }
@Test public void testMissingExecutorUpdate() throws Exception { final Executor executor = new Executor(1, "localhost", 1234, true); assertThatThrownBy(() -> this.executorDao.updateExecutor(executor)) .isInstanceOf(ExecutorManagerException.class) .hasMessageContaining("No executor with id"); }
@Test public void testCreatingExecutorSelectorWithEmptyFilterComparatorList() throws Exception { final List<Executor> executorList = new ArrayList<>(); executorList.add(new Executor(1, "host1", 80, true)); executorList.add(new Executor(2, "host2", 80, true)); executorList.add(new Executor(3, "host3", 80, true)); executorList.get(0) .setExecutorInfo(new ExecutorInfo(99.9, 14095, 50, System.currentTimeMillis(), 89, 0)); executorList.get(1) .setExecutorInfo(new ExecutorInfo(50, 14095, 50, System.currentTimeMillis(), 90, 0)); executorList.get(2) .setExecutorInfo(new ExecutorInfo(99.9, 14095, 50, System.currentTimeMillis(), 90, 0)); final ExecutableFlow flow = new ExecutableFlow(); final ExecutorSelector selector = new ExecutorSelector(null, null); final Executor executor = selector.getBest(executorList, flow); Assert.assertEquals(executorList.get(2), executor); }
@Test public void updateExecutions() throws Exception { final ImmutableMap<String, String> map = ImmutableMap.of("test", "response"); when(this.client .httpPost(eq(new URI("http://executor-2:1234/executor")), this.params.capture())) .thenReturn(JSONUtils.toJSON(map)); final Map<String, Object> response = this.gateway .updateExecutions(new Executor(2, "executor-2", 1234, true), Collections.singletonList(new ExecutableFlow())); assertEquals(map, response); assertEquals(new Pair<>("executionId", "[-1]"), this.params.getValue().get(0)); assertEquals(new Pair<>("updatetime", "[-1]"), this.params.getValue().get(1)); assertEquals(new Pair<>("action", "update"), this.params.getValue().get(2)); assertEquals(new Pair<>("execid", "null"), this.params.getValue().get(3)); assertEquals(new Pair<>("user", null), this.params.getValue().get(4)); } }
@Before public void setUp() throws Exception { this.props = new Props(); this.props.put(ConfigurationKeys.AZKABAN_EXECUTOR_MAX_FAILURE_COUNT, 2); this.props.put(ConfigurationKeys.AZKABAN_ADMIN_ALERT_EMAIL, AZ_ADMIN_ALERT_EMAIL); this.loader = mock(ExecutorLoader.class); this.mailAlerter = mock(Alerter.class); this.alerterHolder = mock(AlerterHolder.class); this.apiGateway = mock(ExecutorApiGateway.class); this.executorHealthChecker = new ExecutorHealthChecker(this.props, this.loader, this .apiGateway, this.alerterHolder); this.flow1 = TestUtils.createTestExecutableFlow("exectest1", "exec1"); this.flow1.setExecutionId(EXECUTION_ID_11); this.flow1.setStatus(Status.RUNNING); this.executor1 = new Executor(1, "localhost", 12345, true); when(this.loader.fetchActiveFlows()).thenReturn(this.activeFlows); when(this.alerterHolder.get("email")).thenReturn(this.mailAlerter); }
@Test public void testExecutorSelectorE2E() throws Exception { final List<String> filterList = new ArrayList<>(ExecutorFilter.getAvailableFilterNames()); final Map<String, Integer> comparatorMap; comparatorMap = new HashMap<>(); final List<Executor> executorList = new ArrayList<>(); executorList.add(new Executor(1, "host1", 80, true)); executorList.add(new Executor(2, "host2", 80, true)); executorList.add(new Executor(3, "host3", 80, true)); executorList.get(0) .setExecutorInfo(new ExecutorInfo(99.9, 14095, 50, System.currentTimeMillis(), 89, 0)); executorList.get(1) .setExecutorInfo(new ExecutorInfo(50, 14095, 50, System.currentTimeMillis(), 90, 0)); executorList.get(2) .setExecutorInfo(new ExecutorInfo(99.9, 14095, 50, System.currentTimeMillis(), 90, 0)); final ExecutableFlow flow = new ExecutableFlow(); for (final String name : ExecutorComparator.getAvailableComparatorNames()) { comparatorMap.put(name, 1); } final ExecutorSelector selector = new ExecutorSelector(filterList, comparatorMap); Executor executor = selector.getBest(executorList, flow); Assert.assertEquals(executorList.get(0), executor); // simulate that once the flow is assigned, executor1's remaining TMP storage dropped to 2048 // now we do the getBest again executor3 is expected to be selected as it has a earlier last dispatched time. executorList.get(0) .setExecutorInfo(new ExecutorInfo(99.9, 4095, 50, System.currentTimeMillis(), 90, 1)); executor = selector.getBest(executorList, flow); Assert.assertEquals(executorList.get(2), executor); }
@Before public void setUp() throws Exception { MockitoAnnotations.initMocks(this); this.execution = new ExecutableFlow(); this.execution.setExecutionId(EXECUTION_ID_77); this.activeExecutor = new Executor(1, "activeExecutor-1", 9999, true); this.runningExecutions = new RunningExecutions(); this.runningExecutions.get().put(EXECUTION_ID_77, new Pair<>( new ExecutionReference(EXECUTION_ID_77, this.activeExecutor), this.execution)); this.updater = new RunningExecutionsUpdater(this.updaterStage, this.alerterHolder, this.commonMetrics, this.apiGateway, this.runningExecutions, this.executionFinalizer, this.executorLoader); when(this.alerterHolder.get("email")).thenReturn(this.mailAlerter); }
@Test public void alertOnFailedUpdate() throws Exception { final Flow flow = this.project.getFlow("jobe"); flow.addFailureEmails(this.receiveAddrList); Assert.assertNotNull(flow); final ExecutableFlow exFlow = new ExecutableFlow(this.project, flow); final CommonMetrics commonMetrics = new CommonMetrics(new MetricsManager(new MetricRegistry())); final Emailer emailer = new Emailer(this.props, commonMetrics, this.messageCreator, this.executorLoader); final Executor executor = new Executor(1, "executor1-host", 1234, true); final List<ExecutableFlow> executions = Arrays.asList(exFlow, exFlow); final ExecutorManagerException exception = DefaultMailCreatorTest.createTestStracktrace(); emailer.alertOnFailedUpdate(executor, executions, exception); verify(this.message).addAllToAddress(this.receiveAddrList); verify(this.message) .setSubject("Flow status could not be updated from executor1-host on azkaban"); assertThat(TestUtils.readResource("failedUpdateMessage2.html", this)) .isEqualToIgnoringWhitespace(this.message.getBody()); }
private void testSetUpForRunningFlows() throws Exception { this.loader = mock(ExecutorLoader.class); this.apiGateway = mock(ExecutorApiGateway.class); this.user = TestUtils.getTestUser(); this.props.put(Constants.ConfigurationKeys.USE_MULTIPLE_EXECUTORS, "true"); //To test runningFlows, AZKABAN_QUEUEPROCESSING_ENABLED should be set to true //so that flows will be dispatched to executors. this.props.put(Constants.ConfigurationKeys.QUEUEPROCESSING_ENABLED, "true"); // allow two concurrent runs give one Flow this.props.put(Constants.ConfigurationKeys.MAX_CONCURRENT_RUNS_ONEFLOW, 2); final List<Executor> executors = new ArrayList<>(); final Executor executor1 = new Executor(1, "localhost", 12345, true); final Executor executor2 = new Executor(2, "localhost", 12346, true); executors.add(executor1); executors.add(executor2); when(this.loader.fetchActiveExecutors()).thenReturn(executors); this.manager = createExecutorManager(); this.flow1 = TestUtils.createTestExecutableFlow("exectest1", "exec1"); this.flow2 = TestUtils.createTestExecutableFlow("exectest1", "exec2"); this.flow1.setExecutionId(1); this.flow2.setExecutionId(2); final ExecutionReference ref1 = new ExecutionReference(this.flow1.getExecutionId(), executor1); final ExecutionReference ref2 = new ExecutionReference(this.flow2.getExecutionId(), executor2); this.activeFlows.put(this.flow1.getExecutionId(), new Pair<>(ref1, this.flow1)); this.activeFlows.put(this.flow2.getExecutionId(), new Pair<>(ref2, this.flow2)); when(this.loader.fetchActiveFlows()).thenReturn(this.activeFlows); }
this.alertHolder, this.executorHealthChecker); final Executor executor1 = new Executor(1, "localhost", 12345, true); final Executor executor2 = new Executor(2, "localhost", 12346, true); final Executor executor3 = new Executor(3, "localhost", 12347, false); this.activeExecutors = ImmutableList.of(executor1, executor2); this.allExecutors = ImmutableList.of(executor1, executor2, executor3);
@Before public void setUp() throws Exception { this.defaultTz = TimeZone.getDefault(); assertNotNull(this.defaultTz); // EEST TimeZone.setDefault(TimeZone.getTimeZone("Europe/Helsinki")); DateTimeUtils.setCurrentMillisFixed(FIXED_CURRENT_TIME_MILLIS); this.mailCreator = new DefaultMailCreator(); this.executor = new Executor(1, "executor1-host", 1234, true); this.flow = new Flow("mail-creator-test"); this.project = new Project(1, "test-project"); this.options = new ExecutionOptions(); this.message = new EmailMessage("localhost", EmailMessageCreator.DEFAULT_SMTP_PORT, "", "", null); this.azkabanName = "unit-tests"; this.scheme = "http"; this.clientHostname = "localhost"; this.clientPortNumber = "8081"; final Node failedNode = new Node("test-job"); failedNode.setType("noop"); this.flow.addNode(failedNode); this.executableFlow = new ExecutableFlow(this.project, this.flow); this.executableFlow.setExecutionOptions(this.options); this.executableFlow.setStartTime(START_TIME_MILLIS); this.options.setFailureEmails(ImmutableList.of("test@example.com")); this.options.setSuccessEmails(ImmutableList.of("test@example.com")); }
@Override public List<Executor> handle(final ResultSet rs) throws SQLException { if (!rs.next()) { return Collections.emptyList(); } final List<Executor> executors = new ArrayList<>(); do { final int id = rs.getInt(1); final String host = rs.getString(2); final int port = rs.getInt(3); final boolean active = rs.getBoolean(4); final Executor executor = new Executor(id, host, port, active); executors.add(executor); } while (rs.next()); return executors; } }
executor = null; } else { executor = new Executor(executorId, host, port, executorStatus);
this.executorLoader.updateExecutor(executor); newExecutors.add(new Executor(executor.getId(), executorHost, executorPort, true)); } else {