@Test public void testProcessSucceed() throws Exception { final TriggerInstance triggerInstance = createTriggerInstance(); this.processor.processSucceed(triggerInstance); this.submitFlowLatch.await(10L, TimeUnit.SECONDS); verify(this.executorManager).submitExecutableFlow(any(), anyString()); this.updateExecIDLatch.await(10L, TimeUnit.SECONDS); verify(this.triggerInstLoader).updateAssociatedFlowExecId(triggerInstance); }
@Ignore("Flaky test - ignored until stabilized") @Test public void testStartZeroDependencyTrigger() throws InterruptedException, ExecutorManagerException { final List<FlowTriggerDependency> deps = new ArrayList<>(); final FlowTrigger flowTrigger = TestUtil.createTestFlowTrigger(deps, Duration.ofSeconds(10)); for (int i = 0; i < 30; i++) { flowTriggerService.startTrigger(flowTrigger, "testflow", 1, "test", createProject()); } Thread.sleep(Duration.ofSeconds(1).toMillis()); // zero dependency trigger will launch associated flow immediately final Collection<TriggerInstance> triggerInstances = flowTriggerService.getRunningTriggers(); assertThat(triggerInstances).isEmpty(); verify(executorManager, times(30)).submitExecutableFlow(any(), anyString()); }
@Before public void setUp() throws Exception { this.message = EmailerTest.mockEmailMessage(); this.messageCreator = EmailerTest.mockMessageCreator(this.message); this.triggerInstLoader = mock(FlowTriggerInstanceLoader.class); this.executorManager = mock(ExecutorManager.class); this.executorLoader = new MockExecutorLoader(); when(this.executorManager.submitExecutableFlow(any(), anyString())).thenReturn("return"); final CommonMetrics commonMetrics = new CommonMetrics(new MetricsManager(new MetricRegistry())); this.emailer = Mockito.spy(new Emailer(EmailerTest.createMailProperties(), commonMetrics, this.messageCreator, this.executorLoader)); this.sendEmailLatch = new CountDownLatch(1); doAnswer(invocation -> { this.sendEmailLatch.countDown(); return null; }).when(this.emailer).sendEmail(any(), any(), any()); this.submitFlowLatch = new CountDownLatch(1); doAnswer(invocation -> { this.submitFlowLatch.countDown(); return null; }).when(this.executorManager).submitExecutableFlow(any(), anyString()); this.updateExecIDLatch = new CountDownLatch(1); doAnswer(invocation -> { this.updateExecIDLatch.countDown(); return null; }).when(this.triggerInstLoader).updateAssociatedFlowExecId(any()); this.processor = new TriggerInstanceProcessor(this.executorManager, this.triggerInstLoader, this.emailer); }
@BeforeClass public static void setup() throws Exception { testDepCheck = new TestDependencyCheck(); final FlowTriggerDependencyPluginManager pluginManager = mock(FlowTriggerDependencyPluginManager .class); when(pluginManager.getDependencyCheck(ArgumentMatchers.eq("TestDependencyCheck"))) .thenReturn(testDepCheck); executorManager = mock(ExecutorManager.class); when(executorManager.submitExecutableFlow(any(), anyString())).thenReturn("return"); final Emailer emailer = mock(Emailer.class); Mockito.doNothing().when(emailer).sendEmail(any(), anyString(), anyString()); final TriggerInstanceProcessor triggerInstProcessor = new TriggerInstanceProcessor( executorManager, flowTriggerInstanceLoader, emailer); final DependencyInstanceProcessor depInstProcessor = new DependencyInstanceProcessor (flowTriggerInstanceLoader); final FlowTriggerExecutionCleaner executionCleaner = new FlowTriggerExecutionCleaner( flowTriggerInstanceLoader); flowTriggerService = new FlowTriggerService(pluginManager, triggerInstProcessor, depInstProcessor, flowTriggerInstanceLoader, executionCleaner); flowTriggerService.start(); }
@Test public void testSubmitFlows() throws Exception { testSetUpForRunningFlows(); final ExecutableFlow flow1 = TestUtils.createTestExecutableFlow("exectest1", "exec1"); this.manager.submitExecutableFlow(flow1, this.user.getUserId()); verify(this.loader).uploadExecutableFlow(flow1); verify(this.loader).addActiveExecutableReference(any()); }
@Test(expected = ExecutorManagerException.class) public void testDuplicateQueuedFlows() throws Exception { final ExecutorManager manager = createMultiExecutorManagerInstance(); final ExecutableFlow flow1 = TestUtils.createTestExecutableFlow("exectest1", "exec1"); flow1.getExecutionOptions().setConcurrentOption( ExecutionOptions.CONCURRENT_OPTION_SKIP); final User testUser = TestUtils.getTestUser(); manager.submitExecutableFlow(flow1, testUser.getUserId()); manager.submitExecutableFlow(flow1, testUser.getUserId()); }
private void executeFlowAndUpdateExecID(final TriggerInstance triggerInst) { try { final Project project = triggerInst.getProject(); final Flow flow = FlowUtils.getFlow(project, triggerInst.getFlowId()); final ExecutableFlow executableFlow = FlowUtils.createExecutableFlow(project, flow); // execute the flow with default execution option(concurrency option being "ignore // currently running") this.executorManager.submitExecutableFlow(executableFlow, triggerInst.getSubmitUser()); triggerInst.setFlowExecId(executableFlow.getExecutionId()); } catch (final Exception ex) { logger.error("exception when executing the associated flow and updating flow exec id for " + "trigger instance[id: {}]", triggerInst.getId(), ex); // if flow fails to be executed(e.g. running execution exceeds the allowed concurrent run // limit), set associated flow exec id to Constants.FAILED_EXEC_ID. Upon web server // restart, recovery process will skip those flows. triggerInst.setFlowExecId(Constants.FAILED_EXEC_ID); } this.flowTriggerInstanceLoader.updateAssociatedFlowExecId(triggerInst); }
/** * 1. Executor 1 throws an exception when trying to dispatch to it 2. ExecutorManager should try * next executor 3. Executor 2 accepts the dispatched execution */ @Test public void testDispatchException() throws Exception { testSetUpForRunningFlows(); this.manager.start(); final ExecutableFlow flow1 = TestUtils.createTestExecutableFlow("exectest1", "exec1"); doReturn(flow1).when(this.loader).fetchExecutableFlow(-1); mockFlowDoesNotExist(); when(this.apiGateway.callWithExecutable(any(), any(), eq(ConnectorParams.EXECUTE_ACTION))) .thenThrow(new ExecutorManagerException("Mocked dispatch exception")) .thenReturn(null); this.manager.submitExecutableFlow(flow1, this.user.getUserId()); waitFlowFinished(flow1); verify(this.apiGateway) .callWithExecutable(flow1, this.manager.fetchExecutor(1), ConnectorParams.EXECUTE_ACTION); verify(this.apiGateway) .callWithExecutable(flow1, this.manager.fetchExecutor(2), ConnectorParams.EXECUTE_ACTION); verify(this.loader, Mockito.times(1)).unassignExecutor(-1); }
/** * ExecutorManager should try to dispatch to all executors & when both fail it should remove the * execution from queue and finalize it. */ @Ignore @Test public void testDispatchFailed() throws Exception { testSetUpForRunningFlows(); this.manager.start(); final ExecutableFlow flow1 = TestUtils.createTestExecutableFlow("exectest1", "exec1"); flow1.getExecutionOptions().setFailureEmails(Arrays.asList("test@example.com")); when(this.loader.fetchExecutableFlow(-1)).thenReturn(flow1); when(this.apiGateway.callWithExecutable(any(), any(), eq(ConnectorParams.EXECUTE_ACTION))) .thenThrow(new ExecutorManagerException("Mocked dispatch exception")); this.manager.submitExecutableFlow(flow1, this.user.getUserId()); waitFlowFinished(flow1); verify(this.apiGateway) .callWithExecutable(flow1, this.manager.fetchExecutor(1), ConnectorParams.EXECUTE_ACTION); verify(this.apiGateway) .callWithExecutable(flow1, this.manager.fetchExecutor(2), ConnectorParams.EXECUTE_ACTION); verify(this.loader, Mockito.times(2)).unassignExecutor(-1); verify(this.mailAlerter).alertOnError(eq(flow1), eq("Failed to dispatch queued execution derived-member-data because reached " + "azkaban.maxDispatchingErrors (tried 2 executors)"), contains("Mocked dispatch exception")); }
@Test(expected = ExecutorManagerException.class) public void testTooManySubmitFlows() throws Exception { testSetUpForRunningFlows(); final ExecutableFlow flow1 = TestUtils .createTestExecutableFlowFromYaml("basicyamlshelltest", "bashSleep"); flow1.setExecutionId(101); final ExecutableFlow flow2 = TestUtils .createTestExecutableFlowFromYaml("basicyamlshelltest", "bashSleep"); flow2.setExecutionId(102); final ExecutableFlow flow3 = TestUtils .createTestExecutableFlowFromYaml("basicyamlshelltest", "bashSleep"); flow3.setExecutionId(103); final ExecutableFlow flow4 = TestUtils .createTestExecutableFlowFromYaml("basicyamlshelltest", "bashSleep"); flow4.setExecutionId(104); this.manager.submitExecutableFlow(flow1, this.user.getUserId()); verify(this.loader).uploadExecutableFlow(flow1); this.manager.submitExecutableFlow(flow2, this.user.getUserId()); verify(this.loader).uploadExecutableFlow(flow2); this.manager.submitExecutableFlow(flow3, this.user.getUserId()); this.manager.submitExecutableFlow(flow4, this.user.getUserId()); }
/** * ExecutorManager should try to dispatch to all executors until it succeeds. */ @Test public void testDispatchMultipleRetries() throws Exception { this.props.put(Constants.ConfigurationKeys.MAX_DISPATCHING_ERRORS_PERMITTED, 4); testSetUpForRunningFlows(); this.manager.start(); final ExecutableFlow flow1 = TestUtils.createTestExecutableFlow("exectest1", "exec1"); flow1.getExecutionOptions().setFailureEmails(Arrays.asList("test@example.com")); when(this.loader.fetchExecutableFlow(-1)).thenReturn(flow1); // fail 2 first dispatch attempts, then succeed when(this.apiGateway.callWithExecutable(any(), any(), eq(ConnectorParams.EXECUTE_ACTION))) .thenThrow(new ExecutorManagerException("Mocked dispatch exception 1")) .thenThrow(new ExecutorManagerException("Mocked dispatch exception 2")) .thenReturn(null); // this is just to clean up the execution as FAILED after it has been submitted mockFlowDoesNotExist(); this.manager.submitExecutableFlow(flow1, this.user.getUserId()); waitFlowFinished(flow1); // it's random which executor is chosen each time, but both should have been tried at least once verify(this.apiGateway, Mockito.atLeast(1)) .callWithExecutable(flow1, this.manager.fetchExecutor(1), ConnectorParams.EXECUTE_ACTION); verify(this.apiGateway, Mockito.atLeast(1)) .callWithExecutable(flow1, this.manager.fetchExecutor(2), ConnectorParams.EXECUTE_ACTION); // verify that there was a 3rd (successful) dispatch call verify(this.apiGateway, Mockito.times(3)) .callWithExecutable(eq(flow1), any(), eq(ConnectorParams.EXECUTE_ACTION)); verify(this.loader, Mockito.times(2)).unassignExecutor(-1); }
@Test public void testQueuedFlows() throws Exception { final ExecutorManager manager = createMultiExecutorManagerInstance(); final ExecutableFlow flow1 = TestUtils.createTestExecutableFlow("exectest1", "exec1"); flow1.setExecutionId(1); final ExecutableFlow flow2 = TestUtils.createTestExecutableFlow("exectest1", "exec2"); flow2.setExecutionId(2); final User testUser = TestUtils.getTestUser(); manager.submitExecutableFlow(flow1, testUser.getUserId()); manager.submitExecutableFlow(flow2, testUser.getUserId()); final List<Integer> testFlows = Arrays.asList(flow1.getExecutionId(), flow2.getExecutionId()); final List<Pair<ExecutionReference, ExecutableFlow>> queuedFlowsDB = this.loader.fetchQueuedFlows(); Assert.assertEquals(queuedFlowsDB.size(), testFlows.size()); // Verify things are correctly setup in db for (final Pair<ExecutionReference, ExecutableFlow> pair : queuedFlowsDB) { Assert.assertTrue(testFlows.contains(pair.getSecond().getExecutionId())); } // Verify running flows using old definition of "running" flows i.e. a // non-dispatched flow is also considered running final List<Integer> managerActiveFlows = manager.getRunningFlows() .stream().map(ExecutableFlow::getExecutionId).collect(Collectors.toList()); Assert.assertTrue(managerActiveFlows.containsAll(testFlows) && testFlows.containsAll(managerActiveFlows)); // Verify getQueuedFlowIds method Assert.assertEquals("[1, 2]", manager.getQueuedFlowIds()); }
@Test public void testNotFoundFlows() throws Exception { testSetUpForRunningFlows(); this.manager.start(); final ExecutableFlow flow1 = TestUtils.createTestExecutableFlow("exectest1", "exec1"); when(this.loader.fetchExecutableFlow(-1)).thenReturn(flow1); mockFlowDoesNotExist(); this.manager.submitExecutableFlow(flow1, this.user.getUserId()); final ExecutableFlow fetchedFlow = waitFlowFinished(flow1); Assert.assertEquals(fetchedFlow.getStatus(), Status.FAILED); }
@Test public void testKillQueuedFlow() throws Exception { final ExecutorManager manager = createMultiExecutorManagerInstance(); final ExecutableFlow flow1 = TestUtils.createTestExecutableFlow("exectest1", "exec1"); final User testUser = TestUtils.getTestUser(); manager.submitExecutableFlow(flow1, testUser.getUserId()); manager.cancelFlow(flow1, testUser.getUserId()); final ExecutableFlow fetchedFlow = this.loader.fetchExecutableFlow(flow1.getExecutionId()); Assert.assertEquals(fetchedFlow.getStatus(), Status.FAILED); Assert.assertFalse(manager.getRunningFlows().contains(flow1)); }