Integer.valueOf(options.getFlowParameters().get( ExecutionOptions.USE_EXECUTOR)); executor = fetchExecutor(executorId);
/** * Manage servlet call for stats servlet in Azkaban execution server {@inheritDoc} * * @see azkaban.executor.ExecutorManagerAdapter#callExecutorStats(int, java.lang.String, * azkaban.utils.Pair[]) */ @Override public Map<String, Object> callExecutorStats(final int executorId, final String action, final Pair<String, String>... params) throws IOException, ExecutorManagerException { final Executor executor = fetchExecutor(executorId); final List<Pair<String, String>> paramList = new ArrayList<>(); // if params = null if (params != null) { paramList.addAll(Arrays.asList(params)); } paramList .add(new Pair<>(ConnectorParams.ACTION_PARAM, action)); return this.apiGateway.callForJsonObjectMap(executor.getHost(), executor.getPort(), "/stats", paramList); }
@Ignore @Test public void testFetchActiveFlowWithExecutor() throws Exception { testSetUpForRunningFlows(); final List<Pair<ExecutableFlow, Optional<Executor>>> activeFlowsWithExecutor = this.manager.getActiveFlowsWithExecutor(); Assert.assertTrue(activeFlowsWithExecutor.contains(new Pair<>(this.flow1, Optional.ofNullable(this.manager.fetchExecutor(this.flow1.getExecutionId()))))); Assert.assertTrue(activeFlowsWithExecutor.contains(new Pair<>(this.flow2, Optional.ofNullable(this.manager.fetchExecutor(this.flow2.getExecutionId()))))); }
/** * 1. Executor 1 throws an exception when trying to dispatch to it 2. ExecutorManager should try * next executor 3. Executor 2 accepts the dispatched execution */ @Test public void testDispatchException() throws Exception { testSetUpForRunningFlows(); this.manager.start(); final ExecutableFlow flow1 = TestUtils.createTestExecutableFlow("exectest1", "exec1"); doReturn(flow1).when(this.loader).fetchExecutableFlow(-1); mockFlowDoesNotExist(); when(this.apiGateway.callWithExecutable(any(), any(), eq(ConnectorParams.EXECUTE_ACTION))) .thenThrow(new ExecutorManagerException("Mocked dispatch exception")) .thenReturn(null); this.manager.submitExecutableFlow(flow1, this.user.getUserId()); waitFlowFinished(flow1); verify(this.apiGateway) .callWithExecutable(flow1, this.manager.fetchExecutor(1), ConnectorParams.EXECUTE_ACTION); verify(this.apiGateway) .callWithExecutable(flow1, this.manager.fetchExecutor(2), ConnectorParams.EXECUTE_ACTION); verify(this.loader, Mockito.times(1)).unassignExecutor(-1); }
/** * ExecutorManager should try to dispatch to all executors & when both fail it should remove the * execution from queue and finalize it. */ @Ignore @Test public void testDispatchFailed() throws Exception { testSetUpForRunningFlows(); this.manager.start(); final ExecutableFlow flow1 = TestUtils.createTestExecutableFlow("exectest1", "exec1"); flow1.getExecutionOptions().setFailureEmails(Arrays.asList("test@example.com")); when(this.loader.fetchExecutableFlow(-1)).thenReturn(flow1); when(this.apiGateway.callWithExecutable(any(), any(), eq(ConnectorParams.EXECUTE_ACTION))) .thenThrow(new ExecutorManagerException("Mocked dispatch exception")); this.manager.submitExecutableFlow(flow1, this.user.getUserId()); waitFlowFinished(flow1); verify(this.apiGateway) .callWithExecutable(flow1, this.manager.fetchExecutor(1), ConnectorParams.EXECUTE_ACTION); verify(this.apiGateway) .callWithExecutable(flow1, this.manager.fetchExecutor(2), ConnectorParams.EXECUTE_ACTION); verify(this.loader, Mockito.times(2)).unassignExecutor(-1); verify(this.mailAlerter).alertOnError(eq(flow1), eq("Failed to dispatch queued execution derived-member-data because reached " + "azkaban.maxDispatchingErrors (tried 2 executors)"), contains("Mocked dispatch exception")); }
@Test public void testFetchAllActiveExecutorServerHosts() throws Exception { testSetUpForRunningFlows(); final Set<String> activeExecutorServerHosts = this.manager.getAllActiveExecutorServerHosts(); final Executor executor1 = this.manager.fetchExecutor(this.flow1.getExecutionId()); final Executor executor2 = this.manager.fetchExecutor(this.flow2.getExecutionId()); Assert.assertTrue( activeExecutorServerHosts.contains(executor1.getHost() + ":" + executor1.getPort())); Assert.assertTrue( activeExecutorServerHosts.contains(executor2.getHost() + ":" + executor2.getPort())); }
/** * ExecutorManager should try to dispatch to all executors until it succeeds. */ @Test public void testDispatchMultipleRetries() throws Exception { this.props.put(Constants.ConfigurationKeys.MAX_DISPATCHING_ERRORS_PERMITTED, 4); testSetUpForRunningFlows(); this.manager.start(); final ExecutableFlow flow1 = TestUtils.createTestExecutableFlow("exectest1", "exec1"); flow1.getExecutionOptions().setFailureEmails(Arrays.asList("test@example.com")); when(this.loader.fetchExecutableFlow(-1)).thenReturn(flow1); // fail 2 first dispatch attempts, then succeed when(this.apiGateway.callWithExecutable(any(), any(), eq(ConnectorParams.EXECUTE_ACTION))) .thenThrow(new ExecutorManagerException("Mocked dispatch exception 1")) .thenThrow(new ExecutorManagerException("Mocked dispatch exception 2")) .thenReturn(null); // this is just to clean up the execution as FAILED after it has been submitted mockFlowDoesNotExist(); this.manager.submitExecutableFlow(flow1, this.user.getUserId()); waitFlowFinished(flow1); // it's random which executor is chosen each time, but both should have been tried at least once verify(this.apiGateway, Mockito.atLeast(1)) .callWithExecutable(flow1, this.manager.fetchExecutor(1), ConnectorParams.EXECUTE_ACTION); verify(this.apiGateway, Mockito.atLeast(1)) .callWithExecutable(flow1, this.manager.fetchExecutor(2), ConnectorParams.EXECUTE_ACTION); // verify that there was a 3rd (successful) dispatch call verify(this.apiGateway, Mockito.times(3)) .callWithExecutable(eq(flow1), any(), eq(ConnectorParams.EXECUTE_ACTION)); verify(this.loader, Mockito.times(2)).unassignExecutor(-1); }
/** * Manage servlet call for stats servlet in Azkaban execution server {@inheritDoc} * * @see azkaban.executor.ExecutorManagerAdapter#callExecutorStats(int, java.lang.String, * azkaban.utils.Pair[]) */ @Override public Map<String, Object> callExecutorStats(final int executorId, final String action, final Pair<String, String>... params) throws IOException, ExecutorManagerException { final Executor executor = fetchExecutor(executorId); final List<Pair<String, String>> paramList = new ArrayList<>(); // if params = null if (params != null) { paramList.addAll(Arrays.asList(params)); } paramList .add(new Pair<>(ConnectorParams.ACTION_PARAM, action)); return this.apiGateway.callForJsonObjectMap(executor.getHost(), executor.getPort(), "/stats", paramList); }
Integer.valueOf(options.getFlowParameters().get( ExecutionOptions.USE_EXECUTOR)); executor = fetchExecutor(executorId);