@Override public int getNumRunningFlows() { return this.manager.getRunningFlows().size(); }
Integer.valueOf(options.getFlowParameters().get( ExecutionOptions.USE_EXECUTOR)); executor = fetchExecutor(executorId);
@Override public String getExecutorThreadState() { return this.manager.getExecutorManagerThreadState().toString(); }
@Test public void testDisablingQueueProcessThread() throws Exception { final ExecutorManager manager = createMultiExecutorManagerInstance(); manager.enableQueueProcessorThread(); Assert.assertEquals(manager.isQueueProcessorThreadActive(), true); manager.disableQueueProcessorThread(); Assert.assertEquals(manager.isQueueProcessorThreadActive(), false); }
/** * 1. Executor 1 throws an exception when trying to dispatch to it 2. ExecutorManager should try * next executor 3. Executor 2 accepts the dispatched execution */ @Test public void testDispatchException() throws Exception { testSetUpForRunningFlows(); this.manager.start(); final ExecutableFlow flow1 = TestUtils.createTestExecutableFlow("exectest1", "exec1"); doReturn(flow1).when(this.loader).fetchExecutableFlow(-1); mockFlowDoesNotExist(); when(this.apiGateway.callWithExecutable(any(), any(), eq(ConnectorParams.EXECUTE_ACTION))) .thenThrow(new ExecutorManagerException("Mocked dispatch exception")) .thenReturn(null); this.manager.submitExecutableFlow(flow1, this.user.getUserId()); waitFlowFinished(flow1); verify(this.apiGateway) .callWithExecutable(flow1, this.manager.fetchExecutor(1), ConnectorParams.EXECUTE_ACTION); verify(this.apiGateway) .callWithExecutable(flow1, this.manager.fetchExecutor(2), ConnectorParams.EXECUTE_ACTION); verify(this.loader, Mockito.times(1)).unassignExecutor(-1); }
@Test public void testQueuedFlows() throws Exception { final ExecutorManager manager = createMultiExecutorManagerInstance(); final ExecutableFlow flow1 = TestUtils.createTestExecutableFlow("exectest1", "exec1"); flow1.setExecutionId(1); final ExecutableFlow flow2 = TestUtils.createTestExecutableFlow("exectest1", "exec2"); flow2.setExecutionId(2); final User testUser = TestUtils.getTestUser(); manager.submitExecutableFlow(flow1, testUser.getUserId()); manager.submitExecutableFlow(flow2, testUser.getUserId()); final List<Integer> testFlows = Arrays.asList(flow1.getExecutionId(), flow2.getExecutionId()); final List<Pair<ExecutionReference, ExecutableFlow>> queuedFlowsDB = this.loader.fetchQueuedFlows(); Assert.assertEquals(queuedFlowsDB.size(), testFlows.size()); // Verify things are correctly setup in db for (final Pair<ExecutionReference, ExecutableFlow> pair : queuedFlowsDB) { Assert.assertTrue(testFlows.contains(pair.getSecond().getExecutionId())); } // Verify running flows using old definition of "running" flows i.e. a // non-dispatched flow is also considered running final List<Integer> managerActiveFlows = manager.getRunningFlows() .stream().map(ExecutableFlow::getExecutionId).collect(Collectors.toList()); Assert.assertTrue(managerActiveFlows.containsAll(testFlows) && testFlows.containsAll(managerActiveFlows)); // Verify getQueuedFlowIds method Assert.assertEquals("[1, 2]", manager.getQueuedFlowIds()); }
@Test public void testKillQueuedFlow() throws Exception { final ExecutorManager manager = createMultiExecutorManagerInstance(); final ExecutableFlow flow1 = TestUtils.createTestExecutableFlow("exectest1", "exec1"); final User testUser = TestUtils.getTestUser(); manager.submitExecutableFlow(flow1, testUser.getUserId()); manager.cancelFlow(flow1, testUser.getUserId()); final ExecutableFlow fetchedFlow = this.loader.fetchExecutableFlow(flow1.getExecutionId()); Assert.assertEquals(fetchedFlow.getStatus(), Status.FAILED); Assert.assertFalse(manager.getRunningFlows().contains(flow1)); }
final List<Integer> running = getRunningFlows(projectId, flowId); new ExecutionReference(exflow.getExecutionId()); if (isMultiExecutorMode()) { this.executorLoader.addActiveExecutableReference(reference); try { dispatch(reference, exflow, choosenExecutor); this.commonMetrics.markDispatchSuccess(); } catch (final ExecutorManagerException e) { finalizeFlows(exflow); throw e;
exflow.setSubmitTime(System.currentTimeMillis()); List<Integer> running = getRunningFlows(projectId, flowId); applyDisabledJobs(options.getDisabledJobs(), exflow); executorLoader.addActiveExecutableReference(reference); try { callExecutorServer(reference, ConnectorParams.EXECUTE_ACTION); runningFlows.put(exflow.getExecutionId(), new Pair<ExecutionReference, ExecutableFlow>(reference, exflow));
@Ignore @Test public void testFetchActiveFlowByProject() throws Exception { testSetUpForRunningFlows(); final List<Integer> executions = this.manager.getRunningFlows(this.flow1.getProjectId(), this.flow1.getFlowId()); Assert.assertTrue(executions.contains(this.flow1.getExecutionId())); Assert .assertTrue(this.manager.isFlowRunning(this.flow1.getProjectId(), this.flow1.getFlowId())); }
@Test public void testEnablingQueueProcessThread() throws Exception { final ExecutorManager manager = createMultiExecutorManagerInstance(); Assert.assertEquals(manager.isQueueProcessorThreadActive(), false); manager.enableQueueProcessorThread(); Assert.assertEquals(manager.isQueueProcessorThreadActive(), true); }
@Override public List<String> getPrimaryExecutorHostPorts() { return new ArrayList<>(this.manager.getPrimaryServerHosts()); }
@Override public String getRunningFlows() { return this.manager.getRunningFlowIds(); }
@Override public boolean isThreadActive() { return this.manager.isExecutorManagerThreadActive(); }
@Override public String getExecutorThreadStage() { return this.manager.getExecutorThreadStage(); }
@Override public Long getLastThreadCheckTime() { return this.manager.getLastExecutorManagerThreadCheckTime(); }
@Ignore @Test public void testFetchActiveFlowWithExecutor() throws Exception { testSetUpForRunningFlows(); final List<Pair<ExecutableFlow, Optional<Executor>>> activeFlowsWithExecutor = this.manager.getActiveFlowsWithExecutor(); Assert.assertTrue(activeFlowsWithExecutor.contains(new Pair<>(this.flow1, Optional.ofNullable(this.manager.fetchExecutor(this.flow1.getExecutionId()))))); Assert.assertTrue(activeFlowsWithExecutor.contains(new Pair<>(this.flow2, Optional.ofNullable(this.manager.fetchExecutor(this.flow2.getExecutionId()))))); }
@Test public void testFetchAllActiveExecutorServerHosts() throws Exception { testSetUpForRunningFlows(); final Set<String> activeExecutorServerHosts = this.manager.getAllActiveExecutorServerHosts(); final Executor executor1 = this.manager.fetchExecutor(this.flow1.getExecutionId()); final Executor executor2 = this.manager.fetchExecutor(this.flow2.getExecutionId()); Assert.assertTrue( activeExecutorServerHosts.contains(executor1.getHost() + ":" + executor1.getPort())); Assert.assertTrue( activeExecutorServerHosts.contains(executor2.getHost() + ":" + executor2.getPort())); }
@Override public String getQueuedFlows() { return this.manager.getQueuedFlowIds(); }
@Override public boolean isQueueProcessorActive() { return this.manager.isQueueProcessorThreadActive(); }