@Override public Collection<Executor> getAllActiveExecutors() { List<Executor> executors = new ArrayList<>(); try { executors = this.executorLoader.fetchActiveExecutors(); } catch (final ExecutorManagerException e) { logger.error("Failed to get all active executors.", e); } return executors; }
private void loadRunningExecutions() throws ExecutorManagerException { logger.info("Loading running flows from database.."); final Map<Integer, Pair<ExecutionReference, ExecutableFlow>> activeFlows = this.executorLoader .fetchActiveFlows(); logger.info("Loaded " + activeFlows.size() + " running flows"); this.runningExecutions.get().putAll(activeFlows); }
/** * Fetch ExecutableFlow from database. {@inheritDoc} */ @Override public ExecutableFlow getExecutableFlow(final int execId) throws ExecutorManagerException { return this.executorLoader.fetchExecutableFlow(execId); }
@Override public int getExecutableFlows(final int projectId, final String flowId, final int from, final int length, final List<ExecutableFlow> outputList) throws ExecutorManagerException { final List<ExecutableFlow> flows = this.executorLoader.fetchFlowHistory(projectId, flowId, from, length); outputList.addAll(flows); return this.executorLoader.fetchNumExecutableFlows(projectId, flowId); }
} else { this.updaterStage.set("finalizing flow " + execId + " loading from db"); dsFlow = this.executorLoader.fetchExecutableFlow(execId); this.updaterStage.set("finalizing flow " + execId + " failing the flow"); ExecutionControllerUtils.failEverything(dsFlow); this.executorLoader.updateExecutableFlow(dsFlow); this.executorLoader.updateExecutableFlow(dsFlow); this.executorLoader.removeActiveExecutableReference(execId);
@Test public void testSubmitFlows() throws Exception { testSetUpForRunningFlows(); final ExecutableFlow flow1 = TestUtils.createTestExecutableFlow("exectest1", "exec1"); this.manager.submitExecutableFlow(flow1, this.user.getUserId()); verify(this.loader).uploadExecutableFlow(flow1); verify(this.loader).addActiveExecutableReference(any()); }
@Override public List<ExecutableFlow> getExecutableFlows(final String flowIdContains, final int skip, final int size) throws ExecutorManagerException { final List<ExecutableFlow> flows = this.executorLoader.fetchFlowHistory(null, '%' + flowIdContains + '%', null, 0, -1, -1, skip, size); return flows; }
newExecutors.addAll(this.executorLoader.fetchActiveExecutors()); } else if (this.azkProps.containsKey(ConfigurationKeys.EXECUTOR_PORT)) { executorHost, executorPort)); Executor executor = this.executorLoader.fetchExecutor(executorHost, executorPort); if (executor == null) { executor = this.executorLoader.addExecutor(executorHost, executorPort); } else if (!executor.isActive()) { executor.setActive(true); this.executorLoader.updateExecutor(executor);
this.activeExecutors = ImmutableList.of(executor1, executor2); this.allExecutors = ImmutableList.of(executor1, executor2, executor3); when(this.loader.fetchActiveExecutors()).thenReturn(this.activeExecutors); .of(this.flow2.getExecutionId(), new Pair<>(this.ref2, this.flow2), this.flow3.getExecutionId(), new Pair<>(this.ref3, this.flow3)); when(this.loader.fetchActiveFlows()).thenReturn(this.activeFlows); this.queuedFlows = ImmutableList.of(new Pair<>(this.ref1, this.flow1)); when(this.loader.fetchQueuedFlows()).thenReturn(this.queuedFlows);
/** * 1. Executor 1 throws an exception when trying to dispatch to it 2. ExecutorManager should try * next executor 3. Executor 2 accepts the dispatched execution */ @Test public void testDispatchException() throws Exception { testSetUpForRunningFlows(); this.manager.start(); final ExecutableFlow flow1 = TestUtils.createTestExecutableFlow("exectest1", "exec1"); doReturn(flow1).when(this.loader).fetchExecutableFlow(-1); mockFlowDoesNotExist(); when(this.apiGateway.callWithExecutable(any(), any(), eq(ConnectorParams.EXECUTE_ACTION))) .thenThrow(new ExecutorManagerException("Mocked dispatch exception")) .thenReturn(null); this.manager.submitExecutableFlow(flow1, this.user.getUserId()); waitFlowFinished(flow1); verify(this.apiGateway) .callWithExecutable(flow1, this.manager.fetchExecutor(1), ConnectorParams.EXECUTE_ACTION); verify(this.apiGateway) .callWithExecutable(flow1, this.manager.fetchExecutor(2), ConnectorParams.EXECUTE_ACTION); verify(this.loader, Mockito.times(1)).unassignExecutor(-1); }
dsFlow = flow; } else { dsFlow = executorLoader.fetchExecutableFlow(flow.getExecutionId()); executorLoader.updateExecutableFlow(dsFlow); executorLoader.updateExecutableFlow(dsFlow);
executorLoader.uploadExecutableFlow(exflow); executorLoader.addActiveExecutableReference(reference); try { callExecutorServer(reference, ConnectorParams.EXECUTE_ACTION); executorLoader.removeActiveExecutableReference(reference.getExecId()); throw e;
@Test public void testSetupExecutorsSucess() throws Exception { this.props.put(Constants.ConfigurationKeys.USE_MULTIPLE_EXECUTORS, "true"); final Executor executor1 = this.loader.addExecutor("localhost", 12345); final ExecutorManager manager = createExecutorManager(); Assert.assertArrayEquals(manager.getAllActiveExecutors().toArray(), new Executor[]{executor1}); // mark older executor as inactive executor1.setActive(false); this.loader.updateExecutor(executor1); final Executor executor2 = this.loader.addExecutor("localhost", 12346); final Executor executor3 = this.loader.addExecutor("localhost", 12347); manager.setupExecutors(); Assert.assertArrayEquals(manager.getAllActiveExecutors().toArray(), new Executor[]{executor2, executor3}); }
private void testSetUpForRunningFlows() throws Exception { this.loader = mock(ExecutorLoader.class); this.apiGateway = mock(ExecutorApiGateway.class); this.user = TestUtils.getTestUser(); this.props.put(Constants.ConfigurationKeys.USE_MULTIPLE_EXECUTORS, "true"); //To test runningFlows, AZKABAN_QUEUEPROCESSING_ENABLED should be set to true //so that flows will be dispatched to executors. this.props.put(Constants.ConfigurationKeys.QUEUEPROCESSING_ENABLED, "true"); // allow two concurrent runs give one Flow this.props.put(Constants.ConfigurationKeys.MAX_CONCURRENT_RUNS_ONEFLOW, 2); final List<Executor> executors = new ArrayList<>(); final Executor executor1 = new Executor(1, "localhost", 12345, true); final Executor executor2 = new Executor(2, "localhost", 12346, true); executors.add(executor1); executors.add(executor2); when(this.loader.fetchActiveExecutors()).thenReturn(executors); this.manager = createExecutorManager(); this.flow1 = TestUtils.createTestExecutableFlow("exectest1", "exec1"); this.flow2 = TestUtils.createTestExecutableFlow("exectest1", "exec2"); this.flow1.setExecutionId(1); this.flow2.setExecutionId(2); final ExecutionReference ref1 = new ExecutionReference(this.flow1.getExecutionId(), executor1); final ExecutionReference ref2 = new ExecutionReference(this.flow2.getExecutionId(), executor2); this.activeFlows.put(this.flow1.getExecutionId(), new Pair<>(ref1, this.flow1)); this.activeFlows.put(this.flow2.getExecutionId(), new Pair<>(ref2, this.flow2)); when(this.loader.fetchActiveFlows()).thenReturn(this.activeFlows); }
@Override public Executor fetchExecutor(final int executorId) throws ExecutorManagerException { return this.executorLoader.fetchExecutor(executorId); }
private void submitFlow(final ExecutableFlow flow, final ExecutionReference ref) throws Exception { when(this.loader.fetchUnfinishedFlows()).thenReturn(this.unfinishedFlows); when(this.loader.fetchExecutableFlow(flow.getExecutionId())).thenReturn(flow); this.controller.submitExecutableFlow(flow, this.user.getUserId()); this.unfinishedFlows.put(flow.getExecutionId(), new Pair<>(ref, flow)); }
@Test public void testSubmitFlows() throws Exception { this.controller.submitExecutableFlow(this.flow1, this.user.getUserId()); verify(this.loader).uploadExecutableFlow(this.flow1); }
@Override public List<Object> getExecutionJobStats(final ExecutableFlow exFlow, final String jobId, final int attempt) throws ExecutorManagerException { final Pair<ExecutionReference, ExecutableFlow> pair = this.executorLoader.fetchActiveFlowByExecId(exFlow.getExecutionId()); if (pair == null) { return this.executorLoader.fetchAttachments(exFlow.getExecutionId(), jobId, attempt); } final Pair<String, String> jobIdParam = new Pair<>("jobId", jobId); final Pair<String, String> attemptParam = new Pair<>("attempt", String.valueOf(attempt)); @SuppressWarnings("unchecked") final Map<String, Object> result = this.apiGateway.callWithReference(pair.getFirst(), ConnectorParams.ATTACHMENTS_ACTION, jobIdParam, attemptParam); @SuppressWarnings("unchecked") final List<Object> jobStats = (List<Object>) result .get("attachments"); return jobStats; }
@Override public List<Object> getExecutionJobStats(final ExecutableFlow exFlow, final String jobId, final int attempt) throws ExecutorManagerException { final Pair<ExecutionReference, ExecutableFlow> pair = this.runningExecutions.get().get(exFlow.getExecutionId()); if (pair == null) { return this.executorLoader.fetchAttachments(exFlow.getExecutionId(), jobId, attempt); } final Pair<String, String> jobIdParam = new Pair<>("jobId", jobId); final Pair<String, String> attemptParam = new Pair<>("attempt", String.valueOf(attempt)); @SuppressWarnings("unchecked") final Map<String, Object> result = this.apiGateway.callWithReference(pair.getFirst(), ConnectorParams.ATTACHMENTS_ACTION, jobIdParam, attemptParam); @SuppressWarnings("unchecked") final List<Object> jobStats = (List<Object>) result .get("attachments"); return jobStats; }
private ExecutorManager createMultiExecutorManagerInstance() throws Exception { this.props.put(Constants.ConfigurationKeys.USE_MULTIPLE_EXECUTORS, "true"); this.props.put(Constants.ConfigurationKeys.QUEUEPROCESSING_ENABLED, "false"); this.loader.addExecutor("localhost", 12345); this.loader.addExecutor("localhost", 12346); return createExecutorManager(); }