private void logFailedDispatchAttempt(final ExecutionReference reference, final ExecutableFlow exflow, final Executor selectedExecutor, final ExecutorManagerException e) { ExecutorManager.logger.warn(String.format( "Executor %s responded with exception for exec: %d", selectedExecutor, exflow.getExecutionId()), e); ExecutorManager.logger.info(String.format( "Failed dispatch attempt for exec %d with error count %d", exflow.getExecutionId(), reference.getNumErrors())); }
private void getExecutionIdsHelper(final List<Integer> allIds, final Collection<Pair<ExecutionReference, ExecutableFlow>> collection) { collection.stream().forEach(ref -> allIds.add(ref.getSecond().getExecutionId())); Collections.sort(allIds); }
private void handleNoExecutorSelectedCase(final ExecutionReference reference, final ExecutableFlow exflow) throws ExecutorManagerException { ExecutorManager.logger .info(String .format( "Reached handleNoExecutorSelectedCase stage for exec %d with error count %d", exflow.getExecutionId(), reference.getNumErrors())); // TODO: handle scenario where a high priority flow failing to get // schedule can starve all others ExecutorManager.this.queuedFlows.enqueue(exflow, reference); } }
private void assertFound(final Map<Integer, Pair<ExecutionReference, ExecutableFlow>> activeFlows, final ExecutableFlow flow, final boolean executorPresent) { assertThat(activeFlows.containsKey(flow.getExecutionId())).isTrue(); assertThat(activeFlows.get(flow.getExecutionId()).getFirst().getExecutor().isPresent()) .isEqualTo(executorPresent); }
Map<String, Object> callWithExecutable(final ExecutableFlow exflow, final Executor executor, final String action) throws ExecutorManagerException { return callWithExecutionId(executor.getHost(), executor.getPort(), action, exflow.getExecutionId(), null, (Pair<String, String>[]) null); }
private void assertNotFound( final Map<Integer, Pair<ExecutionReference, ExecutableFlow>> activeFlows, final ExecutableFlow flow, final String failMessage) { assertThat(activeFlows.containsKey(flow.getExecutionId())).withFailMessage(failMessage) .isFalse(); }
private void initializeUnfinishedFlows() throws Exception { this.unfinishedFlows = ImmutableMap .of(this.flow1.getExecutionId(), new Pair<>(this.ref1, this.flow1), this.flow2.getExecutionId(), new Pair<>(this.ref2, this.flow2), this.flow3.getExecutionId(), new Pair<>(this.ref3, this.flow3)); when(this.loader.fetchUnfinishedFlows()).thenReturn(this.unfinishedFlows); } }
@Override public void alertOnSuccess(final ExecutableFlow flow) { final EmailMessage message = this.messageCreator.createMessage(); final MailCreator mailCreator = getMailCreator(flow); final boolean mailCreated = mailCreator.createSuccessEmail(flow, message, this.azkabanName, this.scheme, this.clientHostname, this.clientPortNumber); sendEmail(message, mailCreated, "success email message for execution " + flow.getExecutionId()); }
@Override public void updateExecutableFlow(final ExecutableFlow flow) throws ExecutorManagerException { final ExecutableFlow toUpdate = this.flows.get(flow.getExecutionId()); toUpdate.applyUpdateObject(flow.toUpdateObject(0)); this.flowUpdateCount++; }
private void submitFlow(final ExecutableFlow flow, final ExecutionReference ref) throws Exception { when(this.loader.fetchUnfinishedFlows()).thenReturn(this.unfinishedFlows); when(this.loader.fetchExecutableFlow(flow.getExecutionId())).thenReturn(flow); this.controller.submitExecutableFlow(flow, this.user.getUserId()); this.unfinishedFlows.put(flow.getExecutionId(), new Pair<>(ref, flow)); }
@Override public void uploadExecutableFlow(final ExecutableFlow flow) throws ExecutorManagerException { // Clone the flow node to mimick how it would be saved in DB. // If we would keep a handle to the original flow node, we would also see any changes made after // this method was called. We must only store a snapshot of the current state. // Also to avoid modifying statuses of the original job nodes in this.updateExecutableFlow() final ExecutableFlow exFlow = ExecutableFlow.createExecutableFlowFromObject(flow.toObject()); this.flows.put(flow.getExecutionId(), exFlow); this.flowUpdateCount++; }
@Test public void testFetchAllQueuedFlowIds() throws Exception { assertThat(this.controller.getQueuedFlowIds()) .isEqualTo(ImmutableList.of(this.flow1.getExecutionId())); }
@Test public void testFetchActiveFlowByProject() throws Exception { initializeUnfinishedFlows(); final List<Integer> executions = this.controller .getRunningFlows(this.flow2.getProjectId(), this.flow2.getFlowId()); assertThat(executions.contains(this.flow2.getExecutionId())).isTrue(); assertThat(executions.contains(this.flow3.getExecutionId())).isTrue(); assertThat(this.controller.isFlowRunning(this.flow2.getProjectId(), this.flow2.getFlowId())) .isTrue(); assertThat(this.controller.isFlowRunning(this.flow3.getProjectId(), this.flow3.getFlowId())) .isTrue(); }
@Test public void testFetchUnfinishedFlows() throws Exception { final List<ExecutableFlow> flows = createExecutions(); final Map<Integer, Pair<ExecutionReference, ExecutableFlow>> unfinishedFlows = this.fetchActiveFlowDao.fetchUnfinishedFlows(); assertFound(unfinishedFlows, flows.get(0), true); assertFound(unfinishedFlows, flows.get(1), false); assertFound(unfinishedFlows, flows.get(2), true); assertNotFound(unfinishedFlows, flows.get(3), "Returned an execution with a finished status"); assertFound(unfinishedFlows, flows.get(4), false); assertTwoFlowSame(unfinishedFlows.get(flows.get(0).getExecutionId()).getSecond(), flows.get(0)); }
@Test public void testFetchActiveFlowsExecutorAssigned() throws Exception { final List<ExecutableFlow> flows = createExecutions(); final Map<Integer, Pair<ExecutionReference, ExecutableFlow>> activeFlows = this.fetchActiveFlowDao .fetchActiveFlows(); assertFound(activeFlows, flows.get(0), true); assertNotFound(activeFlows, flows.get(1), "Returned a queued execution"); assertFound(activeFlows, flows.get(2), true); assertNotFound(activeFlows, flows.get(3), "Returned an execution with a finished status"); assertFound(activeFlows, flows.get(4), false); assertTwoFlowSame(activeFlows.get(flows.get(0).getExecutionId()).getSecond(), flows.get(0)); }
@Ignore @Test public void testFetchActiveFlowWithExecutor() throws Exception { testSetUpForRunningFlows(); final List<Pair<ExecutableFlow, Optional<Executor>>> activeFlowsWithExecutor = this.manager.getActiveFlowsWithExecutor(); Assert.assertTrue(activeFlowsWithExecutor.contains(new Pair<>(this.flow1, Optional.ofNullable(this.manager.fetchExecutor(this.flow1.getExecutionId()))))); Assert.assertTrue(activeFlowsWithExecutor.contains(new Pair<>(this.flow2, Optional.ofNullable(this.manager.fetchExecutor(this.flow2.getExecutionId()))))); }
@Test public void testAdvancedFilter() throws Exception { createTestProject(); final ExecutableFlow flow = createTestFlow(); this.executionFlowDao.uploadExecutableFlow(flow); final List<ExecutableFlow> flowList1 = this.executionFlowDao .fetchFlowHistory("exectest1", "", "", 0, -1, -1, 0, 16); assertThat(flowList1.size()).isEqualTo(1); final ExecutableFlow fetchFlow = this.executionFlowDao.fetchExecutableFlow(flow.getExecutionId()); assertTwoFlowSame(flowList1.get(0), fetchFlow); }
@Test public void testAssignExecutorInvalidExecutor() throws Exception { final ExecutableFlow flow = TestUtils.createTestExecutableFlow("exectest1", "exec1"); this.executionFlowDao.uploadExecutableFlow(flow); // Since we haven't inserted any executors, 1 should be non-existent executor id. assertThatThrownBy( () -> this.assignExecutor.assignExecutor(1, flow.getExecutionId())) .isInstanceOf(ExecutorManagerException.class) .hasMessageContaining("non-existent executor"); }
@Ignore @Test public void testFetchActiveFlowByProject() throws Exception { testSetUpForRunningFlows(); final List<Integer> executions = this.manager.getRunningFlows(this.flow1.getProjectId(), this.flow1.getFlowId()); Assert.assertTrue(executions.contains(this.flow1.getExecutionId())); Assert .assertTrue(this.manager.isFlowRunning(this.flow1.getProjectId(), this.flow1.getFlowId())); }
@Test public void testKillQueuedFlow() throws Exception { final ExecutorManager manager = createMultiExecutorManagerInstance(); final ExecutableFlow flow1 = TestUtils.createTestExecutableFlow("exectest1", "exec1"); final User testUser = TestUtils.getTestUser(); manager.submitExecutableFlow(flow1, testUser.getUserId()); manager.cancelFlow(flow1, testUser.getUserId()); final ExecutableFlow fetchedFlow = this.loader.fetchExecutableFlow(flow1.getExecutionId()); Assert.assertEquals(fetchedFlow.getStatus(), Status.FAILED); Assert.assertFalse(manager.getRunningFlows().contains(flow1)); }