@Override public void resetForRetry() { super.resetForRetry(); this.setStatus(Status.RUNNING); }
private static Pair<ExecutionReference, ExecutableFlow> getExecutableFlowMetadataHelper( final ResultSet rs) throws SQLException { final Flow flow = new Flow(rs.getString("flow_id")); final Project project = new Project(rs.getInt("project_id"), null); project.setVersion(rs.getInt("version")); final ExecutableFlow exFlow = new ExecutableFlow(project, flow); exFlow.setExecutionId(rs.getInt("exec_id")); exFlow.setStatus(Status.fromInteger(rs.getInt("status"))); exFlow.setSubmitTime(rs.getLong("submit_time")); exFlow.setStartTime(rs.getLong("start_time")); exFlow.setEndTime(rs.getLong("end_time")); exFlow.setSubmitUser(rs.getString("submit_user")); return getPairWithExecutorInfo(rs, exFlow); }
@Test public void testKillQueuedFlow() throws Exception { // Flow1 is not assigned to any executor and is in PREPARING status. submitFlow(this.flow1, this.ref1); this.flow1.setStatus(Status.PREPARING); this.controller.cancelFlow(this.flow1, this.user.getUserId()); // Verify that the status of flow1 is finalized. assertThat(this.flow1.getStatus()).isEqualTo(Status.FAILED); this.flow1.getExecutableNodes().forEach(node -> { assertThat(node.getStatus()).isEqualTo(Status.KILLING); }); }
@Test public void testFetchNumExecutableFlows() throws Exception { final ExecutableFlow flow1 = TestUtils.createTestExecutableFlow("exectest1", "exec1"); flow1.setStatus(Status.PREPARING); this.executionFlowDao.uploadExecutableFlow(flow1); final ExecutableFlow flow2 = TestUtils.createTestExecutableFlow("exectest1", "exec2"); flow2.setStatus(Status.RUNNING); this.executionFlowDao.uploadExecutableFlow(flow2); final ExecutableFlow flow2b = TestUtils.createTestExecutableFlow("exectest1", "exec2"); flow2b.setStatus(Status.FAILED); this.executionFlowDao.uploadExecutableFlow(flow2b); final int count = this.numExecutionsDao.fetchNumExecutableFlows(); assertThat(count).isEqualTo(3); final int flow2Count = this.numExecutionsDao .fetchNumExecutableFlows(1, "derived-member-data-2"); assertThat(flow2Count).isEqualTo(2); }
@Test public void testFetchRecentlyFinishedFlows() throws Exception { final ExecutableFlow flow1 = createTestFlow(); this.executionFlowDao.uploadExecutableFlow(flow1); flow1.setStatus(Status.SUCCEEDED); flow1.setEndTime(System.currentTimeMillis()); this.executionFlowDao.updateExecutableFlow(flow1); //Flow just finished. Fetch recently finished flows immediately. Should get it. final List<ExecutableFlow> flows = this.executionFlowDao.fetchRecentlyFinishedFlows( RECENTLY_FINISHED_LIFETIME); assertThat(flows.size()).isEqualTo(1); assertTwoFlowSame(flow1, flows.get(0)); }
@Test public void testFetchQueuedFlows() throws Exception { final ExecutableFlow flow = createTestFlow(); flow.setStatus(Status.PREPARING); this.executionFlowDao.uploadExecutableFlow(flow); final ExecutableFlow flow2 = TestUtils.createTestExecutableFlow("exectest1", "exec2"); flow2.setStatus(Status.PREPARING); this.executionFlowDao.uploadExecutableFlow(flow2); final List<Pair<ExecutionReference, ExecutableFlow>> fetchedQueuedFlows = this.executionFlowDao .fetchQueuedFlows(); assertThat(fetchedQueuedFlows.size()).isEqualTo(2); final Pair<ExecutionReference, ExecutableFlow> fetchedFlow1 = fetchedQueuedFlows.get(0); final Pair<ExecutionReference, ExecutableFlow> fetchedFlow2 = fetchedQueuedFlows.get(1); assertTwoFlowSame(flow, fetchedFlow1.getSecond()); assertTwoFlowSame(flow2, fetchedFlow2.getSecond()); }
@Before public void setUp() throws Exception { this.props = new Props(); this.props.put(ConfigurationKeys.AZKABAN_EXECUTOR_MAX_FAILURE_COUNT, 2); this.props.put(ConfigurationKeys.AZKABAN_ADMIN_ALERT_EMAIL, AZ_ADMIN_ALERT_EMAIL); this.loader = mock(ExecutorLoader.class); this.mailAlerter = mock(Alerter.class); this.alerterHolder = mock(AlerterHolder.class); this.apiGateway = mock(ExecutorApiGateway.class); this.executorHealthChecker = new ExecutorHealthChecker(this.props, this.loader, this .apiGateway, this.alerterHolder); this.flow1 = TestUtils.createTestExecutableFlow("exectest1", "exec1"); this.flow1.setExecutionId(EXECUTION_ID_11); this.flow1.setStatus(Status.RUNNING); this.executor1 = new Executor(1, "localhost", 12345, true); when(this.loader.fetchActiveFlows()).thenReturn(this.activeFlows); when(this.alerterHolder.get("email")).thenReturn(this.mailAlerter); }
@Test public void testFetchActiveFlowsStatusChanged() throws Exception { final ExecutableFlow flow1 = TestUtils.createTestExecutableFlow("exectest1", "exec1"); this.executionFlowDao.uploadExecutableFlow(flow1); final Executor executor = this.executorDao.addExecutor("test", 1); this.assignExecutor.assignExecutor(executor.getId(), flow1.getExecutionId()); Map<Integer, Pair<ExecutionReference, ExecutableFlow>> activeFlows1 = this.fetchActiveFlowDao.fetchActiveFlows(); assertThat(activeFlows1.containsKey(flow1.getExecutionId())).isTrue(); // When flow status becomes SUCCEEDED/KILLED/FAILED, it should not be in active state flow1.setStatus(Status.SUCCEEDED); this.executionFlowDao.updateExecutableFlow(flow1); activeFlows1 = this.fetchActiveFlowDao.fetchActiveFlows(); assertThat(activeFlows1.containsKey(flow1.getExecutionId())).isFalse(); flow1.setStatus(Status.KILLED); this.executionFlowDao.updateExecutableFlow(flow1); activeFlows1 = this.fetchActiveFlowDao.fetchActiveFlows(); assertThat(activeFlows1.containsKey(flow1.getExecutionId())).isFalse(); flow1.setStatus(Status.FAILED); this.executionFlowDao.updateExecutableFlow(flow1); activeFlows1 = this.fetchActiveFlowDao.fetchActiveFlows(); assertThat(activeFlows1.containsKey(flow1.getExecutionId())).isFalse(); }
@Test public void testKillRunningFlow() throws Exception { // Flow2 is assigned to executor2 and is in RUNNING status. submitFlow(this.flow2, this.ref2); this.flow2.setStatus(Status.RUNNING); this.controller.cancelFlow(this.flow2, this.user.getUserId()); // Verify that executor is called to cancel flow2. verify(this.apiGateway).callWithReferenceByUser(this.ref2, ConnectorParams.CANCEL_ACTION, this.user.getUserId()); }
public synchronized void uploadExecutableFlow(final ExecutableFlow flow) throws ExecutorManagerException { final String INSERT_EXECUTABLE_FLOW = "INSERT INTO execution_flows " + "(project_id, flow_id, version, status, submit_time, submit_user, update_time) " + "values (?,?,?,?,?,?,?)"; final long submitTime = System.currentTimeMillis(); flow.setStatus(Status.PREPARING); flow.setSubmitTime(submitTime); /** * Why we need a transaction to get last insert ID? * Because "SELECT LAST_INSERT_ID()" needs to have the same connection * as inserting the new entry. * See https://dev.mysql.com/doc/refman/5.7/en/information-functions.html#function_last-insert-id */ final SQLTransaction<Long> insertAndGetLastID = transOperator -> { transOperator.update(INSERT_EXECUTABLE_FLOW, flow.getProjectId(), flow.getFlowId(), flow.getVersion(), Status.PREPARING.getNumVal(), submitTime, flow.getSubmitUser(), submitTime); transOperator.getConnection().commit(); return transOperator.getLastInsertId(); }; try { final long id = this.dbOperator.transaction(insertAndGetLastID); logger.info("Flow given " + flow.getFlowId() + " given id " + id); flow.setExecutionId((int) id); updateExecutableFlow(flow); } catch (final SQLException e) { throw new ExecutorManagerException("Error creating execution.", e); } }
exFlow.setStatus(Status.FAILED);
private ExecutableFlow createExecution(final Status status) throws IOException, ExecutorManagerException { final ExecutableFlow flow = TestUtils.createTestExecutableFlow("exectest1", "exec1"); this.executionFlowDao.uploadExecutableFlow(flow); assertEquals(Status.PREPARING, flow.getStatus()); flow.setStatus(status); this.executionFlowDao.updateExecutableFlow(flow); return flow; }
@Test public void testFetchEmptyRecentlyFinishedFlows() throws Exception { final ExecutableFlow flow1 = createTestFlow(); this.executionFlowDao.uploadExecutableFlow(flow1); flow1.setStatus(Status.SUCCEEDED); flow1.setEndTime(DateTimeUtils.currentTimeMillis()); this.executionFlowDao.updateExecutableFlow(flow1); //Todo jamiesjc: use java8.java.time api instead of jodatime //Mock flow finished time to be 2 min ago. DateTimeUtils.setCurrentMillisOffset(-FLOW_FINISHED_TIME.toMillis()); flow1.setEndTime(DateTimeUtils.currentTimeMillis()); this.executionFlowDao.updateExecutableFlow(flow1); //Fetch recently finished flows within 1 min. Should be empty. final List<ExecutableFlow> flows = this.executionFlowDao .fetchRecentlyFinishedFlows(RECENTLY_FINISHED_LIFETIME); assertThat(flows.size()).isEqualTo(0); }
@Test public void createErrorEmail() throws Exception { setJobStatus(Status.FAILED); this.executableFlow.setEndTime(END_TIME_MILLIS); this.executableFlow.setStatus(Status.FAILED); final List<ExecutableFlow> executableFlows = new ArrayList<>(); final ExecutableFlow executableFlow1 = new ExecutableFlow(this.project, this.flow); executableFlow1.setExecutionId(1); executableFlow1.setStartTime(START_TIME_MILLIS); executableFlow1.setEndTime(END_TIME_MILLIS); executableFlow1.setStatus(Status.FAILED); executableFlows.add(executableFlow1); final ExecutableFlow executableFlow2 = new ExecutableFlow(this.project, this.flow); executableFlow2.setExecutionId(2); executableFlow2.setStartTime(START_TIME_MILLIS); executableFlow2.setEndTime(END_TIME_MILLIS); executableFlow2.setStatus(Status.SUCCEEDED); executableFlows.add(executableFlow2); assertTrue(this.mailCreator.createErrorEmail( this.executableFlow, executableFlows, this.message, this.azkabanName, this.scheme, this .clientHostname, this.clientPortNumber)); assertEquals("Flow 'mail-creator-test' has failed on unit-tests", this.message.getSubject()); assertThat(TestUtils.readResource("errorEmail.html", this)) .isEqualToIgnoringWhitespace(this.message.getBody()); }
@Test public void createFirstErrorMessage() throws Exception { setJobStatus(Status.FAILED); this.executableFlow.setStatus(Status.FAILED_FINISHING); assertTrue(this.mailCreator.createFirstErrorMessage( this.executableFlow, this.message, this.azkabanName, this.scheme, this.clientHostname, this.clientPortNumber)); assertEquals("Flow 'mail-creator-test' has encountered a failure on unit-tests", this.message.getSubject()); assertThat(TestUtils.readResource("firstErrorMessage.html", this)) .isEqualToIgnoringWhitespace(this.message.getBody()); }
@Test public void createSuccessEmail() throws Exception { setJobStatus(Status.SUCCEEDED); this.executableFlow.setEndTime(END_TIME_MILLIS); this.executableFlow.setStatus(Status.SUCCEEDED); assertTrue(this.mailCreator.createSuccessEmail( this.executableFlow, this.message, this.azkabanName, this.scheme, this.clientHostname, this.clientPortNumber)); assertEquals("Flow 'mail-creator-test' has succeeded on unit-tests", this.message.getSubject()); assertThat(TestUtils.readResource("successEmail.html", this)) .isEqualToIgnoringWhitespace(this.message.getBody()); }
@Test public void testUpdateExecutableFlow() throws Exception { final ExecutableFlow flow = createTestFlow(); this.executionFlowDao.uploadExecutableFlow(flow); final ExecutableFlow fetchFlow = this.executionFlowDao.fetchExecutableFlow(flow.getExecutionId()); fetchFlow.setEndTime(System.currentTimeMillis()); fetchFlow.setStatus(Status.SUCCEEDED); this.executionFlowDao.updateExecutableFlow(fetchFlow); final ExecutableFlow fetchFlow2 = this.executionFlowDao.fetchExecutableFlow(flow.getExecutionId()); assertTwoFlowSame(fetchFlow, fetchFlow2); }
exFlow.setStatus(Status.RUNNING); exFlow.setUpdateTime(133);
private ExecutableFlow getExecutableFlowMetadata( ExecutableFlow fullExFlow) { final Flow flow = new Flow(fullExFlow.getId()); final Project project = new Project(fullExFlow.getProjectId(), null); project.setVersion(fullExFlow.getVersion()); flow.setVersion(fullExFlow.getVersion()); final ExecutableFlow metadata = new ExecutableFlow(project, flow); metadata.setExecutionId(fullExFlow.getExecutionId()); metadata.setStatus(fullExFlow.getStatus()); metadata.setSubmitTime(fullExFlow.getSubmitTime()); metadata.setStartTime(fullExFlow.getStartTime()); metadata.setEndTime(fullExFlow.getEndTime()); metadata.setSubmitUser(fullExFlow.getSubmitUser()); return metadata; }
public void kill(String user) { synchronized(mainSyncObj) { logger.info("Flow killed by " + user); flow.setStatus(Status.KILLED); kill(); updateFlow(); } interrupt(); }