private boolean isFlowRunningHelper(final int projectId, final String flowId, final Collection<Pair<ExecutionReference, ExecutableFlow>> collection) { for (final Pair<ExecutionReference, ExecutableFlow> ref : collection) { if (ref.getSecond().getProjectId() == projectId && ref.getSecond().getFlowId().equals(flowId)) { return true; } } return false; }
private boolean isFlowRunningHelper(final int projectId, final String flowId, final Collection<Pair<ExecutionReference, ExecutableFlow>> collection) { for (final Pair<ExecutionReference, ExecutableFlow> ref : collection) { if (ref.getSecond().getProjectId() == projectId && ref.getSecond().getFlowId().equals(flowId)) { return true; } } return false; }
private void ajaxFetchExecutableFlow(final HttpServletRequest req, final HttpServletResponse resp, final HashMap<String, Object> ret, final User user, final ExecutableFlow exFlow) throws ServletException { logger.info("Fetching " + exFlow.getExecutionId()); final Project project = getProjectAjaxByPermission(ret, exFlow.getProjectId(), user, Type.READ); if (project == null) { return; } ret.put("submitTime", exFlow.getSubmitTime()); ret.put("submitUser", exFlow.getSubmitUser()); ret.put("execid", exFlow.getExecutionId()); ret.put("projectId", exFlow.getProjectId()); ret.put("project", project.getName()); final Map<String, Object> flowObj = getExecutableNodeInfo(exFlow); ret.putAll(flowObj); }
private List<Integer> getRunningFlowsHelper(final int projectId, final String flowId, final Collection<Pair<ExecutionReference, ExecutableFlow>> collection) { final List<Integer> executionIds = new ArrayList<>(); for (final Pair<ExecutionReference, ExecutableFlow> ref : collection) { if (ref.getSecond().getFlowId().equals(flowId) && ref.getSecond().getProjectId() == projectId) { executionIds.add(ref.getFirst().getExecId()); } } return executionIds; }
private List<Integer> getRunningFlowsHelper(final int projectId, final String flowId, final Collection<Pair<ExecutionReference, ExecutableFlow>> collection) { final List<Integer> executionIds = new ArrayList<>(); for (final Pair<ExecutionReference, ExecutableFlow> ref : collection) { if (ref.getSecond().getFlowId().equals(flowId) && ref.getSecond().getProjectId() == projectId) { executionIds.add(ref.getFirst().getExecId()); } } return executionIds; }
private void ajaxFetchExecutableFlowUpdate(final HttpServletRequest req, final HttpServletResponse resp, final HashMap<String, Object> ret, final User user, final ExecutableFlow exFlow) throws ServletException { final Long lastUpdateTime = Long.parseLong(getParam(req, "lastUpdateTime")); logger.info("Fetching " + exFlow.getExecutionId()); final Project project = getProjectAjaxByPermission(ret, exFlow.getProjectId(), user, Type.READ); if (project == null) { return; } final Map<String, Object> map = getExecutableFlowUpdateInfo(exFlow, lastUpdateTime); map.put("status", exFlow.getStatus()); map.put("startTime", exFlow.getStartTime()); map.put("endTime", exFlow.getEndTime()); map.put("updateTime", exFlow.getUpdateTime()); ret.putAll(map); }
private void cleanUpProjectOldInstallations(final Project project) throws ProjectManagerException, ExecutorManagerException { log.info("Cleaning up old install files older than " + (project.getVersion() - this.projectVersionRetention)); final Map<Integer, Pair<ExecutionReference, ExecutableFlow>> unfinishedFlows = this.executorLoader .fetchUnfinishedFlowsMetadata(); final List<Integer> versionsWithUnfinishedExecutions = unfinishedFlows.values() .stream().map(pair -> pair.getSecond()) .filter(exflow -> exflow.getProjectId() == project.getId()) .map(exflow -> exflow.getVersion()) .collect(Collectors.toList()); this.projectLoader.cleanOlderProjectVersion(project.getId(), project.getVersion() - this.projectVersionRetention, versionsWithUnfinishedExecutions); // Clean up storage this.storageManager.cleanupProjectArtifacts(project.getId()); }
private void ajaxCancelFlow(final HttpServletRequest req, final HttpServletResponse resp, final HashMap<String, Object> ret, final User user, final ExecutableFlow exFlow) throws ServletException { final Project project = getProjectAjaxByPermission(ret, exFlow.getProjectId(), user, Type.EXECUTE); if (project == null) { return; } try { this.executorManagerAdapter.cancelFlow(exFlow, user.getUserId()); } catch (final ExecutorManagerException e) { ret.put("error", e.getMessage()); } }
private void ajaxPauseFlow(final HttpServletRequest req, final HttpServletResponse resp, final HashMap<String, Object> ret, final User user, final ExecutableFlow exFlow) throws ServletException { final Project project = getProjectAjaxByPermission(ret, exFlow.getProjectId(), user, Type.EXECUTE); if (project == null) { return; } try { this.executorManagerAdapter.pauseFlow(exFlow, user.getUserId()); } catch (final ExecutorManagerException e) { ret.put("error", e.getMessage()); } }
private void ajaxResumeFlow(final HttpServletRequest req, final HttpServletResponse resp, final HashMap<String, Object> ret, final User user, final ExecutableFlow exFlow) throws ServletException { final Project project = getProjectAjaxByPermission(ret, exFlow.getProjectId(), user, Type.EXECUTE); if (project == null) { return; } try { this.executorManagerAdapter.resumeFlow(exFlow, user.getUserId()); } catch (final ExecutorManagerException e) { ret.put("resume", e.getMessage()); } }
private void ajaxFetchJobStats(final HttpServletRequest req, final HttpServletResponse resp, final HashMap<String, Object> ret, final User user, final ExecutableFlow exFlow) throws ServletException { final Project project = getProjectAjaxByPermission(ret, exFlow.getProjectId(), user, Type.READ); if (project == null) { return; } final String jobId = this.getParam(req, "jobid"); resp.setCharacterEncoding("utf-8"); try { final ExecutableNode node = exFlow.getExecutableNodePath(jobId); if (node == null) { ret.put("error", "Job " + jobId + " doesn't exist in " + exFlow.getExecutionId()); return; } final List<Object> jsonObj = this.executorManagerAdapter .getExecutionJobStats(exFlow, jobId, node.getAttempt()); ret.put("jobStats", jsonObj); } catch (final ExecutorManagerException e) { ret.put("error", "Error retrieving stats for job " + jobId); return; } }
private void ajaxRestartFailed(final HttpServletRequest req, final HttpServletResponse resp, final HashMap<String, Object> ret, final User user, final ExecutableFlow exFlow) throws ServletException { final Project project = getProjectAjaxByPermission(ret, exFlow.getProjectId(), user, Type.EXECUTE); if (project == null) { return; } if (exFlow.getStatus() == Status.FAILED || exFlow.getStatus() == Status.SUCCEEDED) { ret.put("error", "Flow has already finished. Please re-execute."); return; } try { this.executorManagerAdapter.retryFailures(exFlow, user.getUserId()); } catch (final ExecutorManagerException e) { ret.put("error", e.getMessage()); } }
@Test public void testFetchActiveFlowByProject() throws Exception { initializeUnfinishedFlows(); final List<Integer> executions = this.controller .getRunningFlows(this.flow2.getProjectId(), this.flow2.getFlowId()); assertThat(executions.contains(this.flow2.getExecutionId())).isTrue(); assertThat(executions.contains(this.flow3.getExecutionId())).isTrue(); assertThat(this.controller.isFlowRunning(this.flow2.getProjectId(), this.flow2.getFlowId())) .isTrue(); assertThat(this.controller.isFlowRunning(this.flow3.getProjectId(), this.flow3.getFlowId())) .isTrue(); }
ret.put("submitUser", exFlow.getSubmitUser()); ret.put("execid", exFlow.getExecutionId()); ret.put("projectId", exFlow.getProjectId()); ret.put("project", project.getName()); FlowUtils.applyDisabledJobs(executionOptions.getDisabledJobs(), exFlow);
@Override public void alertOnError(final ExecutableFlow flow, final String... extraReasons) { final EmailMessage message = this.messageCreator.createMessage(); final MailCreator mailCreator = getMailCreator(flow); List<ExecutableFlow> last72hoursExecutions = new ArrayList<>(); if (flow.getStartTime() > 0) { final long startTime = flow.getStartTime() - Duration.ofHours(72).toMillis(); try { last72hoursExecutions = this.executorLoader.fetchFlowHistory(flow.getProjectId(), flow .getFlowId(), startTime); } catch (final ExecutorManagerException e) { logger.error("unable to fetch past executions", e); } } final boolean mailCreated = mailCreator.createErrorEmail(flow, last72hoursExecutions, message, this.azkabanName, this.scheme, this.clientHostname, this.clientPortNumber, extraReasons); sendEmail(message, mailCreated, "error email message for execution " + flow.getExecutionId()); }
@Test public void fetchFlowHistory() throws Exception { final ExecutableFlow flow = createTestFlow(); this.executionFlowDao.uploadExecutableFlow(flow); final List<ExecutableFlow> flowList1 = this.executionFlowDao.fetchFlowHistory(0, 2); assertThat(flowList1.size()).isEqualTo(1); final List<ExecutableFlow> flowList2 = this.executionFlowDao .fetchFlowHistory(flow.getProjectId(), flow.getId(), 0, 2); assertThat(flowList2.size()).isEqualTo(1); final ExecutableFlow fetchFlow = this.executionFlowDao.fetchExecutableFlow(flow.getExecutionId()); assertTwoFlowSame(flowList1.get(0), flowList2.get(0)); assertTwoFlowSame(flowList1.get(0), fetchFlow); }
private void assertTwoFlowSame(final ExecutableFlow flow1, final ExecutableFlow flow2, final boolean compareFlowData) { assertThat(flow1.getExecutionId()).isEqualTo(flow2.getExecutionId()); assertThat(flow1.getStatus()).isEqualTo(flow2.getStatus()); assertThat(flow1.getEndTime()).isEqualTo(flow2.getEndTime()); assertThat(flow1.getStartTime()).isEqualTo(flow2.getStartTime()); assertThat(flow1.getSubmitTime()).isEqualTo(flow2.getSubmitTime()); assertThat(flow1.getFlowId()).isEqualTo(flow2.getFlowId()); assertThat(flow1.getProjectId()).isEqualTo(flow2.getProjectId()); assertThat(flow1.getVersion()).isEqualTo(flow2.getVersion()); assertThat(flow1.getSubmitUser()).isEqualTo(flow2.getSubmitUser()); if (compareFlowData) { assertThat(flow1.getExecutionOptions().getFailureAction()) .isEqualTo(flow2.getExecutionOptions().getFailureAction()); assertThat(new HashSet<>(flow1.getEndNodes())).isEqualTo(new HashSet<>(flow2.getEndNodes())); } }
@Ignore @Test public void testFetchActiveFlowByProject() throws Exception { testSetUpForRunningFlows(); final List<Integer> executions = this.manager.getRunningFlows(this.flow1.getProjectId(), this.flow1.getFlowId()); Assert.assertTrue(executions.contains(this.flow1.getExecutionId())); Assert .assertTrue(this.manager.isFlowRunning(this.flow1.getProjectId(), this.flow1.getFlowId())); }
@Test public void testCreateExecutableFlow() throws Exception { final Flow flow = FlowUtils.getFlow(this.project, "jobe"); final ExecutableFlow exFlow = FlowUtils.createExecutableFlow(this.project, flow); Assert.assertEquals(exFlow.getProjectId(), this.project.getId()); Assert.assertEquals(exFlow.getFlowId(), flow.getId()); Assert.assertEquals(exFlow.getProxyUsers(), this.project.getProxyUsers()); }
private ExecutableFlow getExecutableFlowMetadata( ExecutableFlow fullExFlow) { final Flow flow = new Flow(fullExFlow.getId()); final Project project = new Project(fullExFlow.getProjectId(), null); project.setVersion(fullExFlow.getVersion()); flow.setVersion(fullExFlow.getVersion()); final ExecutableFlow metadata = new ExecutableFlow(project, flow); metadata.setExecutionId(fullExFlow.getExecutionId()); metadata.setStatus(fullExFlow.getStatus()); metadata.setSubmitTime(fullExFlow.getSubmitTime()); metadata.setStartTime(fullExFlow.getStartTime()); metadata.setEndTime(fullExFlow.getEndTime()); metadata.setSubmitUser(fullExFlow.getSubmitUser()); return metadata; }