private ExecutableFlow getExecutableFlowMetadata( ExecutableFlow fullExFlow) { final Flow flow = new Flow(fullExFlow.getId()); final Project project = new Project(fullExFlow.getProjectId(), null); project.setVersion(fullExFlow.getVersion()); flow.setVersion(fullExFlow.getVersion()); final ExecutableFlow metadata = new ExecutableFlow(project, flow); metadata.setExecutionId(fullExFlow.getExecutionId()); metadata.setStatus(fullExFlow.getStatus()); metadata.setSubmitTime(fullExFlow.getSubmitTime()); metadata.setStartTime(fullExFlow.getStartTime()); metadata.setEndTime(fullExFlow.getEndTime()); metadata.setSubmitUser(fullExFlow.getSubmitUser()); return metadata; }
for (final ExecutableNode node : exFlow.getExecutableNodes()) { switch (node.getStatus()) { case SUCCEEDED: if (exFlow.getEndTime() == -1) { exFlow.setEndTime(time); exFlow.setStatus(Status.FAILED);
public String submitExecutableFlow(final ExecutableFlow exflow, final String userId) throws ExecutorManagerException { final String exFlowKey = exflow.getProjectName() + "." + exflow.getId() + ".submitFlow"; final String flowId = exflow.getFlowId(); logger.info("Submitting execution flow " + flowId + " by " + userId); final int projectId = exflow.getProjectId(); exflow.setSubmitUser(userId); exflow.setSubmitTime(System.currentTimeMillis()); ExecutionOptions options = exflow.getExecutionOptions(); if (options == null) { options = new ExecutionOptions(); !ProjectWhitelist.isProjectWhitelisted(exflow.getProjectId(), ProjectWhitelist.WhitelistType.MemoryCheck); options.setMemoryCheck(memoryCheck); message += "Execution queued successfully with exec id " + exflow.getExecutionId(); return message;
public static ExecutableFlow createExecutableFlowFromObject(final Object obj) { final ExecutableFlow exFlow = new ExecutableFlow(); final HashMap<String, Object> flowObj = (HashMap<String, Object>) obj; exFlow.fillExecutableFromMapObject(flowObj); return exFlow; }
public static ExecutableFlow createExecutableFlow(final Project project, final Flow flow) { final ExecutableFlow exflow = new ExecutableFlow(project, flow); exflow.addAllProxyUsers(project.getProxyUsers()); return exflow; }
private void ajaxFetchExecutableFlowUpdate(final HttpServletRequest req, final HttpServletResponse resp, final HashMap<String, Object> ret, final User user, final ExecutableFlow exFlow) throws ServletException { final Long lastUpdateTime = Long.parseLong(getParam(req, "lastUpdateTime")); logger.info("Fetching " + exFlow.getExecutionId()); final Project project = getProjectAjaxByPermission(ret, exFlow.getProjectId(), user, Type.READ); if (project == null) { return; } final Map<String, Object> map = getExecutableFlowUpdateInfo(exFlow, lastUpdateTime); map.put("status", exFlow.getStatus()); map.put("startTime", exFlow.getStartTime()); map.put("endTime", exFlow.getEndTime()); map.put("updateTime", exFlow.getUpdateTime()); ret.putAll(map); }
private void assertTwoFlowSame(final ExecutableFlow flow1, final ExecutableFlow flow2, final boolean compareFlowData) { assertThat(flow1.getExecutionId()).isEqualTo(flow2.getExecutionId()); assertThat(flow1.getStatus()).isEqualTo(flow2.getStatus()); assertThat(flow1.getEndTime()).isEqualTo(flow2.getEndTime()); assertThat(flow1.getStartTime()).isEqualTo(flow2.getStartTime()); assertThat(flow1.getSubmitTime()).isEqualTo(flow2.getSubmitTime()); assertThat(flow1.getFlowId()).isEqualTo(flow2.getFlowId()); assertThat(flow1.getProjectId()).isEqualTo(flow2.getProjectId()); assertThat(flow1.getVersion()).isEqualTo(flow2.getVersion()); assertThat(flow1.getSubmitUser()).isEqualTo(flow2.getSubmitUser()); if (compareFlowData) { assertThat(flow1.getExecutionOptions().getFailureAction()) .isEqualTo(flow2.getExecutionOptions().getFailureAction()); assertThat(new HashSet<>(flow1.getEndNodes())).isEqualTo(new HashSet<>(flow2.getEndNodes())); } }
final ExecutionOptions option = flow.getExecutionOptions(); final List<String> emailList = option.getSuccessEmails(); final int execId = flow.getExecutionId(); message.setSubject("Flow '" + flow.getFlowId() + "' has succeeded on " + azkabanName); message.println("<h2> Execution '" + flow.getExecutionId() + "' of flow '" + flow.getFlowId() + "' of project '" + flow.getProjectName() + "' has succeeded on " + azkabanName + "</h2>"); message.println("<table>"); message.println("<tr><td>Start Time</td><td>" + convertMSToString(flow.getStartTime()) + "</td></tr>"); message.println("<tr><td>End Time</td><td>" + convertMSToString(flow.getEndTime()) + "</td></tr>"); message.println("<tr><td>Duration</td><td>" + Utils.formatDuration(flow.getStartTime(), flow.getEndTime()) + "</td></tr>"); message.println("<tr><td>Status</td><td>" + flow.getStatus() + "</td></tr>"); message.println("</table>"); message.println(""); scheme + "://" + clientHostname + ":" + clientPortNumber + "/" + "executor?" + "execid=" + execId; message.println("<a href=\"" + executionUrl + "\">" + flow.getFlowId() + " Execution Link</a>"); return true;
for (final ExecutableFlow flow : exFlows) { final HashMap<String, Object> flowInfo = new HashMap<>(); flowInfo.put("execId", flow.getExecutionId()); flowInfo.put("flowId", flow.getFlowId()); flowInfo.put("projectId", flow.getProjectId()); flowInfo.put("status", flow.getStatus().toString()); flowInfo.put("submitTime", flow.getSubmitTime()); flowInfo.put("startTime", flow.getStartTime()); flowInfo.put("endTime", flow.getEndTime()); flowInfo.put("submitUser", flow.getSubmitUser());
node = flow.getExecutableNodePath(jobId); if (node == null) { page.add("errorMsg", "Job " + jobId + " doesn't exist in " + flow.getExecutionId()); return; final int projectId = flow.getProjectId(); final Project project = getProjectByPermission(projectId, user, Type.READ); if (project == null) { page.add("flowid", flow.getId()); page.add("parentflowid", node.getParentFlow().getFlowId()); page.add("jobname", node.getId());
private static Pair<ExecutionReference, ExecutableFlow> getExecutableFlowMetadataHelper( final ResultSet rs) throws SQLException { final Flow flow = new Flow(rs.getString("flow_id")); final Project project = new Project(rs.getInt("project_id"), null); project.setVersion(rs.getInt("version")); final ExecutableFlow exFlow = new ExecutableFlow(project, flow); exFlow.setExecutionId(rs.getInt("exec_id")); exFlow.setStatus(Status.fromInteger(rs.getInt("status"))); exFlow.setSubmitTime(rs.getLong("submit_time")); exFlow.setStartTime(rs.getLong("start_time")); exFlow.setEndTime(rs.getLong("end_time")); exFlow.setSubmitUser(rs.getString("submit_user")); return getPairWithExecutorInfo(rs, exFlow); }
private void updateExecutableFlow(final ExecutableFlow flow, final EncodingType encType) throws ExecutorManagerException { final String UPDATE_EXECUTABLE_FLOW_DATA = "UPDATE execution_flows " + "SET status=?,update_time=?,start_time=?,end_time=?,enc_type=?,flow_data=? " + "WHERE exec_id=?"; final String json = JSONUtils.toJSON(flow.toObject()); byte[] data = null; try { final byte[] stringData = json.getBytes("UTF-8"); data = stringData; // Todo kunkun-tang: use a common method to transform stringData to data. if (encType == EncodingType.GZIP) { data = GZIPUtils.gzipBytes(stringData); } } catch (final IOException e) { throw new ExecutorManagerException("Error encoding the execution flow."); } try { this.dbOperator.update(UPDATE_EXECUTABLE_FLOW_DATA, flow.getStatus() .getNumVal(), flow.getUpdateTime(), flow.getStartTime(), flow .getEndTime(), encType.getNumVal(), data, flow.getExecutionId()); } catch (final SQLException e) { throw new ExecutorManagerException("Error updating flow.", e); } }
@Override public void alertOnError(final ExecutableFlow flow, final String... extraReasons) { final EmailMessage message = this.messageCreator.createMessage(); final MailCreator mailCreator = getMailCreator(flow); List<ExecutableFlow> last72hoursExecutions = new ArrayList<>(); if (flow.getStartTime() > 0) { final long startTime = flow.getStartTime() - Duration.ofHours(72).toMillis(); try { last72hoursExecutions = this.executorLoader.fetchFlowHistory(flow.getProjectId(), flow .getFlowId(), startTime); } catch (final ExecutorManagerException e) { logger.error("unable to fetch past executions", e); } } final boolean mailCreated = mailCreator.createErrorEmail(flow, last72hoursExecutions, message, this.azkabanName, this.scheme, this.clientHostname, this.clientPortNumber, extraReasons); sendEmail(message, mailCreated, "error email message for execution " + flow.getExecutionId()); }
final ExecutableFlow exflow) throws ServletException { final Project project = getProjectAjaxByPermission(ret, exflow.getProjectId(), user, Type.READ); if (project == null) { return; final Flow flow = project.getFlow(exflow.getFlowId()); if (flow == null) { ret.put("error", "Error loading flow. Flow " + exflow.getFlowId() + " doesn't exist in " + exflow.getProjectId()); return; final ExecutionOptions options = exflow.getExecutionOptions(); for (final ExecutableNode node : exflow.getExecutableNodes()) { nodeStatus.put(node.getId(), node.getStatus().toString());
flow.setExecutionId(10); final ExecutableNode oldNode = flow.getExecutableNode("job10"); oldNode.setStartTime(System.currentTimeMillis()); this.executionJobDao.uploadExecutableNode(oldNode, props); assertThat(flow.getEndTime()).isEqualTo(info.getEndTime()); assertThat(flow.getProjectId()).isEqualTo(info.getProjectId()); assertThat(flow.getVersion()).isEqualTo(info.getVersion()); assertThat(flow.getFlowId()).isEqualTo(info.getFlowId());
private void writeHistoryData(List<HashMap<String, Object>> output, ExecutableFlow history) { HashMap<String, Object> data = new HashMap<String, Object>(); data.put("scheduleid", history.getScheduleId()); Project project = projectManager.getProject(history.getProjectId()); data.put("flowname", history.getFlowId()); data.put("projectname", project.getName()); data.put("time", history.getStartTime()); data.put("period", "0"); long endTime = history.getEndTime(); if(endTime == -1){ endTime = System.currentTimeMillis(); } data.put("length", endTime - history.getStartTime()); data.put("history", true); data.put("status", history.getStatus().getNumVal()); output.add(data); }
@Test public void testFetchActiveFlowByProject() throws Exception { initializeUnfinishedFlows(); final List<Integer> executions = this.controller .getRunningFlows(this.flow2.getProjectId(), this.flow2.getFlowId()); assertThat(executions.contains(this.flow2.getExecutionId())).isTrue(); assertThat(executions.contains(this.flow3.getExecutionId())).isTrue(); assertThat(this.controller.isFlowRunning(this.flow2.getProjectId(), this.flow2.getFlowId())) .isTrue(); assertThat(this.controller.isFlowRunning(this.flow3.getProjectId(), this.flow3.getFlowId())) .isTrue(); }
private boolean isFlowRunningHelper(final int projectId, final String flowId, final Collection<Pair<ExecutionReference, ExecutableFlow>> collection) { for (final Pair<ExecutionReference, ExecutableFlow> ref : collection) { if (ref.getSecond().getProjectId() == projectId && ref.getSecond().getFlowId().equals(flowId)) { return true; } } return false; }
@Override public void resumeFlow(final ExecutableFlow exFlow, final String userId) throws ExecutorManagerException { synchronized (exFlow) { final Pair<ExecutionReference, ExecutableFlow> pair = this.runningExecutions.get().get(exFlow.getExecutionId()); if (pair == null) { throw new ExecutorManagerException("Execution " + exFlow.getExecutionId() + " of flow " + exFlow.getFlowId() + " isn't running."); } this.apiGateway .callWithReferenceByUser(pair.getFirst(), ConnectorParams.RESUME_ACTION, userId); } }
dsFlow = flow; } else { dsFlow = executorLoader.fetchExecutableFlow(flow.getExecutionId()); if (flow.getEndTime() == -1) { flow.setEndTime(System.currentTimeMillis()); executorLoader.updateExecutableFlow(dsFlow); logger.error("Failed to finalize flow " + flow.getExecutionId() + ", do not alert user.", e);