private void ajaxFetchExecutableFlowUpdate(final HttpServletRequest req, final HttpServletResponse resp, final HashMap<String, Object> ret, final User user, final ExecutableFlow exFlow) throws ServletException { final Long lastUpdateTime = Long.parseLong(getParam(req, "lastUpdateTime")); logger.info("Fetching " + exFlow.getExecutionId()); final Project project = getProjectAjaxByPermission(ret, exFlow.getProjectId(), user, Type.READ); if (project == null) { return; } final Map<String, Object> map = getExecutableFlowUpdateInfo(exFlow, lastUpdateTime); map.put("status", exFlow.getStatus()); map.put("startTime", exFlow.getStartTime()); map.put("endTime", exFlow.getEndTime()); map.put("updateTime", exFlow.getUpdateTime()); ret.putAll(map); }
flowInfo.put("status", flow.getStatus().toString()); flowInfo.put("submitTime", flow.getSubmitTime()); flowInfo.put("startTime", flow.getStartTime()); flowInfo.put("endTime", flow.getEndTime()); flowInfo.put("submitUser", flow.getSubmitUser());
@Override public void alertOnError(final ExecutableFlow flow, final String... extraReasons) { final EmailMessage message = this.messageCreator.createMessage(); final MailCreator mailCreator = getMailCreator(flow); List<ExecutableFlow> last72hoursExecutions = new ArrayList<>(); if (flow.getStartTime() > 0) { final long startTime = flow.getStartTime() - Duration.ofHours(72).toMillis(); try { last72hoursExecutions = this.executorLoader.fetchFlowHistory(flow.getProjectId(), flow .getFlowId(), startTime); } catch (final ExecutorManagerException e) { logger.error("unable to fetch past executions", e); } } final boolean mailCreated = mailCreator.createErrorEmail(flow, last72hoursExecutions, message, this.azkabanName, this.scheme, this.clientHostname, this.clientPortNumber, extraReasons); sendEmail(message, mailCreated, "error email message for execution " + flow.getExecutionId()); }
"Here is details : <br/>" + "Flow " + flowName + " in execution " + execId + " is expected to FINISH within " + duration + " from " + fmt.print(new DateTime(flow.getStartTime())) + "<br/>"; final String actual = "Actual flow status is " + flow.getStatus(); return basicinfo + expected + actual; "Here is details : <br/>" + "Flow " + flowName + " in execution " + execId + " expected to FINISH within " + duration + " from " + fmt.print(new DateTime(flow.getStartTime())) + "<br/>"; final String actual = "Actual flow status is " + flow.getStatus(); return basicinfo + expected + actual;
private Boolean isSlaGood(final ExecutableFlow flow) { final String type = this.slaOption.getType(); if (flow.getStartTime() < 0) { return Boolean.FALSE; if (this.checkTime < flow.getStartTime()) { final ReadablePeriod duration = Utils.parsePeriodString((String) this.slaOption.getInfo().get( SlaOption.INFO_DURATION)); final DateTime startTime = new DateTime(flow.getStartTime()); final DateTime nextCheckTime = startTime.plus(duration); this.checkTime = nextCheckTime.getMillis(); return isFlowFinished(status); } else if (type.equals(SlaOption.TYPE_FLOW_SUCCEED)) { if (this.checkTime < flow.getStartTime()) { final ReadablePeriod duration = Utils.parsePeriodString((String) this.slaOption.getInfo().get( SlaOption.INFO_DURATION)); final DateTime startTime = new DateTime(flow.getStartTime()); final DateTime nextCheckTime = startTime.plus(duration); this.checkTime = nextCheckTime.getMillis();
private Boolean isSlaMissed(final ExecutableFlow flow) { final String type = this.slaOption.getType(); if (flow.getStartTime() < 0) { return Boolean.FALSE; if (this.checkTime < flow.getStartTime()) { final ReadablePeriod duration = Utils.parsePeriodString((String) this.slaOption.getInfo().get( SlaOption.INFO_DURATION)); final DateTime startTime = new DateTime(flow.getStartTime()); final DateTime nextCheckTime = startTime.plus(duration); this.checkTime = nextCheckTime.getMillis(); if (this.checkTime < flow.getStartTime()) { final ReadablePeriod duration = Utils.parsePeriodString((String) this.slaOption.getInfo().get( SlaOption.INFO_DURATION)); final DateTime startTime = new DateTime(flow.getStartTime()); final DateTime nextCheckTime = startTime.plus(duration); this.checkTime = nextCheckTime.getMillis();
private void assertTwoFlowSame(final ExecutableFlow flow1, final ExecutableFlow flow2, final boolean compareFlowData) { assertThat(flow1.getExecutionId()).isEqualTo(flow2.getExecutionId()); assertThat(flow1.getStatus()).isEqualTo(flow2.getStatus()); assertThat(flow1.getEndTime()).isEqualTo(flow2.getEndTime()); assertThat(flow1.getStartTime()).isEqualTo(flow2.getStartTime()); assertThat(flow1.getSubmitTime()).isEqualTo(flow2.getSubmitTime()); assertThat(flow1.getFlowId()).isEqualTo(flow2.getFlowId()); assertThat(flow1.getProjectId()).isEqualTo(flow2.getProjectId()); assertThat(flow1.getVersion()).isEqualTo(flow2.getVersion()); assertThat(flow1.getSubmitUser()).isEqualTo(flow2.getSubmitUser()); if (compareFlowData) { assertThat(flow1.getExecutionOptions().getFailureAction()) .isEqualTo(flow2.getExecutionOptions().getFailureAction()); assertThat(new HashSet<>(flow1.getEndNodes())).isEqualTo(new HashSet<>(flow2.getEndNodes())); } }
private void updateExecutableFlow(final ExecutableFlow flow, final EncodingType encType) throws ExecutorManagerException { final String UPDATE_EXECUTABLE_FLOW_DATA = "UPDATE execution_flows " + "SET status=?,update_time=?,start_time=?,end_time=?,enc_type=?,flow_data=? " + "WHERE exec_id=?"; final String json = JSONUtils.toJSON(flow.toObject()); byte[] data = null; try { final byte[] stringData = json.getBytes("UTF-8"); data = stringData; // Todo kunkun-tang: use a common method to transform stringData to data. if (encType == EncodingType.GZIP) { data = GZIPUtils.gzipBytes(stringData); } } catch (final IOException e) { throw new ExecutorManagerException("Error encoding the execution flow."); } try { this.dbOperator.update(UPDATE_EXECUTABLE_FLOW_DATA, flow.getStatus() .getNumVal(), flow.getUpdateTime(), flow.getStartTime(), flow .getEndTime(), encType.getNumVal(), data, flow.getExecutionId()); } catch (final SQLException e) { throw new ExecutorManagerException("Error updating flow.", e); } }
message.println("<table>"); message.println("<tr><td>Start Time</td><td>" + convertMSToString(flow.getStartTime()) + "</td></tr>"); message.println("<tr><td>End Time</td><td>" + convertMSToString(flow.getEndTime()) + "</td></tr>"); message.println("<tr><td>Duration</td><td>" + Utils.formatDuration(flow.getStartTime(), flow.getEndTime()) + "</td></tr>"); message.println("<tr><td>Status</td><td>" + flow.getStatus() + "</td></tr>");
message.println("<table>"); message.println("<tr><td>Start Time</td><td>" + convertMSToString(flow.getStartTime()) + "</td></tr>"); message.println("<tr><td>End Time</td><td>" + convertMSToString(flow.getEndTime()) + "</td></tr>"); message.println("<tr><td>Duration</td><td>" + Utils.formatDuration(flow.getStartTime(), flow.getEndTime()) + "</td></tr>"); message.println("<tr><td>Status</td><td>" + flow.getStatus() + "</td></tr>"); "<tr><td>Execution Id</td><td>" + (executableFlow.getExecutionId()) + "</td></tr>"); message.println("<tr><td>Start Time</td><td>" + convertMSToString(executableFlow.getStartTime()) + "</td></tr>"); message.println("<tr><td>End Time</td><td>" + convertMSToString(executableFlow.getEndTime()) + "</td></tr>");
+ convertMSToString(flow.getStartTime()) + "</td></tr>"); message.println("<tr><td>End Time</td><td>" + convertMSToString(flow.getEndTime()) + "</td></tr>"); message.println("<tr><td>Duration</td><td>" + Utils.formatDuration(flow.getStartTime(), flow.getEndTime()) + "</td></tr>"); message.println("<tr><td>Status</td><td>" + flow.getStatus() + "</td></tr>");
private ExecutableFlow getExecutableFlowMetadata( ExecutableFlow fullExFlow) { final Flow flow = new Flow(fullExFlow.getId()); final Project project = new Project(fullExFlow.getProjectId(), null); project.setVersion(fullExFlow.getVersion()); flow.setVersion(fullExFlow.getVersion()); final ExecutableFlow metadata = new ExecutableFlow(project, flow); metadata.setExecutionId(fullExFlow.getExecutionId()); metadata.setStatus(fullExFlow.getStatus()); metadata.setSubmitTime(fullExFlow.getSubmitTime()); metadata.setStartTime(fullExFlow.getStartTime()); metadata.setEndTime(fullExFlow.getEndTime()); metadata.setSubmitUser(fullExFlow.getSubmitUser()); return metadata; }
private void writeHistoryData(List<HashMap<String, Object>> output, ExecutableFlow history) { HashMap<String, Object> data = new HashMap<String, Object>(); data.put("scheduleid", history.getScheduleId()); Project project = projectManager.getProject(history.getProjectId()); data.put("flowname", history.getFlowId()); data.put("projectname", project.getName()); data.put("time", history.getStartTime()); data.put("period", "0"); long endTime = history.getEndTime(); if(endTime == -1){ endTime = System.currentTimeMillis(); } data.put("length", endTime - history.getStartTime()); data.put("history", true); data.put("status", history.getStatus().getNumVal()); output.add(data); }
private void ajaxFetchExecutableFlowUpdate( HttpServletRequest req, HttpServletResponse resp, HashMap<String, Object> ret, User user, ExecutableFlow exFlow) throws ServletException { Long lastUpdateTime = Long.parseLong(getParam(req, "lastUpdateTime")); System.out.println("Fetching " + exFlow.getExecutionId()); Project project = getProjectAjaxByPermission(ret, exFlow.getProjectId(), user, Type.READ); if (project == null) { return; } Map<String, Object> map = getExecutableFlowUpdateInfo(exFlow, lastUpdateTime); map.put("status", exFlow.getStatus()); map.put("startTime", exFlow.getStartTime()); map.put("endTime", exFlow.getEndTime()); map.put("updateTime", exFlow.getUpdateTime()); ret.putAll(map); }
public static String createSlaMessage(SlaOption slaOption, ExecutableFlow flow) { String type = slaOption.getType(); int execId = flow.getExecutionId(); if(type.equals(SlaOption.TYPE_FLOW_FINISH)) { String flowName = (String) slaOption.getInfo().get(SlaOption.INFO_FLOW_NAME); String duration = (String) slaOption.getInfo().get(SlaOption.INFO_DURATION); String basicinfo = "SLA Alert: Your flow " + flowName + " failed to FINISH within " + duration + "</br>"; String expected = "Here is details : </br>" + "Flow " + flowName + " in execution " + execId + " is expected to FINISH within " + duration + " from " + fmt.print(new DateTime(flow.getStartTime())) + "</br>"; String actual = "Actual flow status is " + flow.getStatus(); return basicinfo + expected + actual; } else if(type.equals(SlaOption.TYPE_FLOW_SUCCEED)) { String flowName = (String) slaOption.getInfo().get(SlaOption.INFO_FLOW_NAME); String duration = (String) slaOption.getInfo().get(SlaOption.INFO_DURATION); String basicinfo = "SLA Alert: Your flow " + flowName + " failed to SUCCEED within " + duration + "</br>"; String expected = "Here is details : </br>" + "Flow " + flowName + " in execution " + execId + " expected to FINISH within " + duration + " from " + fmt.print(new DateTime(flow.getStartTime())) + "</br>"; String actual = "Actual flow status is " + flow.getStatus(); return basicinfo + expected + actual; } else if(type.equals(SlaOption.TYPE_JOB_FINISH)) { String jobName = (String) slaOption.getInfo().get(SlaOption.INFO_JOB_NAME); String duration = (String) slaOption.getInfo().get(SlaOption.INFO_DURATION); return "SLA Alert: Your job " + jobName + " failed to FINISH within " + duration + " in execution " + execId; } else if(type.equals(SlaOption.TYPE_JOB_SUCCEED)) { String jobName = (String) slaOption.getInfo().get(SlaOption.INFO_JOB_NAME); String duration = (String) slaOption.getInfo().get(SlaOption.INFO_DURATION); return "SLA Alert: Your job " + jobName + " failed to SUCCEED within " + duration + " in execution " + execId; } else { return "Unrecognized SLA type " + type; } }
@Override public void alertOnError(final ExecutableFlow flow, final String... extraReasons) { final EmailMessage message = this.messageCreator.createMessage(); final MailCreator mailCreator = getMailCreator(flow); List<ExecutableFlow> last72hoursExecutions = new ArrayList<>(); if (flow.getStartTime() > 0) { final long startTime = flow.getStartTime() - Duration.ofHours(72).toMillis(); try { last72hoursExecutions = this.executorLoader.fetchFlowHistory(flow.getProjectId(), flow .getFlowId(), startTime); } catch (final ExecutorManagerException e) { logger.error("unable to fetch past executions", e); } } final boolean mailCreated = mailCreator.createErrorEmail(flow, last72hoursExecutions, message, this.azkabanName, this.scheme, this.clientHostname, this.clientPortNumber, extraReasons); sendEmail(message, mailCreated, "error email message for execution " + flow.getExecutionId()); }
long time = flow.getEndTime() - flow.getStartTime(); average += time; if (time < min) {
private void updateExecutableFlow(final ExecutableFlow flow, final EncodingType encType) throws ExecutorManagerException { final String UPDATE_EXECUTABLE_FLOW_DATA = "UPDATE execution_flows " + "SET status=?,update_time=?,start_time=?,end_time=?,enc_type=?,flow_data=? " + "WHERE exec_id=?"; final String json = JSONUtils.toJSON(flow.toObject()); byte[] data = null; try { final byte[] stringData = json.getBytes("UTF-8"); data = stringData; // Todo kunkun-tang: use a common method to transform stringData to data. if (encType == EncodingType.GZIP) { data = GZIPUtils.gzipBytes(stringData); } } catch (final IOException e) { throw new ExecutorManagerException("Error encoding the execution flow."); } try { this.dbOperator.update(UPDATE_EXECUTABLE_FLOW_DATA, flow.getStatus() .getNumVal(), flow.getUpdateTime(), flow.getStartTime(), flow .getEndTime(), encType.getNumVal(), data, flow.getExecutionId()); } catch (final SQLException e) { throw new ExecutorManagerException("Error updating flow.", e); } }
@Override public boolean createSuccessEmail(ExecutableFlow flow, EmailMessage message, String azkabanName, String clientHostname, String clientPortNumber, String... vars) { ExecutionOptions option = flow.getExecutionOptions(); List<String> emailList = option.getSuccessEmails(); int execId = flow.getExecutionId(); if (emailList != null && !emailList.isEmpty()) { message.addAllToAddress(emailList); message.setMimeType("text/html"); message.setSubject("Flow '" + flow.getFlowId() + "' has succeeded on " + azkabanName); message.println("<h2> Execution '" + flow.getExecutionId() + "' of flow '" + flow.getFlowId() + "' has succeeded on " + azkabanName + "</h2>"); message.println("<table>"); message.println("<tr><td>Start Time</td><td>" + flow.getStartTime() + "</td></tr>"); message.println("<tr><td>End Time</td><td>" + flow.getEndTime() + "</td></tr>"); message.println("<tr><td>Duration</td><td>" + Utils.formatDuration(flow.getStartTime(), flow.getEndTime()) + "</td></tr>"); message.println("</table>"); message.println(""); String executionUrl = "https://" + clientHostname + ":" + clientPortNumber + "/" + "executor?" + "execid=" + execId; message.println("<a href=\"" + executionUrl + "\">" + flow.getFlowId() + " Execution Link</a>"); return true; } return false; } }
message.println("<table>"); message.println("<tr><td>Start Time</td><td>" + convertMSToString(flow.getStartTime()) + "</td></tr>"); message.println("<tr><td>End Time</td><td>" + convertMSToString(flow.getEndTime()) + "</td></tr>"); message.println("<tr><td>Duration</td><td>" + Utils.formatDuration(flow.getStartTime(), flow.getEndTime()) + "</td></tr>"); message.println("<tr><td>Status</td><td>" + flow.getStatus() + "</td></tr>");