private void ajaxFetchExecutableFlowUpdate(final HttpServletRequest req, final HttpServletResponse resp, final HashMap<String, Object> ret, final User user, final ExecutableFlow exFlow) throws ServletException { final Long lastUpdateTime = Long.parseLong(getParam(req, "lastUpdateTime")); logger.info("Fetching " + exFlow.getExecutionId()); final Project project = getProjectAjaxByPermission(ret, exFlow.getProjectId(), user, Type.READ); if (project == null) { return; } final Map<String, Object> map = getExecutableFlowUpdateInfo(exFlow, lastUpdateTime); map.put("status", exFlow.getStatus()); map.put("startTime", exFlow.getStartTime()); map.put("endTime", exFlow.getEndTime()); map.put("updateTime", exFlow.getUpdateTime()); ret.putAll(map); }
flowInfo.put("submitTime", flow.getSubmitTime()); flowInfo.put("startTime", flow.getStartTime()); flowInfo.put("endTime", flow.getEndTime()); flowInfo.put("submitUser", flow.getSubmitUser());
private void assertTwoFlowSame(final ExecutableFlow flow1, final ExecutableFlow flow2, final boolean compareFlowData) { assertThat(flow1.getExecutionId()).isEqualTo(flow2.getExecutionId()); assertThat(flow1.getStatus()).isEqualTo(flow2.getStatus()); assertThat(flow1.getEndTime()).isEqualTo(flow2.getEndTime()); assertThat(flow1.getStartTime()).isEqualTo(flow2.getStartTime()); assertThat(flow1.getSubmitTime()).isEqualTo(flow2.getSubmitTime()); assertThat(flow1.getFlowId()).isEqualTo(flow2.getFlowId()); assertThat(flow1.getProjectId()).isEqualTo(flow2.getProjectId()); assertThat(flow1.getVersion()).isEqualTo(flow2.getVersion()); assertThat(flow1.getSubmitUser()).isEqualTo(flow2.getSubmitUser()); if (compareFlowData) { assertThat(flow1.getExecutionOptions().getFailureAction()) .isEqualTo(flow2.getExecutionOptions().getFailureAction()); assertThat(new HashSet<>(flow1.getEndNodes())).isEqualTo(new HashSet<>(flow2.getEndNodes())); } }
if (flow.getEndTime() == -1) { flow.setEndTime(System.currentTimeMillis()); executorLoader.updateExecutableFlow(dsFlow);
if (exFlow.getEndTime() == -1) { exFlow.setEndTime(time);
private void updateExecutableFlow(final ExecutableFlow flow, final EncodingType encType) throws ExecutorManagerException { final String UPDATE_EXECUTABLE_FLOW_DATA = "UPDATE execution_flows " + "SET status=?,update_time=?,start_time=?,end_time=?,enc_type=?,flow_data=? " + "WHERE exec_id=?"; final String json = JSONUtils.toJSON(flow.toObject()); byte[] data = null; try { final byte[] stringData = json.getBytes("UTF-8"); data = stringData; // Todo kunkun-tang: use a common method to transform stringData to data. if (encType == EncodingType.GZIP) { data = GZIPUtils.gzipBytes(stringData); } } catch (final IOException e) { throw new ExecutorManagerException("Error encoding the execution flow."); } try { this.dbOperator.update(UPDATE_EXECUTABLE_FLOW_DATA, flow.getStatus() .getNumVal(), flow.getUpdateTime(), flow.getStartTime(), flow .getEndTime(), encType.getNumVal(), data, flow.getExecutionId()); } catch (final SQLException e) { throw new ExecutorManagerException("Error updating flow.", e); } }
if (flow.getEndTime() == -1) { flow.setEndTime(System.currentTimeMillis()); this.executorLoader.updateExecutableFlow(dsFlow);
assertThat(flow.getEndTime()).isEqualTo(info.getEndTime()); assertThat(flow.getProjectId()).isEqualTo(info.getProjectId()); assertThat(flow.getVersion()).isEqualTo(info.getVersion());
+ convertMSToString(flow.getStartTime()) + "</td></tr>"); message.println("<tr><td>End Time</td><td>" + convertMSToString(flow.getEndTime()) + "</td></tr>"); message.println("<tr><td>Duration</td><td>" + Utils.formatDuration(flow.getStartTime(), flow.getEndTime()) + "</td></tr>"); message.println("<tr><td>Status</td><td>" + flow.getStatus() + "</td></tr>");
+ convertMSToString(flow.getStartTime()) + "</td></tr>"); message.println("<tr><td>End Time</td><td>" + convertMSToString(flow.getEndTime()) + "</td></tr>"); message.println("<tr><td>Duration</td><td>" + Utils.formatDuration(flow.getStartTime(), flow.getEndTime()) + "</td></tr>"); message.println("<tr><td>Status</td><td>" + flow.getStatus() + "</td></tr>"); + convertMSToString(executableFlow.getStartTime()) + "</td></tr>"); message.println("<tr><td>End Time</td><td>" + convertMSToString(executableFlow.getEndTime()) + "</td></tr>"); message.println("<tr><td>Status</td><td>" + executableFlow.getStatus() + "</td></tr>"); message.println("</table>");
+ convertMSToString(flow.getStartTime()) + "</td></tr>"); message.println("<tr><td>End Time</td><td>" + convertMSToString(flow.getEndTime()) + "</td></tr>"); message.println("<tr><td>Duration</td><td>" + Utils.formatDuration(flow.getStartTime(), flow.getEndTime()) + "</td></tr>"); message.println("<tr><td>Status</td><td>" + flow.getStatus() + "</td></tr>");
private ExecutableFlow getExecutableFlowMetadata( ExecutableFlow fullExFlow) { final Flow flow = new Flow(fullExFlow.getId()); final Project project = new Project(fullExFlow.getProjectId(), null); project.setVersion(fullExFlow.getVersion()); flow.setVersion(fullExFlow.getVersion()); final ExecutableFlow metadata = new ExecutableFlow(project, flow); metadata.setExecutionId(fullExFlow.getExecutionId()); metadata.setStatus(fullExFlow.getStatus()); metadata.setSubmitTime(fullExFlow.getSubmitTime()); metadata.setStartTime(fullExFlow.getStartTime()); metadata.setEndTime(fullExFlow.getEndTime()); metadata.setSubmitUser(fullExFlow.getSubmitUser()); return metadata; }
private void evictOldRecentlyFinished(long ageMs) { ArrayList<Integer> recentlyFinishedKeys = new ArrayList<Integer>(recentlyFinished.keySet()); long oldAgeThreshold = System.currentTimeMillis() - ageMs; for (Integer key: recentlyFinishedKeys) { ExecutableFlow flow = recentlyFinished.get(key); if (flow.getEndTime() < oldAgeThreshold) { // Evict recentlyFinished.remove(key); } } }
private void cleanRecentlyFinished() { long cleanupThreshold = System.currentTimeMillis() - RECENTLY_FINISHED_TIME_TO_LIVE; ArrayList<Integer> executionToKill = new ArrayList<Integer>(); for (ExecutableFlow flow : recentlyFinishedFlows.values()) { if (flow.getEndTime() < cleanupThreshold) { executionToKill.add(flow.getExecutionId()); } } for (Integer id: executionToKill) { logger.info("Cleaning execution " + id + " from recently finished flows list."); recentlyFinishedFlows.remove(id); } }
private void writeHistoryData(List<HashMap<String, Object>> output, ExecutableFlow history) { HashMap<String, Object> data = new HashMap<String, Object>(); data.put("scheduleid", history.getScheduleId()); Project project = projectManager.getProject(history.getProjectId()); data.put("flowname", history.getFlowId()); data.put("projectname", project.getName()); data.put("time", history.getStartTime()); data.put("period", "0"); long endTime = history.getEndTime(); if(endTime == -1){ endTime = System.currentTimeMillis(); } data.put("length", endTime - history.getStartTime()); data.put("history", true); data.put("status", history.getStatus().getNumVal()); output.add(data); }
private void ajaxFetchExecutableFlowUpdate( HttpServletRequest req, HttpServletResponse resp, HashMap<String, Object> ret, User user, ExecutableFlow exFlow) throws ServletException { Long lastUpdateTime = Long.parseLong(getParam(req, "lastUpdateTime")); System.out.println("Fetching " + exFlow.getExecutionId()); Project project = getProjectAjaxByPermission(ret, exFlow.getProjectId(), user, Type.READ); if (project == null) { return; } Map<String, Object> map = getExecutableFlowUpdateInfo(exFlow, lastUpdateTime); map.put("status", exFlow.getStatus()); map.put("startTime", exFlow.getStartTime()); map.put("endTime", exFlow.getEndTime()); map.put("updateTime", exFlow.getUpdateTime()); ret.putAll(map); }
private void failEverything(final ExecutableFlow exFlow) { final long time = System.currentTimeMillis(); for (final ExecutableNode node : exFlow.getExecutableNodes()) { switch (node.getStatus()) { case SUCCEEDED: case FAILED: case KILLED: case SKIPPED: case DISABLED: continue; // case UNKNOWN: case READY: node.setStatus(Status.KILLING); break; default: node.setStatus(Status.FAILED); break; } if (node.getStartTime() == -1) { node.setStartTime(time); } if (node.getEndTime() == -1) { node.setEndTime(time); } } if (exFlow.getEndTime() == -1) { exFlow.setEndTime(time); } exFlow.setStatus(Status.FAILED); }
if (exFlow.getEndTime() == -1) { exFlow.setEndTime(time);
private void updateExecutableFlow(final ExecutableFlow flow, final EncodingType encType) throws ExecutorManagerException { final String UPDATE_EXECUTABLE_FLOW_DATA = "UPDATE execution_flows " + "SET status=?,update_time=?,start_time=?,end_time=?,enc_type=?,flow_data=? " + "WHERE exec_id=?"; final String json = JSONUtils.toJSON(flow.toObject()); byte[] data = null; try { final byte[] stringData = json.getBytes("UTF-8"); data = stringData; // Todo kunkun-tang: use a common method to transform stringData to data. if (encType == EncodingType.GZIP) { data = GZIPUtils.gzipBytes(stringData); } } catch (final IOException e) { throw new ExecutorManagerException("Error encoding the execution flow."); } try { this.dbOperator.update(UPDATE_EXECUTABLE_FLOW_DATA, flow.getStatus() .getNumVal(), flow.getUpdateTime(), flow.getStartTime(), flow .getEndTime(), encType.getNumVal(), data, flow.getExecutionId()); } catch (final SQLException e) { throw new ExecutorManagerException("Error updating flow.", e); } }
@Override public boolean createSuccessEmail(ExecutableFlow flow, EmailMessage message, String azkabanName, String clientHostname, String clientPortNumber, String... vars) { ExecutionOptions option = flow.getExecutionOptions(); List<String> emailList = option.getSuccessEmails(); int execId = flow.getExecutionId(); if (emailList != null && !emailList.isEmpty()) { message.addAllToAddress(emailList); message.setMimeType("text/html"); message.setSubject("Flow '" + flow.getFlowId() + "' has succeeded on " + azkabanName); message.println("<h2> Execution '" + flow.getExecutionId() + "' of flow '" + flow.getFlowId() + "' has succeeded on " + azkabanName + "</h2>"); message.println("<table>"); message.println("<tr><td>Start Time</td><td>" + flow.getStartTime() + "</td></tr>"); message.println("<tr><td>End Time</td><td>" + flow.getEndTime() + "</td></tr>"); message.println("<tr><td>Duration</td><td>" + Utils.formatDuration(flow.getStartTime(), flow.getEndTime()) + "</td></tr>"); message.println("</table>"); message.println(""); String executionUrl = "https://" + clientHostname + ":" + clientPortNumber + "/" + "executor?" + "execid=" + execId; message.println("<a href=\"" + executionUrl + "\">" + flow.getFlowId() + " Execution Link</a>"); return true; } return false; } }