public void updateExecutableNode(final ExecutableNode node) throws ExecutorManagerException { final String UPSERT_EXECUTION_NODE = "UPDATE execution_jobs " + "SET start_time=?, end_time=?, status=?, output_params=? " + "WHERE exec_id=? AND flow_id=? AND job_id=? AND attempt=?"; byte[] outputParam = null; final Props outputProps = node.getOutputProps(); if (outputProps != null) { try { final String jsonString = JSONUtils.toJSON(PropsUtils.toHierarchicalMap(outputProps)); outputParam = GZIPUtils.gzipString(jsonString, "UTF-8"); } catch (final IOException e) { throw new ExecutorManagerException("Error encoding input params"); } } try { this.dbOperator.update(UPSERT_EXECUTION_NODE, node.getStartTime(), node .getEndTime(), node.getStatus().getNumVal(), outputParam, node .getExecutableFlow().getExecutionId(), node.getParentFlow() .getFlowPath(), node.getId(), node.getAttempt()); } catch (final SQLException e) { throw new ExecutorManagerException("Error updating job " + node.getId(), e); } }
private Map<String, Object> getExecutableNodeInfo(final ExecutableNode node) { final HashMap<String, Object> nodeObj = new HashMap<>(); nodeObj.put("id", node.getId()); nodeObj.put("status", node.getStatus()); nodeObj.put("startTime", node.getStartTime()); nodeObj.put("endTime", node.getEndTime()); nodeObj.put("updateTime", node.getUpdateTime()); nodeObj.put("type", node.getType()); if (node.getCondition() != null) { nodeObj.put("condition", node.getCondition()); nodeObj.put("nestedId", node.getNestedId()); nodeObj.put("attempt", node.getAttempt()); if (node.getAttempt() > 0) { nodeObj.put("pastAttempts", node.getAttemptObjects()); if (node.getInNodes() != null && !node.getInNodes().isEmpty()) { nodeObj.put("in", node.getInNodes());
@Override public void fillExecutableFromMapObject( final TypedMapWrapper<String, Object> flowObjMap) { super.fillExecutableFromMapObject(flowObjMap); final ExecutableNode exJob = new ExecutableNode(); exJob.fillExecutableFromMapObject(nodeObjMap); exJob.setParentFlow(this); this.executableNodes.put(exJob.getId(), exJob);
/** * Creates a new EventData instance. * * @param node node. */ public EventData(final ExecutableNode node) { this(node.getStatus(), node.getNestedId()); }
public ExecutionAttempt(final int attempt, final ExecutableNode executable) { this.attempt = attempt; this.startTime = executable.getStartTime(); this.endTime = executable.getEndTime(); this.status = executable.getStatus(); }
public String getPrintableId(final String delimiter) { if (this.getParentFlow() == null || this.getParentFlow() instanceof ExecutableFlow) { return getId(); } return getParentFlow().getPrintableId(delimiter) + delimiter + getId(); }
public Map<String, Object> toUpdateObject() { final Map<String, Object> updatedNodeMap = new HashMap<>(); updatedNodeMap.put(ID_PARAM, getId()); updatedNodeMap.put(STATUS_PARAM, getStatus().getNumVal()); updatedNodeMap.put(STARTTIME_PARAM, getStartTime()); updatedNodeMap.put(ENDTIME_PARAM, getEndTime()); updatedNodeMap.put(UPDATETIME_PARAM, getUpdateTime()); updatedNodeMap.put(ATTEMPT_PARAM, getAttempt()); if (getAttempt() > 0) { final ArrayList<Map<String, Object>> pastAttempts = new ArrayList<>(); for (final ExecutionAttempt attempt : getPastAttemptList()) { pastAttempts.add(attempt.toObject()); } updatedNodeMap.put(PASTATTEMPTS_PARAM, pastAttempts); } return updatedNodeMap; }
@Override public void updateExecutableNode(final ExecutableNode node) throws ExecutorManagerException { final ExecutableNode foundNode = this.nodes.get(node.getId()); foundNode.setEndTime(node.getEndTime()); foundNode.setStartTime(node.getStartTime()); foundNode.setStatus(node.getStatus()); foundNode.setUpdateTime(node.getUpdateTime()); Integer value = this.jobUpdateCount.get(node.getId()); if (value == null) { throw new ExecutorManagerException("The node has not been uploaded"); } else { this.jobUpdateCount.put(node.getId(), ++value); } this.flowUpdateCount++; }
final ExecutableNode nodeB = exB.getExecutableNode(nodeA.getId()); Assert.assertNotNull(nodeB); Assert.assertEquals(a, nodeA.getParentFlow()); Assert.assertEquals(b, nodeB.getParentFlow()); Assert.assertEquals(a.getId(), b.getId()); Assert.assertEquals(a.getStatus(), b.getStatus()); Assert.assertEquals(a.getStartTime(), b.getStartTime()); Assert.assertEquals(a.getEndTime(), b.getEndTime()); Assert.assertEquals(a.getUpdateTime(), b.getUpdateTime()); Assert.assertEquals(a.getAttempt(), b.getAttempt()); Assert.assertEquals(a.getJobSource(), b.getJobSource()); Assert.assertEquals(a.getPropsSource(), b.getPropsSource()); Assert.assertEquals(a.getInNodes(), a.getInNodes()); Assert.assertEquals(a.getOutNodes(), a.getOutNodes());
final long time = System.currentTimeMillis(); for (final ExecutableNode node : exFlow.getExecutableNodes()) { switch (node.getStatus()) { case SUCCEEDED: case FAILED: node.setStatus(Status.KILLING); break; default: node.setStatus(Status.FAILED); break; if (node.getStartTime() == -1) { node.setStartTime(time); if (node.getEndTime() == -1) { node.setEndTime(time);
if (node.getUpdateTime() > lastUpdateTime || !nodeObj.isEmpty()) { nodeObj.put("id", node.getId()); nodeObj.put("status", node.getStatus()); nodeObj.put("startTime", node.getStartTime()); nodeObj.put("endTime", node.getEndTime()); nodeObj.put("updateTime", node.getUpdateTime()); nodeObj.put("attempt", node.getAttempt()); if (node.getAttempt() > 0) { nodeObj.put("pastAttempts", node.getAttemptObjects());
while (!queue.isEmpty()) { ExecutableNode node = queue.poll(); Status oldStatus = node.getStatus(); maxStartTime = Math.max(node.getStartTime(), maxStartTime); if (node.getStatus() == Status.SUCCEEDED) { else if (node.getStatus() == Status.RUNNING) { continue; else if (node.getStatus() == Status.SKIPPED) { node.setStatus(Status.DISABLED); node.setEndTime(-1); node.setStartTime(-1); node.setUpdateTime(currentTime); switch (base.getStatus()) { case CANCELLED: node.setStatus(Status.READY); node.setEndTime(-1); node.setStartTime(-1); node.setUpdateTime(currentTime); else if (node.getStatus() == Status.CANCELLED) { node.setStatus(Status.READY); node.setStartTime(-1); node.setEndTime(-1); node.setUpdateTime(currentTime);
public void uploadAttachmentFile(final ExecutableNode node, final File file) throws ExecutorManagerException { final String UPDATE_EXECUTION_NODE_ATTACHMENTS = "UPDATE execution_jobs " + "SET attachments=? " + "WHERE exec_id=? AND flow_id=? AND job_id=? AND attempt=?"; try { final String jsonString = FileUtils.readFileToString(file); final byte[] attachments = GZIPUtils.gzipString(jsonString, "UTF-8"); this.dbOperator.update(UPDATE_EXECUTION_NODE_ATTACHMENTS, attachments, node.getExecutableFlow().getExecutionId(), node.getParentFlow() .getNestedId(), node.getId(), node.getAttempt()); } catch (final IOException | SQLException e) { throw new ExecutorManagerException("Error uploading attachments.", e); } }
public static String createLogFileName(ExecutableNode node, int attempt) { int executionId = node.getExecutableFlow().getExecutionId(); String jobId = node.getId(); if (node.getExecutableFlow() != node.getParentFlow()) { // Posix safe file delimiter jobId = node.getPrintableId("._."); } return attempt > 0 ? "_job." + executionId + "." + attempt + "." + jobId + ".log" : "_job." + executionId + "." + jobId + ".log"; }
props.put("test", "test2"); final ExecutableNode oldNode = flow.getExecutableNode("job10"); oldNode.setStartTime(System.currentTimeMillis()); this.executionJobDao.uploadExecutableNode(oldNode, props); assertThat(flow.getFlowId()).isEqualTo(info.getFlowId()); assertThat(oldNode.getId()).isEqualTo(info.getJobId()); assertThat(oldNode.getStatus()).isEqualTo(info.getStatus()); assertThat(oldNode.getStartTime()).isEqualTo(info.getStartTime()); oldNode.setOutputProps(outputProps); oldNode.setEndTime(System.currentTimeMillis()); this.executionJobDao.updateExecutableNode(oldNode);
if (node.getStatus() == Status.FAILED || this.isKilled()) { return false; if (node.getAttempt() > 0) { logInfo("Starting job " + this.jobId + " attempt " + node.getAttempt() + " at " + node.getStartTime()); logInfo("Starting job " + this.jobId + " at " + node.getStartTime()); if (node.getExecutableFlow() != node.getParentFlow()) { String subFlow = node.getPrintableId(":"); props.put(CommonJobProperties.NESTED_FLOW_PATH, subFlow); props.put(CommonJobProperties.JOB_ATTEMPT, node.getAttempt()); props.put(CommonJobProperties.JOB_METADATA_FILE, createMetaDataFileName(node)); props.put(CommonJobProperties.JOB_ATTACHMENT_FILE, attachmentFileName);
@Override public void uploadExecutableNode(final ExecutableNode node, final Props inputParams) throws ExecutorManagerException { // Clone the job node to mimick how it would be saved in DB. // If we would keep a handle to the original job node, we would also see any changes made after // this method was called. We must only store a snapshot of the current state. // Also to avoid modifying statuses of the original job nodes in this.updateExecutableNode() final ExecutableNode exNode = new ExecutableNode(); exNode.fillExecutableFromMapObject(node.toObject()); this.nodes.put(node.getId(), exNode); this.jobUpdateCount.put(node.getId(), 1); }
@Override public synchronized void handleEvent(Event event) { JobRunner runner = (JobRunner)event.getRunner(); if (event.getType() == Type.JOB_STATUS_CHANGED) { updateFlow(); } else if (event.getType() == Type.JOB_FINISHED) { ExecutableNode node = runner.getNode(); long seconds = (node.getEndTime() - node.getStartTime())/1000; synchronized(mainSyncObj) { logger.info("Job " + node.getNestedId() + " finished with status " + node.getStatus() + " in " + seconds + " seconds"); // Cancellation is handled in the main thread, but if the flow is paused, the main thread is paused too. // This unpauses the flow for cancellation. if (flowPaused && node.getStatus() == Status.FAILED && failureAction == FailureAction.CANCEL_ALL) { flowPaused = false; } finishedNodes.add(node); node.getParentFlow().setUpdateTime(System.currentTimeMillis()); interrupt(); fireEventListeners(event); } } } }
ExecutableFlow flow = node.getExecutableFlow(); String flowId = node.getParentFlow().getFlowPath(); System.out.println("Uploading flowId " + flowId); QueryRunner runner = createQueryRunner(); flow.getVersion(), flowId, node.getId(), node.getStartTime(), node.getEndTime(), node.getStatus().getNumVal(), inputParam, } catch (SQLException e) { throw new ExecutorManagerException( "Error writing job " + node.getId(), e);
(String) this.slaOption.getInfo().get(SlaOption.INFO_JOB_NAME); final ExecutableNode node = flow.getExecutableNode(jobName); if (node.getStartTime() < 0) { return Boolean.FALSE; if (this.checkTime < node.getStartTime()) { final ReadablePeriod duration = Utils.parsePeriodString((String) this.slaOption.getInfo().get( SlaOption.INFO_DURATION)); final DateTime startTime = new DateTime(node.getStartTime()); final DateTime nextCheckTime = startTime.plus(duration); this.checkTime = nextCheckTime.getMillis(); status = node.getStatus(); return isJobFinished(status); } else if (type.equals(SlaOption.TYPE_JOB_SUCCEED)) { (String) this.slaOption.getInfo().get(SlaOption.INFO_JOB_NAME); final ExecutableNode node = flow.getExecutableNode(jobName); if (node.getStartTime() < 0) { return Boolean.FALSE; if (this.checkTime < node.getStartTime()) { final ReadablePeriod duration = Utils.parsePeriodString((String) this.slaOption.getInfo().get( SlaOption.INFO_DURATION)); final DateTime startTime = new DateTime(node.getStartTime()); final DateTime nextCheckTime = startTime.plus(duration); this.checkTime = nextCheckTime.getMillis();