public String getPrintableId(final String delimiter) { if (this.getParentFlow() == null || this.getParentFlow() instanceof ExecutableFlow) { return getId(); } return getParentFlow().getPrintableId(delimiter) + delimiter + getId(); }
public void uploadAttachmentFile(final ExecutableNode node, final File file) throws ExecutorManagerException { final String UPDATE_EXECUTION_NODE_ATTACHMENTS = "UPDATE execution_jobs " + "SET attachments=? " + "WHERE exec_id=? AND flow_id=? AND job_id=? AND attempt=?"; try { final String jsonString = FileUtils.readFileToString(file); final byte[] attachments = GZIPUtils.gzipString(jsonString, "UTF-8"); this.dbOperator.update(UPDATE_EXECUTION_NODE_ATTACHMENTS, attachments, node.getExecutableFlow().getExecutionId(), node.getParentFlow() .getNestedId(), node.getId(), node.getAttempt()); } catch (final IOException | SQLException e) { throw new ExecutorManagerException("Error uploading attachments.", e); } }
public void uploadExecutableNode(final ExecutableNode node, final Props inputProps) throws ExecutorManagerException { final String INSERT_EXECUTION_NODE = "INSERT INTO execution_jobs " + "(exec_id, project_id, version, flow_id, job_id, start_time, " + "end_time, status, input_params, attempt) VALUES (?,?,?,?,?,?,?,?,?,?)"; byte[] inputParam = null; if (inputProps != null) { try { final String jsonString = JSONUtils.toJSON(PropsUtils.toHierarchicalMap(inputProps)); inputParam = GZIPUtils.gzipString(jsonString, "UTF-8"); } catch (final IOException e) { throw new ExecutorManagerException("Error encoding input params"); } } final ExecutableFlow flow = node.getExecutableFlow(); final String flowId = node.getParentFlow().getFlowPath(); logger.info("Uploading flowId " + flowId); try { this.dbOperator.update(INSERT_EXECUTION_NODE, flow.getExecutionId(), flow.getProjectId(), flow.getVersion(), flowId, node.getId(), node.getStartTime(), node.getEndTime(), node.getStatus().getNumVal(), inputParam, node.getAttempt()); } catch (final SQLException e) { throw new ExecutorManagerException("Error writing job " + node.getId(), e); } }
public void updateExecutableNode(final ExecutableNode node) throws ExecutorManagerException { final String UPSERT_EXECUTION_NODE = "UPDATE execution_jobs " + "SET start_time=?, end_time=?, status=?, output_params=? " + "WHERE exec_id=? AND flow_id=? AND job_id=? AND attempt=?"; byte[] outputParam = null; final Props outputProps = node.getOutputProps(); if (outputProps != null) { try { final String jsonString = JSONUtils.toJSON(PropsUtils.toHierarchicalMap(outputProps)); outputParam = GZIPUtils.gzipString(jsonString, "UTF-8"); } catch (final IOException e) { throw new ExecutorManagerException("Error encoding input params"); } } try { this.dbOperator.update(UPSERT_EXECUTION_NODE, node.getStartTime(), node .getEndTime(), node.getStatus().getNumVal(), outputParam, node .getExecutableFlow().getExecutionId(), node.getParentFlow() .getFlowPath(), node.getId(), node.getAttempt()); } catch (final SQLException e) { throw new ExecutorManagerException("Error updating job " + node.getId(), e); } }
page.add("parentflowid", node.getParentFlow().getFlowId()); page.add("jobname", node.getId());
page.add("parentflowid", node.getParentFlow().getFlowId()); page.add("jobname", node.getId()); page.add("jobLinkUrl", jobLinkUrl);
final ExecutableNode nodeB = exB.getExecutableNode(nodeA.getId()); Assert.assertNotNull(nodeB); Assert.assertEquals(a, nodeA.getParentFlow()); Assert.assertEquals(b, nodeB.getParentFlow());
public String getPrintableId(String delimiter) { if (this.getParentFlow() == null || this.getParentFlow() instanceof ExecutableFlow) { return getId(); } return getParentFlow().getPrintableId(delimiter) + delimiter + getId(); }
public String getPrintableId(final String delimiter) { if (this.getParentFlow() == null || this.getParentFlow() instanceof ExecutableFlow) { return getId(); } return getParentFlow().getPrintableId(delimiter) + delimiter + getId(); }
public JobRunner(ExecutableNode node, File workingDir, ExecutorLoader loader, JobTypeManager jobtypeManager) { this.props = node.getInputProps(); this.node = node; this.workingDir = workingDir; this.executionId = node.getParentFlow().getExecutionId(); this.jobId = node.getId(); this.loader = loader; this.jobtypeManager = jobtypeManager; }
public static String createLogFileName(ExecutableNode node, int attempt) { int executionId = node.getExecutableFlow().getExecutionId(); String jobId = node.getId(); if (node.getExecutableFlow() != node.getParentFlow()) { // Posix safe file delimiter jobId = node.getPrintableId("._."); } return attempt > 0 ? "_job." + executionId + "." + attempt + "." + jobId + ".log" : "_job." + executionId + "." + jobId + ".log"; }
public static String createMetaDataFileName(ExecutableNode node, int attempt) { int executionId = node.getExecutableFlow().getExecutionId(); String jobId = node.getId(); if (node.getExecutableFlow() != node.getParentFlow()) { // Posix safe file delimiter jobId = node.getPrintableId("._."); } return attempt > 0 ? "_job." + executionId + "." + attempt + "." + jobId + ".meta" : "_job." + executionId + "." + jobId + ".meta"; }
public static String createAttachmentFileName(ExecutableNode node, int attempt) { int executionId = node.getExecutableFlow().getExecutionId(); String jobId = node.getId(); if (node.getExecutableFlow() != node.getParentFlow()) { // Posix safe file delimiter jobId = node.getPrintableId("._."); } return attempt > 0 ? "_job." + executionId + "." + attempt + "." + jobId + ".attach" : "_job." + executionId + "." + jobId + ".attach"; } }
private Props collectOutputProps(ExecutableNode node) { Props previousOutput = null; // Iterate the in nodes again and create the dependencies for (String dependency : node.getInNodes()) { Props output = node.getParentFlow().getExecutableNode(dependency).getOutputProps(); if (output != null) { output = Props.clone(output); output.setParent(previousOutput); previousOutput = output; } } return previousOutput; }
public void uploadAttachmentFile(final ExecutableNode node, final File file) throws ExecutorManagerException { final String UPDATE_EXECUTION_NODE_ATTACHMENTS = "UPDATE execution_jobs " + "SET attachments=? " + "WHERE exec_id=? AND flow_id=? AND job_id=? AND attempt=?"; try { final String jsonString = FileUtils.readFileToString(file); final byte[] attachments = GZIPUtils.gzipString(jsonString, "UTF-8"); this.dbOperator.update(UPDATE_EXECUTION_NODE_ATTACHMENTS, attachments, node.getExecutableFlow().getExecutionId(), node.getParentFlow() .getNestedId(), node.getId(), node.getAttempt()); } catch (final IOException | SQLException e) { throw new ExecutorManagerException("Error uploading attachments.", e); } }
private void uploadAttachmentFile( Connection connection, ExecutableNode node, File file, EncodingType encType) throws SQLException, IOException { String jsonString = FileUtils.readFileToString(file); byte[] attachments = GZIPUtils.gzipString(jsonString, "UTF-8"); final String UPDATE_EXECUTION_NODE_ATTACHMENTS = "UPDATE execution_jobs " + "SET attachments=? " + "WHERE exec_id=? AND flow_id=? AND job_id=? AND attempt=?"; QueryRunner runner = new QueryRunner(); runner.update( connection, UPDATE_EXECUTION_NODE_ATTACHMENTS, attachments, node.getExecutableFlow().getExecutionId(), node.getParentFlow().getNestedId(), node.getId(), node.getAttempt()); }
private void retryAllFailures() throws IOException { logger.info("Restarting all failed jobs"); this.retryFailedJobs = false; this.flowKilled = false; this.flowFailed = false; this.flow.setStatus(Status.RUNNING); ArrayList<ExecutableNode> retryJobs = new ArrayList<ExecutableNode>(); resetFailedState(this.flow, retryJobs); for (ExecutableNode node: retryJobs) { if (node.getStatus() == Status.READY || node.getStatus() == Status.DISABLED) { runReadyJob(node); } else if (node.getStatus() == Status.SUCCEEDED){ for (String outNodeId: node.getOutNodes()) { ExecutableFlowBase base = node.getParentFlow(); runReadyJob(base.getExecutableNode(outNodeId)); } } runReadyJob(node); } updateFlow(); }
@Override public synchronized void handleEvent(Event event) { JobRunner runner = (JobRunner)event.getRunner(); if (event.getType() == Type.JOB_STATUS_CHANGED) { updateFlow(); } else if (event.getType() == Type.JOB_FINISHED) { ExecutableNode node = runner.getNode(); long seconds = (node.getEndTime() - node.getStartTime())/1000; synchronized(mainSyncObj) { logger.info("Job " + node.getNestedId() + " finished with status " + node.getStatus() + " in " + seconds + " seconds"); // Cancellation is handled in the main thread, but if the flow is paused, the main thread is paused too. // This unpauses the flow for cancellation. if (flowPaused && node.getStatus() == Status.FAILED && failureAction == FailureAction.CANCEL_ALL) { flowPaused = false; } finishedNodes.add(node); node.getParentFlow().setUpdateTime(System.currentTimeMillis()); interrupt(); fireEventListeners(event); } } } }
public void uploadExecutableNode(final ExecutableNode node, final Props inputProps) throws ExecutorManagerException { final String INSERT_EXECUTION_NODE = "INSERT INTO execution_jobs " + "(exec_id, project_id, version, flow_id, job_id, start_time, " + "end_time, status, input_params, attempt) VALUES (?,?,?,?,?,?,?,?,?,?)"; byte[] inputParam = null; if (inputProps != null) { try { final String jsonString = JSONUtils.toJSON(PropsUtils.toHierarchicalMap(inputProps)); inputParam = GZIPUtils.gzipString(jsonString, "UTF-8"); } catch (final IOException e) { throw new ExecutorManagerException("Error encoding input params"); } } final ExecutableFlow flow = node.getExecutableFlow(); final String flowId = node.getParentFlow().getFlowPath(); logger.info("Uploading flowId " + flowId); try { this.dbOperator.update(INSERT_EXECUTION_NODE, flow.getExecutionId(), flow.getProjectId(), flow.getVersion(), flowId, node.getId(), node.getStartTime(), node.getEndTime(), node.getStatus().getNumVal(), inputParam, node.getAttempt()); } catch (final SQLException e) { throw new ExecutorManagerException("Error writing job " + node.getId(), e); } }
public void updateExecutableNode(final ExecutableNode node) throws ExecutorManagerException { final String UPSERT_EXECUTION_NODE = "UPDATE execution_jobs " + "SET start_time=?, end_time=?, status=?, output_params=? " + "WHERE exec_id=? AND flow_id=? AND job_id=? AND attempt=?"; byte[] outputParam = null; final Props outputProps = node.getOutputProps(); if (outputProps != null) { try { final String jsonString = JSONUtils.toJSON(PropsUtils.toHierarchicalMap(outputProps)); outputParam = GZIPUtils.gzipString(jsonString, "UTF-8"); } catch (final IOException e) { throw new ExecutorManagerException("Error encoding input params"); } } try { this.dbOperator.update(UPSERT_EXECUTION_NODE, node.getStartTime(), node .getEndTime(), node.getStatus().getNumVal(), outputParam, node .getExecutableFlow().getExecutionId(), node.getParentFlow() .getFlowPath(), node.getId(), node.getAttempt()); } catch (final SQLException e) { throw new ExecutorManagerException("Error updating job " + node.getId(), e); } }