private void cleanUpProjectOldInstallations(final Project project) throws ProjectManagerException, ExecutorManagerException { log.info("Cleaning up old install files older than " + (project.getVersion() - this.projectVersionRetention)); final Map<Integer, Pair<ExecutionReference, ExecutableFlow>> unfinishedFlows = this.executorLoader .fetchUnfinishedFlowsMetadata(); final List<Integer> versionsWithUnfinishedExecutions = unfinishedFlows.values() .stream().map(pair -> pair.getSecond()) .filter(exflow -> exflow.getProjectId() == project.getId()) .map(exflow -> exflow.getVersion()) .collect(Collectors.toList()); this.projectLoader.cleanOlderProjectVersion(project.getId(), project.getVersion() - this.projectVersionRetention, versionsWithUnfinishedExecutions); // Clean up storage this.storageManager.cleanupProjectArtifacts(project.getId()); }
private void assertTwoFlowSame(final ExecutableFlow flow1, final ExecutableFlow flow2, final boolean compareFlowData) { assertThat(flow1.getExecutionId()).isEqualTo(flow2.getExecutionId()); assertThat(flow1.getStatus()).isEqualTo(flow2.getStatus()); assertThat(flow1.getEndTime()).isEqualTo(flow2.getEndTime()); assertThat(flow1.getStartTime()).isEqualTo(flow2.getStartTime()); assertThat(flow1.getSubmitTime()).isEqualTo(flow2.getSubmitTime()); assertThat(flow1.getFlowId()).isEqualTo(flow2.getFlowId()); assertThat(flow1.getProjectId()).isEqualTo(flow2.getProjectId()); assertThat(flow1.getVersion()).isEqualTo(flow2.getVersion()); assertThat(flow1.getSubmitUser()).isEqualTo(flow2.getSubmitUser()); if (compareFlowData) { assertThat(flow1.getExecutionOptions().getFailureAction()) .isEqualTo(flow2.getExecutionOptions().getFailureAction()); assertThat(new HashSet<>(flow1.getEndNodes())).isEqualTo(new HashSet<>(flow2.getEndNodes())); } }
public synchronized void uploadExecutableFlow(final ExecutableFlow flow) throws ExecutorManagerException { final String INSERT_EXECUTABLE_FLOW = "INSERT INTO execution_flows " + "(project_id, flow_id, version, status, submit_time, submit_user, update_time) " + "values (?,?,?,?,?,?,?)"; final long submitTime = System.currentTimeMillis(); flow.setStatus(Status.PREPARING); flow.setSubmitTime(submitTime); /** * Why we need a transaction to get last insert ID? * Because "SELECT LAST_INSERT_ID()" needs to have the same connection * as inserting the new entry. * See https://dev.mysql.com/doc/refman/5.7/en/information-functions.html#function_last-insert-id */ final SQLTransaction<Long> insertAndGetLastID = transOperator -> { transOperator.update(INSERT_EXECUTABLE_FLOW, flow.getProjectId(), flow.getFlowId(), flow.getVersion(), Status.PREPARING.getNumVal(), submitTime, flow.getSubmitUser(), submitTime); transOperator.getConnection().commit(); return transOperator.getLastInsertId(); }; try { final long id = this.dbOperator.transaction(insertAndGetLastID); logger.info("Flow given " + flow.getFlowId() + " given id " + id); flow.setExecutionId((int) id); updateExecutableFlow(flow); } catch (final SQLException e) { throw new ExecutorManagerException("Error creating execution.", e); } }
assertThat(flow.getEndTime()).isEqualTo(info.getEndTime()); assertThat(flow.getProjectId()).isEqualTo(info.getProjectId()); assertThat(flow.getVersion()).isEqualTo(info.getVersion()); assertThat(flow.getFlowId()).isEqualTo(info.getFlowId());
public void uploadExecutableNode(final ExecutableNode node, final Props inputProps) throws ExecutorManagerException { final String INSERT_EXECUTION_NODE = "INSERT INTO execution_jobs " + "(exec_id, project_id, version, flow_id, job_id, start_time, " + "end_time, status, input_params, attempt) VALUES (?,?,?,?,?,?,?,?,?,?)"; byte[] inputParam = null; if (inputProps != null) { try { final String jsonString = JSONUtils.toJSON(PropsUtils.toHierarchicalMap(inputProps)); inputParam = GZIPUtils.gzipString(jsonString, "UTF-8"); } catch (final IOException e) { throw new ExecutorManagerException("Error encoding input params"); } } final ExecutableFlow flow = node.getExecutableFlow(); final String flowId = node.getParentFlow().getFlowPath(); logger.info("Uploading flowId " + flowId); try { this.dbOperator.update(INSERT_EXECUTION_NODE, flow.getExecutionId(), flow.getProjectId(), flow.getVersion(), flowId, node.getId(), node.getStartTime(), node.getEndTime(), node.getStatus().getNumVal(), inputParam, node.getAttempt()); } catch (final SQLException e) { throw new ExecutorManagerException("Error writing job " + node.getId(), e); } }
private ExecutableFlow getExecutableFlowMetadata( ExecutableFlow fullExFlow) { final Flow flow = new Flow(fullExFlow.getId()); final Project project = new Project(fullExFlow.getProjectId(), null); project.setVersion(fullExFlow.getVersion()); flow.setVersion(fullExFlow.getVersion()); final ExecutableFlow metadata = new ExecutableFlow(project, flow); metadata.setExecutionId(fullExFlow.getExecutionId()); metadata.setStatus(fullExFlow.getStatus()); metadata.setSubmitTime(fullExFlow.getSubmitTime()); metadata.setStartTime(fullExFlow.getStartTime()); metadata.setEndTime(fullExFlow.getEndTime()); metadata.setSubmitUser(fullExFlow.getSubmitUser()); return metadata; }
Assert.assertEquals(exA.getVersion(), exB.getVersion()); Assert.assertEquals(exA.getSubmitTime(), exB.getSubmitTime()); Assert.assertEquals(exA.getSubmitUser(), exB.getSubmitUser());
Pair<Integer, Integer> projectVersionKey = new Pair<Integer,Integer>(flow.getProjectId(), flow.getVersion()); projectVersion = installedProjects.get(projectVersionKey); if (projectVersion == null) { projectVersion = new ProjectVersion(flow.getProjectId(), flow.getVersion()); installedProjects.put(projectVersionKey, projectVersion);
for(FlowRunner runner: runningFlows.values()) { ExecutableFlow flow = runner.getExecutableFlow(); activeProjectVersions.add(new Pair<Integer,Integer>(flow.getProjectId(), flow.getVersion()));
props = projectLoader.fetchProjectProperty(flow.getProjectId(), flow.getVersion(), node.getId()+".jor");
@SuppressWarnings("unchecked") private void setupFlowExecution() { int projectId = flow.getProjectId(); int version = flow.getVersion(); String flowId = flow.getFlowId();
public synchronized void uploadExecutableFlow(final ExecutableFlow flow) throws ExecutorManagerException { final String INSERT_EXECUTABLE_FLOW = "INSERT INTO execution_flows " + "(project_id, flow_id, version, status, submit_time, submit_user, update_time) " + "values (?,?,?,?,?,?,?)"; final long submitTime = System.currentTimeMillis(); flow.setStatus(Status.PREPARING); /** * Why we need a transaction to get last insert ID? * Because "SELECT LAST_INSERT_ID()" needs to have the same connection * as inserting the new entry. * See https://dev.mysql.com/doc/refman/5.7/en/information-functions.html#function_last-insert-id */ final SQLTransaction<Long> insertAndGetLastID = transOperator -> { transOperator.update(INSERT_EXECUTABLE_FLOW, flow.getProjectId(), flow.getFlowId(), flow.getVersion(), Status.PREPARING.getNumVal(), submitTime, flow.getSubmitUser(), submitTime); transOperator.getConnection().commit(); return transOperator.getLastInsertId(); }; try { final long id = this.dbOperator.transaction(insertAndGetLastID); logger.info("Flow given " + flow.getFlowId() + " given id " + id); flow.setExecutionId((int) id); updateExecutableFlow(flow); } catch (final SQLException e) { throw new ExecutorManagerException("Error creating execution.", e); } }
flow.getProjectId(), flow.getFlowId(), flow.getVersion(), Status.PREPARING.getNumVal(), submitTime,
flow.getExecutionId(), flow.getProjectId(), flow.getVersion(), flowId, node.getId(),
public void uploadExecutableNode(final ExecutableNode node, final Props inputProps) throws ExecutorManagerException { final String INSERT_EXECUTION_NODE = "INSERT INTO execution_jobs " + "(exec_id, project_id, version, flow_id, job_id, start_time, " + "end_time, status, input_params, attempt) VALUES (?,?,?,?,?,?,?,?,?,?)"; byte[] inputParam = null; if (inputProps != null) { try { final String jsonString = JSONUtils.toJSON(PropsUtils.toHierarchicalMap(inputProps)); inputParam = GZIPUtils.gzipString(jsonString, "UTF-8"); } catch (final IOException e) { throw new ExecutorManagerException("Error encoding input params"); } } final ExecutableFlow flow = node.getExecutableFlow(); final String flowId = node.getParentFlow().getFlowPath(); logger.info("Uploading flowId " + flowId); try { this.dbOperator.update(INSERT_EXECUTION_NODE, flow.getExecutionId(), flow.getProjectId(), flow.getVersion(), flowId, node.getId(), node.getStartTime(), node.getEndTime(), node.getStatus().getNumVal(), inputParam, node.getAttempt()); } catch (final SQLException e) { throw new ExecutorManagerException("Error writing job " + node.getId(), e); } }