void executeJob(Job job, String token) { try { executorManager.execute(job, token); } catch (Exception e) { logger.error("Error executing job {}.", job.getUid(), e); } }
void checkQueuedJob(Job job, Path tempJobFolder, CatalogIOManager catalogIOManager) { Path tmpOutdirPath = getJobTemporaryFolder(job.getUid(), tempJobFolder); if (!tmpOutdirPath.toFile().exists()) { logger.warn("Attempting to create the temporal output directory again"); try { catalogIOManager.createDirectory(tmpOutdirPath.toUri()); } catch (CatalogIOException e) { logger.error("Could not create the temporal output directory to run the job"); } } else { String status = executorManager.status(tmpOutdirPath, job); if (!status.equalsIgnoreCase(Job.JobStatus.UNKNOWN) && !status.equalsIgnoreCase(Job.JobStatus.QUEUED)) { try { logger.info("Updating job {} from {} to {}", job.getUid(), Job.JobStatus.QUEUED, Job.JobStatus.RUNNING); setNewStatus(job.getUid(), Job.JobStatus.RUNNING, "The job is running"); } catch (CatalogException e) { logger.warn("Could not update job {} to status running", job.getUid()); } } } }
void closeSessionId(Job job) { // Remove the session id from the job attributes job.getAttributes().remove("sessionId"); ObjectMap params = new ObjectMap(JobDBAdaptor.QueryParams.ATTRIBUTES.key(), job.getAttributes()); try { jobDBAdaptor.update(job.getUid(), params, QueryOptions.empty()); } catch (CatalogException e) { logger.error("Could not remove session id from attributes of job {}. ", job.getUid(), e); } } }
void cleanPrivateJobInformation(Job job) { // Remove the session id from the job attributes job.getAttributes().remove(Job.OPENCGA_TMP_DIR); job.getAttributes().remove(Job.OPENCGA_OUTPUT_DIR); job.getAttributes().remove(Job.OPENCGA_STUDY); ObjectMap params = new ObjectMap(JobDBAdaptor.QueryParams.ATTRIBUTES.key(), job.getAttributes()); try { dbAdaptorFactory.getCatalogJobDBAdaptor().update(job.getUid(), params, QueryOptions.empty()); } catch (CatalogException e) { logger.error("Could not remove session id from attributes of job {}. ", job.getUid(), e); } }
@Deprecated @Override public QueryResult<Job> run(Job job) throws Exception { // TODO: Lock job before submit. Avoid double submission // SgeManager.queueJob(job.getToolName(), job.getResourceManagerAttributes().get(Job.JOB_SCHEDULER_NAME).toString(), // -1, job.getTmpOutDirUri().getPath(), job.getCommandLine(), null, "job." + job.getId()); return catalogManager.getJobManager().update(job.getUid(), new ObjectMap(JobDBAdaptor.QueryParams.STATUS_NAME.key(), Job.JobStatus.QUEUED), null, sessionId); //TODO: Add query options }
@Override protected String getStatus(Job job) { String status; try { status = SgeManager.status(Objects.toString(job.getResourceManagerAttributes().get(Job.JOB_SCHEDULER_NAME))); } catch (Exception e) { logger.error("Could not obtain the status of the job {} from SGE. Error: {}", job.getUid(), e.getMessage()); return Job.JobStatus.UNKNOWN; } switch (status) { case SgeManager.ERROR: case SgeManager.EXECUTION_ERROR: return Job.JobStatus.ERROR; case SgeManager.FINISHED: return Job.JobStatus.READY; case SgeManager.QUEUED: return Job.JobStatus.QUEUED; case SgeManager.RUNNING: case SgeManager.TRANSFERRED: return Job.JobStatus.RUNNING; case SgeManager.UNKNOWN: default: return job.getStatus().getName(); } }
private void checkPreparedJob(Job job) { Path path = getJobTemporaryFolder(job.getUid(), tempJobFolder); try { catalogIOManager.createDirectory(path.toUri()); String stderr = path.resolve(job.getName() + '_' + job.getUid() + ".err").toString(); String stdout = path.resolve(job.getName() + '_' + job.getUid() + ".out").toString(); commandLine.append("tools execute "); commandLine.append("--job ").append(job.getUid()).append(' '); logger.info("Updating job {} from {} to {}", job.getUid(), Job.JobStatus.PREPARED, Job.JobStatus.QUEUED); try { job.getAttributes().put(Job.OPENCGA_TMP_DIR, path.toString()); .append(JobDBAdaptor.QueryParams.RESOURCE_MANAGER_ATTRIBUTES.key(), job.getResourceManagerAttributes()); QueryResult<Job> update = jobDBAdaptor.update(job.getUid(), params, QueryOptions.empty()); if (update.getNumResults() == 1) { job = update.first(); executeJob(job, userToken); } else { logger.error("Could not update nor run job {}" + job.getUid()); logger.error("Could not update job {}. {}", job.getUid(), e.getMessage()); e.printStackTrace();
private void checkRunningJob(Job job) throws CatalogIOException { Path tmpOutdirPath = getJobTemporaryFolder(job.getUid(), tempJobFolder); Job.JobStatus jobStatus; jobStatus = new Job.JobStatus(Job.JobStatus.ERROR, "Temporal output directory not found"); try { logger.info("Updating job {} from {} to {}", job.getUid(), Job.JobStatus.RUNNING, jobStatus.getName()); outputRecorder.updateJobStatus(job, jobStatus); } catch (CatalogException e) { logger.warn("Could not update job {} to status error", job.getUid(), e); } finally { cleanPrivateJobInformation(job); String status = executorManager.status(tmpOutdirPath, job); if (!status.equalsIgnoreCase(Job.JobStatus.UNKNOWN) && !status.equalsIgnoreCase(Job.JobStatus.RUNNING)) { logger.info("Updating job {} from {} to {}", job.getUid(), Job.JobStatus.RUNNING, status); try { outputRecorder.updateJobStatus(job, new Job.JobStatus(status));
catalogManager.getJobManager().update(job.getUid(), parameters, null, sessionId); catalogManager.getJobManager().update(job.getUid(), parameters, null, sessionId); objectMapper.writer().writeValue(outdir.resolve(JOB_STATUS_FILE).toFile(), new Job.JobStatus(Job.JobStatus.ERROR, "Job finished with error.")); return catalogManager.getJobManager().get(job.getUid(), new QueryOptions(), sessionId);
public List<QueryResult<JobAclEntry>> getAcls(String studyStr, List<String> jobList, String member, boolean silent, String sessionId) throws CatalogException { List<QueryResult<JobAclEntry>> jobAclList = new ArrayList<>(jobList.size()); for (String job : jobList) { try { MyResource<Job> resource = getUid(job, studyStr, sessionId); QueryResult<JobAclEntry> allJobAcls; if (StringUtils.isNotEmpty(member)) { allJobAcls = authorizationManager.getJobAcl(resource.getStudy().getUid(), resource.getResource().getUid(), resource.getUser(), member); } else { allJobAcls = authorizationManager.getAllJobAcls(resource.getStudy().getUid(), resource.getResource().getUid(), resource.getUser()); } allJobAcls.setId(job); jobAclList.add(allJobAcls); } catch (CatalogException e) { if (silent) { jobAclList.add(new QueryResult<>(job, 0, 0, 0, "", e.toString(), new ArrayList<>(0))); } else { throw e; } } } return jobAclList; }
public int execute(Job job) throws CatalogException, AnalysisExecutionException { return execute(job.getToolId(), job.getExecution(), catalogManager.getJobManager().getStudyId(job.getUid()), job.getParams()); }
@Override public Document convertToStorageType(File file, List<VariableSet> variableSetList) { Document document = super.convertToStorageType(file, variableSetList); document.remove(SampleDBAdaptor.QueryParams.ANNOTATION_SETS.key()); document.put("uid", file.getUid()); document.put("studyUid", file.getStudyUid()); long jobId = file.getJob() != null ? (file.getJob().getUid() == 0 ? -1L : file.getJob().getUid()) : -1L; document.put("job", new Document("uid", jobId)); document.put("samples", convertSamples(file.getSamples())); document.put(FileMongoDBAdaptor.REVERSE_NAME, StringUtils.reverse(file.getName())); return document; }
private void checkGetAllJobs(Collection<Long> expectedJobs, String sessionId) throws CatalogException { QueryResult<Job> allJobs = catalogManager.getJobManager().get(studyFqn, (Query) null, null, sessionId); assertEquals(expectedJobs.size(), allJobs.getNumResults()); allJobs.getResult().forEach(job -> assertTrue(expectedJobs + " does not contain job " + job.getName(), expectedJobs.contains(job .getUid()))); }
public static Job runStorageJob(CatalogManager catalogManager, Job job, Logger logger, String sessionId) throws CatalogException, IOException { try { ExecutorManager.execute(catalogManager, job, sessionId); } catch (ExecutionException e) { throw new IOException(e.getCause()); } return catalogManager.getJobManager().get(job.getUid(), null, sessionId).first(); }
@Test public void getJobTest() throws CatalogException { long studyId = user3.getProjects().get(0).getStudies().get(0).getUid(); Job job = catalogJobDBAdaptor.insert(new Job("name", user3.getId(), "", "", "", new File().setUid(4), Collections.emptyList(), 1), studyId, null).first(); long jobId = job.getUid(); job = catalogJobDBAdaptor.get(jobId, null).first(); System.out.println(job); try { catalogJobDBAdaptor.get(-1, null); fail("error: expected exception"); } catch (CatalogDBException e) { System.out.println("correct exception: " + e); } }
@Test public void getAllJobs() throws CatalogException { Job job = new Job() .setId("job1") .setToolId("toolId") .setOutDir(new File().setPath(data_d1_d2)); long job1 = catalogManager.getJobManager().create(studyFqn, job, null, ownerSessionId).first().getUid(); job.setId("job2"); long job2 = catalogManager.getJobManager().create(studyFqn, job, null, ownerSessionId).first().getUid(); job.setId("job3"); long job3 = catalogManager.getJobManager().create(studyFqn, job, null, ownerSessionId).first().getUid(); job.setId("job4"); long job4 = catalogManager.getJobManager().create(studyFqn, job, null, ownerSessionId).first().getUid(); checkGetAllJobs(Arrays.asList(job1, job2, job3, job4), ownerSessionId); //Owner can see everything checkGetAllJobs(Collections.emptyList(), externalSessionId); //Can't see inside data_d1_d2_d3 }
@Override public Document convertToStorageType(Job object) { Document document = super.convertToStorageType(object); document.put("uid", object.getUid()); document.put("studyUid", object.getStudyUid()); document.put("outDir", convertFileToDocument(object.getOutDir())); document.put("input", convertFilesToDocument(object.getInput())); document.put("output", convertFilesToDocument(object.getOutput())); return document; }
@Test public void SetVisitedJob() throws CatalogException { long studyId = user3.getProjects().get(0).getStudies().get(0).getUid(); Job jobBefore = catalogJobDBAdaptor.insert(new Job("name", user3.getId(), "", "", "", new File().setUid(4), Collections.emptyList(), 1), studyId, null).first(); long jobId = jobBefore.getUid(); assertTrue(!jobBefore.isVisited()); ObjectMap params = new ObjectMap(JobDBAdaptor.QueryParams.VISITED.key(), true); catalogJobDBAdaptor.update(jobBefore.getUid(), params, QueryOptions.empty()); Job jobAfter = catalogJobDBAdaptor.get(jobId, null).first(); assertTrue(jobAfter.isVisited()); }
@Test public void deleteJobTest() throws CatalogException { long studyId = user3.getProjects().get(0).getStudies().get(0).getUid(); Job job = catalogJobDBAdaptor.insert(new Job("name", user3.getId(), "", "", "", new File().setUid(4), Collections.emptyList(), 1), studyId, null).first(); long jobId = job.getUid(); assertEquals(Job.JobStatus.PREPARED, job.getStatus().getName()); catalogJobDBAdaptor.delete(jobId); thrown.expect(CatalogDBException.class); thrown.expectMessage("not exist"); catalogJobDBAdaptor.get(jobId, QueryOptions.empty()); }
@Test public void updateInputAndOutputFiles() throws Exception { Job job = new Job() .setOutDir(new File().setUid(5)) .setStatus(new Job.JobStatus()); long studyId = user3.getProjects().get(0).getStudies().get(0).getUid(); job.setName("jobName1"); QueryResult<Job> insert = catalogJobDBAdaptor.insert(job, studyId, null); List<File> fileInput = Arrays.asList( new File().setUid(5L).setName("file1").setStatus(new File.FileStatus()), new File().setUid(6L).setName("file2").setStatus(new File.FileStatus()), new File().setUid(7L).setName("file3").setStatus(new File.FileStatus()) ); List<File> fileOutput = Arrays.asList( new File().setUid(15L).setName("file1").setStatus(new File.FileStatus()), new File().setUid(16L).setName("file2").setStatus(new File.FileStatus()), new File().setUid(17L).setName("file3").setStatus(new File.FileStatus()) ); ObjectMap params = new ObjectMap() .append(JobDBAdaptor.QueryParams.INPUT.key(), fileInput) .append(JobDBAdaptor.QueryParams.OUTPUT.key(), fileOutput); QueryResult<Job> update = catalogJobDBAdaptor.update(insert.first().getUid(), params, QueryOptions.empty()); assertEquals(3, update.first().getInput().size()); assertEquals(3, update.first().getOutput().size()); assertTrue(Arrays.asList(5L, 6L, 7L).containsAll(update.first().getInput().stream().map(File::getUid).collect(Collectors.toList()))); assertTrue(Arrays.asList(15L, 16L, 17L) .containsAll(update.first().getOutput().stream().map(File::getUid).collect(Collectors.toList()))); }