private void removeJobReferences(MyResources<File> resource) throws CatalogException { QueryOptions options = new QueryOptions(QueryOptions.INCLUDE, Arrays.asList( JobDBAdaptor.QueryParams.UID.key(), JobDBAdaptor.QueryParams.INPUT.key(), JobDBAdaptor.QueryParams.OUTPUT.key(), JobDBAdaptor.QueryParams.OUT_DIR.key(), JobDBAdaptor.QueryParams.ATTRIBUTES.key())); .append(JobDBAdaptor.QueryParams.STUDY_UID.key(), resource.getStudy().getUid()) .append(JobDBAdaptor.QueryParams.INPUT_UID.key(), resourceList) .append(JobDBAdaptor.QueryParams.STATUS_NAME.key(), Constants.ALL_STATUS); QueryResult<Job> jobInputFiles = jobDBAdaptor.get(query, options); .append(JobDBAdaptor.QueryParams.STUDY_UID.key(), resource.getStudy().getUid()) .append(JobDBAdaptor.QueryParams.OUTPUT_UID.key(), resourceList) .append(JobDBAdaptor.QueryParams.STATUS_NAME.key(), Constants.ALL_STATUS); QueryResult<Job> jobOutputFiles = jobDBAdaptor.get(query, options); .append(JobDBAdaptor.QueryParams.STUDY_UID.key(), resource.getStudy().getUid()) .append(JobDBAdaptor.QueryParams.OUT_DIR_UID.key(), resourceList) .append(JobDBAdaptor.QueryParams.STATUS_NAME.key(), Constants.ALL_STATUS); QueryResult<Job> jobOutDirFolders = jobDBAdaptor.get(query, options); .append(JobDBAdaptor.QueryParams.ATTRIBUTES.key(), attributes) .append(JobDBAdaptor.QueryParams.INPUT.key(), inputFiles); jobDBAdaptor.update(job.getUid(), params, QueryOptions.empty()); .append(JobDBAdaptor.QueryParams.ATTRIBUTES.key(), attributes) .append(JobDBAdaptor.QueryParams.OUTPUT.key(), outputFiles); jobDBAdaptor.update(job.getUid(), params, QueryOptions.empty()); .append(JobDBAdaptor.QueryParams.ATTRIBUTES.key(), attributes) .append(JobDBAdaptor.QueryParams.OUT_DIR.key(), -1L);
fixComplexQueryParam(QueryParams.ATTRIBUTES.key(), query); fixComplexQueryParam(QueryParams.BATTRIBUTES.key(), query); fixComplexQueryParam(QueryParams.NATTRIBUTES.key(), query); QueryParams queryParam = (QueryParams.getParam(entry.getKey()) != null) ? QueryParams.getParam(entry.getKey()) : QueryParams.getParam(key); if (queryParam == null) { throw new CatalogDBException("Unexpected parameter " + entry.getKey() + ". The parameter does not exist or cannot be " switch (queryParam) { case UID: addAutoOrQuery(PRIVATE_UID, queryParam.key(), query, queryParam.type(), andBsonList); break; case STUDY_UID: addAutoOrQuery(PRIVATE_STUDY_ID, queryParam.key(), query, queryParam.type(), andBsonList); break; case ATTRIBUTES: addAutoOrQuery(entry.getKey(), entry.getKey(), query, queryParam.type(), andBsonList); break; case RESOURCE_MANAGER_ATTRIBUTES: addAutoOrQuery(entry.getKey(), entry.getKey(), query, queryParam.type(), andBsonList); break; case BATTRIBUTES: String mongoKey = entry.getKey().replace(QueryParams.BATTRIBUTES.key(), QueryParams.ATTRIBUTES.key()); addAutoOrQuery(mongoKey, entry.getKey(), query, queryParam.type(), andBsonList); break; case NATTRIBUTES: mongoKey = entry.getKey().replace(QueryParams.NATTRIBUTES.key(), QueryParams.ATTRIBUTES.key());
outDir = catalogManager.getFileManager().createFolder(studyStr, outputDir, new File.FileStatus(), true, "", QueryOptions.empty(), userToken).first(); parameters.append(JobDBAdaptor.QueryParams.OUT_DIR.key(), outDir); } catch (CatalogException e) { logger.error("Cannot find file {}. Error: {}", job.getOutDir().getPath(), e.getMessage()); parameters.put(JobDBAdaptor.QueryParams.OUTPUT.key(), files); parameters.put(JobDBAdaptor.QueryParams.END_TIME.key(), System.currentTimeMillis()); try { catalogManager.getJobManager().update(studyStr, job.getUuid(), parameters, null, this.sessionId);
String status = executorManager.status(tmpOutdirPath, job); if (!status.equalsIgnoreCase(Job.JobStatus.UNKNOWN) && !status.equalsIgnoreCase(Job.JobStatus.RUNNING)) { ObjectMap parameters = new ObjectMap(JobDBAdaptor.QueryParams.END_TIME.key(), System.currentTimeMillis());
.append(JobDBAdaptor.QueryParams.STATUS_NAME.key(), Job.JobStatus.RUNNING) .append(JobDBAdaptor.QueryParams.TYPE.key(), "!=" + Job.Type.INDEX); Query queuedJobsQuery = new Query() .append(JobDBAdaptor.QueryParams.STATUS_NAME.key(), Job.JobStatus.QUEUED) .append(JobDBAdaptor.QueryParams.TYPE.key(), "!=" + Job.Type.INDEX); Query preparedJobsQuery = new Query() .append(JobDBAdaptor.QueryParams.STATUS_NAME.key(), Job.JobStatus.PREPARED) .append(JobDBAdaptor.QueryParams.TYPE.key(), "!=" + Job.Type.INDEX); .append(QueryOptions.SORT, JobDBAdaptor.QueryParams.CREATION_DATE.key()) .append(QueryOptions.ORDER, QueryOptions.ASCENDING);
finalQuery.append(JobDBAdaptor.QueryParams.STUDY_UID.key(), study.getUid()); .append(JobDBAdaptor.QueryParams.UID.key(), job.getUid()) .append(JobDBAdaptor.QueryParams.STUDY_UID.key(), study.getUid()); ObjectMap updateParams = new ObjectMap() .append(JobDBAdaptor.QueryParams.STATUS_NAME.key(), Status.DELETED) .append(JobDBAdaptor.QueryParams.NAME.key(), job.getName() + suffixName); QueryResult<Long> update = jobDBAdaptor.update(updateQuery, updateParams, QueryOptions.empty()); if (update.first() > 0) {
updateObjectMap.put(JobDBAdaptor.QueryParams.COMMAND_LINE.key(), commandLine.toString()); updateObjectMap.put(JobDBAdaptor.QueryParams.START_TIME.key(), System.currentTimeMillis()); updateObjectMap.put(JobDBAdaptor.QueryParams.ATTRIBUTES.key(), job.getAttributes()); updateObjectMap.put(JobDBAdaptor.QueryParams.RESOURCE_MANAGER_ATTRIBUTES.key(), job.getResourceManagerAttributes());
query.put(JobDBAdaptor.QueryParams.UID.key(), jobId); Job job = jobManager.get(null, query, QueryOptions.empty(), sessionId).first(); long studyUid = jobManager.getStudyId(jobId);
.append(JobDBAdaptor.QueryParams.COMMAND_LINE.key(), commandLine.toString()) .append(JobDBAdaptor.QueryParams.STATUS_NAME.key(), Job.JobStatus.QUEUED) .append(JobDBAdaptor.QueryParams.ATTRIBUTES.key(), job.getAttributes()) .append(JobDBAdaptor.QueryParams.RESOURCE_MANAGER_ATTRIBUTES.key(), job.getResourceManagerAttributes());
parameters.put(JobDBAdaptor.QueryParams.ERROR.key(), error); parameters.put(JobDBAdaptor.QueryParams.ERROR_DESCRIPTION.key(), Job.ERROR_DESCRIPTIONS.get(error)); catalogManager.getJobManager().update(job.getUid(), parameters, null, sessionId); objectMapper.writer().writeValue(outdir.resolve(JOB_STATUS_FILE).toFile(), new Job.JobStatus(Job.JobStatus.ERROR,
study.setJobs(dbAdaptorFactory.getCatalogJobDBAdaptor().getAllInStudy(studyId, options).getResult()); } else { Query query = new Query(JobDBAdaptor.QueryParams.STUDY_UID.key(), studyId); study.setJobs(dbAdaptorFactory.getCatalogJobDBAdaptor().get(query, options, user).getResult());
String[] acceptedParams = {QueryParams.NAME.key(), QueryParams.USER_ID.key(), QueryParams.TOOL_NAME.key(), QueryParams.CREATION_DATE.key(), QueryParams.DESCRIPTION.key(), QueryParams.OUTPUT_ERROR.key(), QueryParams.COMMAND_LINE.key(), QueryParams.ERROR.key(), QueryParams.ERROR_DESCRIPTION.key(), }; filterStringParams(parameters, jobParameters, acceptedParams); if (parameters.containsKey(QueryParams.STATUS_NAME.key())) { jobParameters.put(QueryParams.STATUS_NAME.key(), parameters.get(QueryParams.STATUS_NAME.key())); jobParameters.put(QueryParams.STATUS_DATE.key(), TimeUtils.getTime()); if (parameters.containsKey(QueryParams.STATUS.key()) && parameters.get(QueryParams.STATUS.key()) instanceof Job.JobStatus) { jobParameters.put(QueryParams.STATUS.key(), getMongoDBDocument(parameters.get(QueryParams.STATUS.key()), "Job.JobStatus")); String[] acceptedBooleanParams = {QueryParams.VISITED.key(), }; filterBooleanParams(parameters, jobParameters, acceptedBooleanParams); String[] acceptedLongParams = {QueryParams.START_TIME.key(), QueryParams.END_TIME.key(), QueryParams.SIZE.key(), QueryParams.OUT_DIR_UID.key(), }; filterLongParams(parameters, jobParameters, acceptedLongParams); if (parameters.containsKey(QueryParams.INPUT.key())) { List<File> fileList = parameters.getAsList(QueryParams.INPUT.key(), File.class); jobParameters.put(QueryParams.INPUT.key(), jobConverter.convertFilesToDocument(fileList)); if (parameters.containsKey(QueryParams.OUTPUT.key())) { List<File> fileList = parameters.getAsList(QueryParams.OUTPUT.key(), File.class); jobParameters.put(QueryParams.OUTPUT.key(), jobConverter.convertFilesToDocument(fileList)); String[] acceptedMapParams = {QueryParams.ATTRIBUTES.key(), QueryParams.RESOURCE_MANAGER_ATTRIBUTES.key()};
/** * Job methods * *************************** */ @Test public void testCreateJob() throws CatalogException, IOException { Query query = new Query(StudyDBAdaptor.QueryParams.OWNER.key(), "user"); String studyId = catalogManager.getStudyManager().get(query, null, sessionIdUser).first().getId(); File outDir = catalogManager.getFileManager().createFolder(studyId, Paths.get("jobs", "myJob").toString(), null, true, null, QueryOptions.empty(), sessionIdUser).first(); catalogManager.getJobManager().create(studyId,new Job().setId("myJob").setToolId("samtool").setDescription("description") .setOutDir(outDir).setExecution("echo \"Hello world!\"").setStatus(new Job.JobStatus(Job.JobStatus.PREPARED)), null, sessionIdUser); catalogManager.getJobManager().create(studyId, new Job().setId("myReadyJob").setToolId("samtool").setDescription("description") .setOutDir(outDir).setExecution("echo \"Hello world!\"").setStatus(new Job.JobStatus(Job.JobStatus.READY)), null, sessionIdUser); catalogManager.getJobManager().create(studyId, new Job().setId("myQueuedJob").setToolId("samtool").setDescription("description") .setOutDir(outDir).setExecution("echo \"Hello world!\"").setStatus(new Job.JobStatus(Job.JobStatus.QUEUED)), null, sessionIdUser); catalogManager.getJobManager().create(studyId, new Job().setId("myErrorJob").setToolId("samtool").setDescription("description") .setOutDir(outDir).setExecution("echo \"Hello world!\"").setStatus(new Job.JobStatus(Job.JobStatus.ERROR)), null, sessionIdUser); query = new Query() .append(JobDBAdaptor.QueryParams.STATUS_NAME.key(), Arrays.asList(Job.JobStatus.PREPARED, Job.JobStatus.QUEUED, Job.JobStatus.RUNNING, Job.JobStatus.DONE)); QueryResult<Job> unfinishedJobs = catalogManager.getJobManager().get(String.valueOf(studyId), query, null, sessionIdUser); assertEquals(2, unfinishedJobs.getNumResults()); QueryResult<Job> allJobs = catalogManager.getJobManager().get(String.valueOf(studyId), (Query) null, null, sessionIdUser); assertEquals(4, allJobs.getNumResults()); }
@Override Job smartResolutor(long studyUid, String entry, String user) throws CatalogException { Query query = new Query() .append(JobDBAdaptor.QueryParams.STUDY_UID.key(), studyUid); if (UUIDUtils.isOpenCGAUUID(entry)) { query.put(JobDBAdaptor.QueryParams.UUID.key(), entry); } else { query.put(JobDBAdaptor.QueryParams.ID.key(), entry); } QueryOptions options = new QueryOptions(QueryOptions.INCLUDE, Arrays.asList( JobDBAdaptor.QueryParams.UUID.key(), JobDBAdaptor.QueryParams.UID.key(), JobDBAdaptor.QueryParams.STUDY_UID.key(), JobDBAdaptor.QueryParams.ID.key(), JobDBAdaptor.QueryParams.STATUS.key())); QueryResult<Job> jobQueryResult = jobDBAdaptor.get(query, options, user); if (jobQueryResult.getNumResults() == 0) { jobQueryResult = jobDBAdaptor.get(query, options); if (jobQueryResult.getNumResults() == 0) { throw new CatalogException("Job " + entry + " not found"); } else { throw new CatalogAuthorizationException("Permission denied. " + user + " is not allowed to see the job " + entry); } } else if (jobQueryResult.getNumResults() > 1) { throw new CatalogException("More than one job found based on " + entry); } else { return jobQueryResult.first(); } }
@Override public QueryResult<Job> insert(Job job, long studyId, QueryOptions options) throws CatalogDBException { long startTime = startQuery(); this.dbAdaptorFactory.getCatalogStudyDBAdaptor().checkId(studyId); List<Bson> filterList = new ArrayList<>(); filterList.add(Filters.eq(QueryParams.ID.key(), job.getId())); filterList.add(Filters.eq(PRIVATE_STUDY_ID, studyId)); filterList.add(Filters.eq(QueryParams.STATUS_NAME.key(), Status.READY)); Bson bson = Filters.and(filterList); QueryResult<Long> count = jobCollection.count(bson); if (count.getResult().get(0) > 0) { throw new CatalogDBException("Job { id: '" + job.getId() + "'} already exists."); } long jobId = getNewId(); job.setUid(jobId); job.setStudyUid(studyId); if (StringUtils.isEmpty(job.getUuid())) { job.setUuid(UUIDUtils.generateOpenCGAUUID(UUIDUtils.Entity.JOB)); } Document jobObject = jobConverter.convertToStorageType(job); if (StringUtils.isNotEmpty(job.getCreationDate())) { jobObject.put(PRIVATE_CREATION_DATE, TimeUtils.toDate(job.getCreationDate())); } else { jobObject.put(PRIVATE_CREATION_DATE, TimeUtils.getDate()); } jobObject.put(PERMISSION_RULES_APPLIED, Collections.emptyList()); jobCollection.insert(jobObject, null); //TODO: Check results.get(0).getN() != 0 return endQuery("Create Job", startTime, get(jobId, filterOptions(options, FILTER_ROUTE_JOBS))); }
@Override public void checkJobPermission(long studyId, long jobId, String userId, JobAclEntry.JobPermissions permission) throws CatalogException { Query query = new Query() .append(JobDBAdaptor.QueryParams.UID.key(), jobId) .append(JobDBAdaptor.QueryParams.STUDY_UID.key(), studyId); StudyAclEntry.StudyPermissions studyPermission; switch (permission) { case VIEW: studyPermission = StudyAclEntry.StudyPermissions.VIEW_JOBS; break; case UPDATE: studyPermission = StudyAclEntry.StudyPermissions.WRITE_JOBS; break; case DELETE: studyPermission = StudyAclEntry.StudyPermissions.DELETE_JOBS; break; default: throw new CatalogAuthorizationException("Permission " + permission.toString() + " not found"); } if (checkUserPermission(userId, query, studyPermission, jobDBAdaptor)) { return; } throw CatalogAuthorizationException.deny(userId, permission.toString(), "Job", jobId, null); }