ParamUtils.checkParameter(cohort.getId(), "name"); ParamUtils.checkObj(cohort.getSamples(), "Sample list"); cohort.setType(ParamUtils.defaultObject(cohort.getType(), Study.Type.COLLECTION)); cohort.setCreationDate(TimeUtils.getTime()); cohort.setDescription(ParamUtils.defaultString(cohort.getDescription(), "")); cohort.setAnnotationSets(ParamUtils.defaultObject(cohort.getAnnotationSets(), Collections::emptyList)); cohort.setAttributes(ParamUtils.defaultObject(cohort.getAttributes(), HashMap::new)); cohort.setRelease(studyManager.getCurrentRelease(study, userId)); cohort.setStats(ParamUtils.defaultObject(cohort.getStats(), Collections::emptyMap)); cohort.setStatus(ParamUtils.defaultObject(cohort.getStatus(), Cohort.CohortStatus::new)); cohort.setSamples(ParamUtils.defaultObject(cohort.getSamples(), Collections::emptyList)); List<VariableSet> variableSetList = validateNewAnnotationSetsAndExtractVariableSets(study.getUid(), cohort.getAnnotationSets()); if (!cohort.getSamples().isEmpty()) { Query query = new Query() .append(SampleDBAdaptor.QueryParams.STUDY_UID.key(), study.getUid()) .append(SampleDBAdaptor.QueryParams.UID.key(), cohort.getSamples().stream() .map(Sample::getUid) .collect(Collectors.toList())); QueryResult<Long> count = sampleDBAdaptor.count(query); if (count.first() != cohort.getSamples().size()) { throw new CatalogException("Error: Some samples do not exist in the study " + study.getFqn()); cohort.setUuid(UUIDUtils.generateOpenCGAUUID(UUIDUtils.Entity.COHORT)); QueryResult<Cohort> queryResult = cohortDBAdaptor.insert(study.getUid(), cohort, variableSetList, null); auditManager.recordCreation(AuditRecord.Resource.cohort, queryResult.first().getUid(), userId, queryResult.first(), null, null); return queryResult;
@Test public void testCreateCohortAlreadyExisting() throws CatalogException { Sample sampleId1 = catalogManager.getSampleManager().create(studyFqn, new Sample().setId("SAMPLE_1"), new QueryOptions(), sessionIdUser).first(); catalogManager.getCohortManager().create(studyFqn, new Cohort().setId("MyCohort").setType(Study.Type.FAMILY) .setSamples(Collections.singletonList(sampleId1)), null, sessionIdUser).first(); thrown.expect(CatalogDBException.class); catalogManager.getCohortManager().create(studyFqn, new Cohort().setId("MyCohort").setType(Study.Type.FAMILY) .setSamples(Collections.singletonList(sampleId1)), null, sessionIdUser).first(); }
cohortSolrModel.setUid(cohort.getUid()); cohortSolrModel.setStudyId(study.getFqn().replace(":", "__")); cohortSolrModel.setType(cohort.getType().name()); Date date = TimeUtils.toDate(cohort.getCreationDate()); LocalDate localDate = date.toInstant().atZone(ZoneId.systemDefault()).toLocalDate(); cohortSolrModel.setCreationDay(localDate.getDayOfMonth()); cohortSolrModel.setCreationDayOfWeek(localDate.getDayOfWeek().toString()); cohortSolrModel.setStatus(cohort.getStatus().getName()); if (cohort.getSamples() != null) { cohortSolrModel.setNumSamples(cohort.getSamples().size()); } else { cohortSolrModel.setNumSamples(0); cohortSolrModel.setRelease(cohort.getRelease()); cohortSolrModel.setAnnotations(SolrConverterUtil.populateAnnotations(variableMap, cohort.getAnnotationSets())); if (cohort.getAnnotationSets() != null) { cohortSolrModel.setAnnotationSets(cohort.getAnnotationSets().stream().map(AnnotationSet::getId).collect(Collectors.toList())); } else { cohortSolrModel.setAnnotationSets(Collections.emptyList()); SolrConverterUtil.parseInternalOpenCGAAcls((List<Map<String, Object>>) cohort.getAttributes().get("OPENCGA_ACL")); List<String> effectivePermissions = SolrConverterUtil.getEffectivePermissions((Map<String, Set<String>>) study.getAttributes().get("OPENCGA_ACL"), cohortAcl,
static CatalogException unableToCalculateCohortReady(Cohort cohort) { return new CatalogException("Unable to calculate stats for cohort " + "{ uid: " + cohort.getUid() + " id: \"" + cohort.getId() + "\" }" + " with status \"" + cohort.getStatus().getName() + "\". " + "Resume or update stats for continue calculation"); }
public void checkCohortCanBeDeleted(Cohort cohort) throws CatalogException { // Check if the cohort is different from DEFAULT_COHORT if (StudyEntry.DEFAULT_COHORT.equals(cohort.getId())) { throw new CatalogException("Cohort " + StudyEntry.DEFAULT_COHORT + " cannot be deleted."); } // Check if the cohort can be deleted if (cohort.getStatus() != null && cohort.getStatus().getName() != null && !cohort.getStatus().getName().equals(Cohort.CohortStatus.NONE)) { throw new CatalogException("The cohort is used in storage."); } }
@Override public Document convertToStorageType(Cohort object, List<VariableSet> variableSetList) { Document document = super.convertToStorageType(object, variableSetList); document.remove(CohortDBAdaptor.QueryParams.ANNOTATION_SETS.key()); document.put("uid", object.getUid()); document.put("studyUid", object.getStudyUid()); document.put("samples", convertSamplesToDocument(object.getSamples())); return document; }
@Test public void CohortToSolrTest() { Study study = new Study().setFqn("user@project:study").setAttributes(new HashMap<>()); Cohort cohort = new Cohort("id", Study.Type.CASE_SET, TimeUtils.getTime(), "test", Arrays.asList(new Sample().setId("1"), new Sample().setId("2")), 2, null) .setAttributes(new HashMap<>()); cohort.setUid(200).setStatus(new Cohort.CohortStatus("CALCULATING")).setAnnotationSets(AnnotationHelper.createAnnotation()); CohortSolrModel cohortSolrModel = new CatalogCohortToSolrCohortConverter(study).convertToStorageType(cohort); assertEquals(cohortSolrModel.getUid(), cohort.getUid()); assertEquals(cohortSolrModel.getStatus(), cohort.getStatus().getName()); Date date = TimeUtils.toDate(cohort.getCreationDate()); LocalDate localDate = date.toInstant().atZone(ZoneId.systemDefault()).toLocalDate(); assertEquals(localDate.getDayOfMonth(), cohortSolrModel.getCreationDay()); assertEquals(localDate.getDayOfWeek().toString(), cohortSolrModel.getCreationDayOfWeek()); cohortSolrModel.setStatus(cohort.getStatus().getName()); assertEquals(cohortSolrModel.getType(), cohort.getType().name()); assertEquals(cohortSolrModel.getRelease(), cohort.getRelease()); assertEquals(cohortSolrModel.getNumSamples(), cohort.getSamples().size());
@Test public void testDeleteCohort() throws CatalogException, IOException { String studyId = "user@1000G:phase1"; Sample sampleId1 = catalogManager.getSampleManager().create(studyId, new Sample().setId("SAMPLE_1"), new QueryOptions(), sessionIdUser).first(); Sample sampleId2 = catalogManager.getSampleManager().create(studyId, new Sample().setId("SAMPLE_2"), new QueryOptions(), sessionIdUser).first(); Sample sampleId3 = catalogManager.getSampleManager().create(studyId, new Sample().setId("SAMPLE_3"), new QueryOptions(), sessionIdUser).first(); Cohort myCohort = catalogManager.getCohortManager().create(studyId, new Cohort().setId("MyCohort").setType(Study.Type.FAMILY) .setSamples(Arrays.asList(sampleId1, sampleId2, sampleId3)), null, sessionIdUser).first(); assertEquals("MyCohort", myCohort.getId()); assertEquals(3, myCohort.getSamples().size()); assertTrue(myCohort.getSamples().stream().map(Sample::getUid).collect(Collectors.toList()).contains(sampleId1.getUid())); assertTrue(myCohort.getSamples().stream().map(Sample::getUid).collect(Collectors.toList()).contains(sampleId2.getUid())); assertTrue(myCohort.getSamples().stream().map(Sample::getUid).collect(Collectors.toList()).contains(sampleId3.getUid())); WriteResult deleteResult = catalogManager.getCohortManager().delete(studyId, new Query(CohortDBAdaptor.QueryParams.UID.key(), myCohort.getUid()), null, sessionIdUser); assertEquals(1, deleteResult.getNumModified()); Query query = new Query() .append(CohortDBAdaptor.QueryParams.UID.key(), myCohort.getUid()) .append(CohortDBAdaptor.QueryParams.STATUS_NAME.key(), "!=" + Cohort.CohortStatus.READY); Cohort cohort = catalogManager.getCohortManager().get(studyId, query, null, sessionIdUser).first(); assertEquals(Status.DELETED, cohort.getStatus().getName()); }
/** * Sample methods * *************************** */ /* * Cohort methods * */ @Test public void testCreateCohort() throws CatalogException { Sample sampleId1 = catalogManager.getSampleManager().create(studyFqn, new Sample().setId("SAMPLE_1"), new QueryOptions(), sessionIdUser).first(); Sample sampleId2 = catalogManager.getSampleManager().create(studyFqn, new Sample().setId("SAMPLE_2"), new QueryOptions(), sessionIdUser).first(); Sample sampleId3 = catalogManager.getSampleManager().create(studyFqn, new Sample().setId("SAMPLE_3"), new QueryOptions(), sessionIdUser).first(); Cohort myCohort = catalogManager.getCohortManager().create(studyFqn, new Cohort().setId("MyCohort") .setSamples(Arrays.asList(sampleId1, sampleId2, sampleId3)), null, sessionIdUser).first(); assertEquals("MyCohort", myCohort.getId()); assertEquals(3, myCohort.getSamples().size()); assertTrue(myCohort.getSamples().stream().map(Sample::getUid).collect(Collectors.toList()).contains(sampleId1.getUid())); assertTrue(myCohort.getSamples().stream().map(Sample::getUid).collect(Collectors.toList()).contains(sampleId2.getUid())); assertTrue(myCohort.getSamples().stream().map(Sample::getUid).collect(Collectors.toList()).contains(sampleId3.getUid())); }
protected List<StoragePipelineResult> loadFiles(List<File> files, List<File> expectedLoadedFiles, QueryOptions queryOptions, String outputId) throws Exception { queryOptions.append(VariantFileIndexerStorageOperation.TRANSFORM, false); queryOptions.append(VariantFileIndexerStorageOperation.LOAD, true); queryOptions.append(StorageOperation.CATALOG_PATH, outputId); boolean calculateStats = queryOptions.getBoolean(VariantStorageEngine.Options.CALCULATE_STATS.key()); String studyId = catalogManager.getFileManager().getStudy(files.get(0), sessionId).getId(); List<String> fileIds = files.stream().map(File::getId).collect(Collectors.toList()); String outdir = opencga.createTmpOutdir(studyId, "_LOAD_", sessionId); List<StoragePipelineResult> etlResults = variantManager.index(studyId, fileIds, outdir, queryOptions, sessionId); assertEquals(expectedLoadedFiles.size(), etlResults.size()); checkEtlResults(studyId, etlResults, FileIndex.IndexStatus.READY); Cohort defaultCohort = getDefaultCohort(studyId); for (File file : expectedLoadedFiles) { List<Long> samplesInFile = file.getSamples().stream().map(Sample::getUid).collect(Collectors.toList()); List<Long> samplesInCohort = defaultCohort.getSamples().stream().map(Sample::getUid).collect(Collectors.toList()); assertTrue(samplesInCohort.containsAll(samplesInFile)); } if (calculateStats) { assertEquals(Cohort.CohortStatus.READY, defaultCohort.getStatus().getName()); checkCalculatedStats(studyId, Collections.singletonMap(DEFAULT_COHORT, defaultCohort), catalogManager, dbName, sessionId); } return etlResults; }
@Deprecated public void checkInUse(long sampleId) throws CatalogDBException { long studyId = getStudyId(sampleId); Query query = new Query(FileDBAdaptor.QueryParams.SAMPLE_UIDS.key(), sampleId); QueryOptions queryOptions = new QueryOptions(MongoDBCollection.INCLUDE, Arrays.asList(FILTER_ROUTE_FILES + FileDBAdaptor .QueryParams.UID.key(), FILTER_ROUTE_FILES + FileDBAdaptor.QueryParams.PATH.key())); QueryResult<File> fileQueryResult = dbAdaptorFactory.getCatalogFileDBAdaptor().get(query, queryOptions); if (fileQueryResult.getNumResults() != 0) { String msg = "Can't delete Sample " + sampleId + ", still in use in \"sampleId\" array of files : " + fileQueryResult.getResult().stream() .map(file -> "{ id: " + file.getUid() + ", path: \"" + file.getPath() + "\" }") .collect(Collectors.joining(", ", "[", "]")); throw new CatalogDBException(msg); } queryOptions = new QueryOptions(CohortDBAdaptor.QueryParams.SAMPLES.key(), sampleId) .append(MongoDBCollection.INCLUDE, Arrays.asList(FILTER_ROUTE_COHORTS + CohortDBAdaptor.QueryParams.UID.key(), FILTER_ROUTE_COHORTS + CohortDBAdaptor.QueryParams.ID.key())); QueryResult<Cohort> cohortQueryResult = dbAdaptorFactory.getCatalogCohortDBAdaptor().getAllInStudy(studyId, queryOptions); if (cohortQueryResult.getNumResults() != 0) { String msg = "Can't delete Sample " + sampleId + ", still in use in cohorts : " + cohortQueryResult.getResult().stream() .map(cohort -> "{ id: " + cohort.getUid() + ", name: \"" + cohort.getId() + "\" }") .collect(Collectors.joining(", ", "[", "]")); throw new CatalogDBException(msg); } }
switch (cohort.getStatus().getName()) { case Cohort.CohortStatus.NONE: case Cohort.CohortStatus.INVALID: throw new IllegalStateException("Unknown status " + cohort.getStatus().getName()); cohortMap.put(cohort.getId(), cohort.getSamples().stream().map(Sample::getId).collect(Collectors.toList()));
.get(studyConfiguration.getStudyName(), cohortName, null, sessionId).first(); List<String> cohortFromCatalog = defaultCohort .getSamples() .stream() .map(Sample::getId) if (defaultCohort.getStatus().getName().equals(Cohort.CohortStatus.CALCULATING)) { String status; if (studyConfiguration.getInvalidStats().contains(cohortId)) { while (iterator.hasNext()) { Cohort cohort = iterator.next(); if (cohort.getStatus() != null && cohort.getStatus().getName().equals(Cohort.CohortStatus.INVALID)) { if (cohort.getSamples().size() != studyConfiguration.getCohorts().get(cohortId).size()) { System.out.println("Skip " + cohort.getId()); continue; Set<String> cohortFromCatalog = cohort.getSamples() .stream() .map(Sample::getId) System.out.println("Skip " + cohort.getId()); continue; if (cohort.getStatus() == null || !cohort.getStatus().getName().equals(Cohort.CohortStatus.READY)) { logger.debug("Cohort \"{}\" change status from {} to {}", cohort.getId(), cohort.getStats(), Cohort.CohortStatus.READY); catalogManager.getCohortManager().setStatus(studyConfiguration.getStudyName(), cohort.getId(),
@Override public QueryResult<Cohort> insert(long studyId, Cohort cohort, List<VariableSet> variableSetList, QueryOptions options) throws CatalogDBException { long startTime = startQuery(); dbAdaptorFactory.getCatalogStudyDBAdaptor().checkId(studyId); checkCohortNameExists(studyId, cohort.getId()); long newId = dbAdaptorFactory.getCatalogMetaDBAdaptor().getNewAutoIncrementId(); cohort.setUid(newId); cohort.setStudyUid(studyId); if (StringUtils.isEmpty(cohort.getUuid())) { cohort.setUuid(UUIDUtils.generateOpenCGAUUID(UUIDUtils.Entity.COHORT)); } Document cohortObject = cohortConverter.convertToStorageType(cohort, variableSetList); if (StringUtils.isNotEmpty(cohort.getCreationDate())) { cohortObject.put(PRIVATE_CREATION_DATE, TimeUtils.toDate(cohort.getCreationDate())); } else { cohortObject.put(PRIVATE_CREATION_DATE, TimeUtils.getDate()); } cohortObject.put(PERMISSION_RULES_APPLIED, Collections.emptyList()); try { cohortCollection.insert(cohortObject, null); } catch (MongoWriteException e) { throw ifDuplicateKeyException(() -> CatalogDBException.alreadyExists("Cohort", studyId, "name", cohort.getId(), e), e); } Query query = new Query() .append(QueryParams.UID.key(), newId) .append(QueryParams.STUDY_UID.key(), studyId); return endQuery("createCohort", startTime, get(query, options)); }
catalog.getCohortManager().create("s1", new Cohort().setId("c1").setSamples(Collections.emptyList()), null, sessionId); createSample("sample3", "individual3"); createSample("sample4", "individual4"); catalog.getCohortManager().create("s1", new Cohort().setId("c2").setSamples(Collections.emptyList()), null, sessionId); catalog.getCohortManager().create("s1", new Cohort().setId(StudyEntry.DEFAULT_COHORT).setSamples(samples), null, sessionId);
for (Sample sampleInCohort : cohort.getSamples()) { if (sampleInCohort.getUid() != sample.getUid()) { remainingSampleList.add(sampleInCohort); Query cohortQuery = new Query(CohortDBAdaptor.QueryParams.UID.key(), cohort.getUid()); WriteResult delete = catalogManager.getCohortManager().delete(studyStr, cohortQuery, new ObjectMap(), sessionId); if (delete.getWarning() != null || delete.getError() != null) { logger.warn("Cohort {} could not be deleted after extracting the sample {}. WriteResult: {}", cohort.getId(), sample.getId(), delete); warningList.add(new Error(-1, "", "Cohort " + cohort.getId() + "could not be deleted after extracting the " + "sample."));
private List<String> createCohortsIfNeeded(String studyId, Set<String> cohortNames, String sessionId) throws CatalogException { List<String> cohorts = new ArrayList<>(); // Silent query, so it does not fail for missing cohorts Set<String> catalogCohorts = catalogManager.getCohortManager().get(studyId, new ArrayList<>(cohortNames), new Query(), new QueryOptions(QueryOptions.INCLUDE, "name,id"), true, sessionId) .stream() .map(QueryResult::first) .filter(Objects::nonNull) .map(Cohort::getId) .collect(Collectors.toSet()); for (String cohortName : cohortNames) { if (!catalogCohorts.contains(cohortName)) { QueryResult<Cohort> cohort = catalogManager.getCohortManager().create(studyId, cohortName, Study.Type.COLLECTION, "", Collections.emptyList(), null, null, sessionId); logger.info("Creating cohort {}", cohortName); cohorts.add(cohort.first().getId()); } else { logger.debug("cohort {} was already created", cohortName); cohorts.add(cohortName); } } return cohorts; }
@Test public void testCalculateAggregatedExacStats() throws Exception { beforeAggregated("exachead.vcf.gz", Aggregation.EXAC); String tagMap = getResourceUri("exac-tag-mapping.properties").getPath(); List<String> cohortIds = createCohorts(sessionId, studyId, tagMap, catalogManager, logger) .stream().map(Cohort::getId).map(Object::toString).collect(Collectors.toList()); QueryOptions options = new QueryOptions(VariantStorageEngine.Options.AGGREGATION_MAPPING_PROPERTIES.key(), tagMap); calculateStats(options, cohortIds); List<Cohort> cohorts = catalogManager.getCohortManager().get(studyId, (Query) null, null, sessionId).getResult(); Set<String> cohortNames = cohorts .stream() .map(Cohort::getId) .collect(Collectors.toSet()); assertEquals(8, cohortNames.size()); for (Cohort cohort : cohorts) { assertEquals(Cohort.CohortStatus.READY, cohort.getStatus().getName()); } // checkCalculatedAggregatedStats(cohorts, dbName); }
Query query = new Query() .append(CohortDBAdaptor.QueryParams.STUDY_UID.key(), resource.getStudy().getUid()) .append(CohortDBAdaptor.QueryParams.UID.key(), resource.getResource().getUid()); QueryResult<Cohort> cohortQueryResult = cohortDBAdaptor.get(query, new QueryOptions(QueryOptions.INCLUDE, CohortDBAdaptor.QueryParams.SAMPLES.key())); throw new CatalogException("No cohort " + cohortStr + " found in study " + studyStr); if (cohortQueryResult.first().getSamples().size() == 0) { return new QueryResult<>("Samples from cohort " + cohortStr); cohortQueryResult.getDbTime(), cohortQueryResult.first().getSamples().size(), cohortQueryResult.first().getSamples().size(), cohortQueryResult.getWarningMsg(), cohortQueryResult.getErrorMsg(), cohortQueryResult.first().getSamples());
sampleSet.addAll(cohort.getSamples().stream().map(Sample::getId).collect(Collectors.toList()));