public HBaseToVariantAnnotationConverter setIncludeFields(Set<VariantField> allIncludeFields) { List<String> list = new ArrayList<>(); if (allIncludeFields != null) { for (VariantField annotationField : VariantField.values()) { if (annotationField.getParent() == VariantField.ANNOTATION && !allIncludeFields.contains(annotationField)) { list.add(annotationField.fieldName().replace(VariantField.ANNOTATION.fieldName() + '.', ""));
/** * Remove intermediate nodes some child is missing, or all children from a node if all are present. * * @param includeFields Set of non pruned fields * @return Pruned set of fields */ public static Set<VariantField> prune(Set<VariantField> includeFields) { if (includeFields.containsAll(VariantField.STUDIES.getChildren())) { includeFields.removeAll(VariantField.STUDIES.getChildren()); } else { includeFields.remove(VariantField.STUDIES); } if (includeFields.containsAll(VariantField.ANNOTATION.getChildren())) { includeFields.removeAll(VariantField.ANNOTATION.getChildren()); } else { includeFields.remove(VariantField.ANNOTATION); } return includeFields; }
public static boolean isIncludeCovered(QueryOptions options) { Set<VariantField> returnedFields = VariantField.getIncludeFields(options); for (VariantField unsupportedVariantField : UNSUPPORTED_VARIANT_FIELDS) { if (returnedFields.contains(unsupportedVariantField)) { return false; } } return true; }
public List<VariantField> getChildren() { if (children.get() == null) { ArrayList<VariantField> childrenList = new ArrayList<>(); for (VariantField variantField : VariantField.values()) { if (variantField.getParent() == this) { childrenList.add(variantField); } } childrenList.trimToSize(); children.compareAndSet(null, Collections.unmodifiableList(childrenList)); } return children.get(); }
private static Map<String, VariantField> getNamesMap() { if (NAMES_MAP.get() == null) { Map<String, VariantField> map = new HashMap<>(); for (VariantField variantField : VariantField.values()) { map.put(variantField.name(), variantField); for (String name : variantField.names) { map.put(name, variantField); } } NAMES_MAP.compareAndSet(null, Collections.unmodifiableMap(map)); } return NAMES_MAP.get(); }
includeFields = new HashSet<>(); for (String include : includeList) { VariantField field = get(include); if (field == null) { throw VariantQueryException.unknownVariantField(QueryOptions.INCLUDE, include); if (field.getParent() != null) { includeFields.add(field.getParent()); includeFields.addAll(field.getChildren()); includeFields = new HashSet<>(Arrays.asList(values())); if (excludeList != null && !excludeList.isEmpty()) { for (String exclude : excludeList) { VariantField field = get(exclude); if (field == null) { throw VariantQueryException.unknownVariantField(QueryOptions.EXCLUDE, exclude); includeFields.removeAll(field.getChildren());
@Override public String toString() { return fieldName(); }
@Test public void checkAnnotationSubFields() throws Exception { Set<String> expectedFields = VariantAnnotation.getClassSchema().getFields().stream().map(Schema.Field::name) .collect(Collectors.toCollection(TreeSet::new)); Set<String> actual = VariantField.ANNOTATION.getChildren().stream().map(f -> f.fieldName().split("\\.")[1]) .collect(Collectors.toCollection(TreeSet::new)); assertEquals(expectedFields, actual); }
private static boolean transformVariantAnnotationField(String key, QueryOptions queryOptions) { StringBuilder sb = new StringBuilder(); final String annotation = VariantField.ANNOTATION.fieldName(); for (String field : queryOptions.getAsStringList(key)) { String newField; if (field.startsWith(annotation + '.') || field.equals(annotation)) { newField = field; } else { newField = annotation + '.' + field; } if (VariantField.get(newField) == null) { throw VariantQueryException.unknownVariantAnnotationField(key, field); } sb.append(newField); sb.append(','); } if (sb.length() > 0) { queryOptions.put(key, sb.toString()); return true; } else { return false; } }
public static SelectVariantElements parseSelectElements( Query query, QueryOptions options, StudyConfigurationManager studyConfigurationManager) { Set<VariantField> includeFields = VariantField.getIncludeFields(options); List<Integer> includeStudies = VariantQueryUtils.getIncludeStudies(query, options, studyConfigurationManager, includeFields); Map<Integer, StudyConfiguration> studyConfigurations = new HashMap<>(); for (Integer studyId : includeStudies) { StudyConfiguration sc = studyConfigurationManager.getStudyConfiguration(studyId, options).first(); if (sc == null) { throw VariantQueryException.studyNotFound(studyId, studyConfigurationManager.getStudyNames(options)); } studyConfigurations.put(studyId, sc); } Function<Integer, StudyConfiguration> provider = studyConfigurations::get; Map<Integer, List<Integer>> sampleIds = VariantQueryUtils.getIncludeSamples(query, options, includeStudies, provider); Map<Integer, List<Integer>> fileIds = VariantQueryUtils.getIncludeFiles(query, includeStudies, includeFields, provider); if (fileIds.values().stream().allMatch(List::isEmpty)) { includeFields.remove(VariantField.STUDIES_FILES); includeFields.removeAll(VariantField.STUDIES_FILES.getChildren()); } if (sampleIds.values().stream().allMatch(List::isEmpty)) { includeFields.remove(VariantField.STUDIES_SAMPLES_DATA); includeFields.removeAll(VariantField.STUDIES_SAMPLES_DATA.getChildren()); } return new SelectVariantElements(includeFields, includeStudies, studyConfigurations, sampleIds, fileIds); }
@Test public void testFieldsMap() { assertEquals(VariantField.values().length, DocumentToVariantConverter.FIELDS_MAP.size()); }
public static VariantField get(String field) { return getNamesMap().get(field); }
protected Iterator<String> variantIdIteratorFromSearch(Query query, int limit, int skip, AtomicLong numTotalResults) throws StorageEngineException { Iterator<String> variantsIterator; QueryOptions queryOptions = new QueryOptions() .append(QueryOptions.LIMIT, limit) .append(QueryOptions.SKIP, skip) .append(QueryOptions.INCLUDE, VariantField.ID.fieldName()); try { // Do not iterate for small queries if (limit < 10000) { VariantQueryResult<VariantSearchModel> nativeResult = getVariantSearchManager().nativeQuery(dbName, query, queryOptions); if (numTotalResults != null) { numTotalResults.set(nativeResult.getNumTotalResults()); } variantsIterator = nativeResult.getResult() .stream() .map(VariantSearchModel::getId) .iterator(); } else { VariantSearchSolrIterator nativeIterator = getVariantSearchManager().nativeIterator(dbName, query, queryOptions); if (numTotalResults != null) { numTotalResults.set(nativeIterator.getNumFound()); } variantsIterator = Iterators.transform(nativeIterator, VariantSearchModel::getId); } } catch (VariantSearchException | IOException e) { throw new VariantQueryException("Error querying " + VariantSearchManager.SEARCH_ENGINE_ID, e); } return variantsIterator; }
public void testQueries(VariantStorageEngine variantStorageEngine) throws StorageEngineException { long count = variantStorageEngine.count(new Query()).first(); long partialCount = 0; int batchSize = (int) Math.ceil(count / 10.0); for (int i = 0; i < 10; i++) { partialCount += variantStorageEngine.getAnnotation("v2", null, new QueryOptions(QueryOptions.LIMIT, batchSize) .append(QueryOptions.SKIP, batchSize * i)).getResult().size(); } assertEquals(count, partialCount); for (int chr = 1; chr < 22; chr += 2) { Query query = new Query(VariantQueryParam.REGION.key(), chr + "," + (chr + 1)); count = variantStorageEngine.count(query).first(); partialCount = variantStorageEngine.getAnnotation("v2", query, new QueryOptions()).getResult().size(); assertEquals(count, partialCount); } String consequenceTypes = VariantField.ANNOTATION_CONSEQUENCE_TYPES.fieldName().replace(VariantField.ANNOTATION.fieldName() + ".", ""); for (VariantAnnotation annotation : variantStorageEngine.getAnnotation("v2", null, new QueryOptions(QueryOptions.INCLUDE, consequenceTypes)).getResult()) { assertEquals(1, annotation.getConsequenceTypes().size()); } for (VariantAnnotation annotation : variantStorageEngine.getAnnotation("v2", null, new QueryOptions(QueryOptions.EXCLUDE, consequenceTypes)).getResult()) { assertTrue(annotation.getConsequenceTypes() == null || annotation.getConsequenceTypes().isEmpty()); } // Get annotations from a deleted snapshot thrown.expectMessage("Variant Annotation snapshot \"v1\" not found!"); assertEquals(0, variantStorageEngine.getAnnotation("v1", null, null).getResult().size()); }
public static List<Integer> getIncludeStudies(Query query, QueryOptions options, StudyConfigurationManager studyConfigurationManager) { return getIncludeStudies(query, options, studyConfigurationManager, VariantField.getIncludeFields(options)); }
.append(INCLUDE_GENOTYPE.key(), true) .append(INCLUDE_FORMAT.key(), ALL) .append(INCLUDE_FILE.key(), NONE), new QueryOptions(QueryOptions.INCLUDE, STUDIES.fieldName())); checkEqualDocuments(expected, projection); expected.append(IDS_FIELD, 1); projection = parser.createProjection(new Query().append(INCLUDE_SAMPLE.key(), ALL) .append(INCLUDE_FILE.key(), NONE), new QueryOptions(QueryOptions.EXCLUDE, ANNOTATION.fieldName())); checkEqualDocuments(expected, projection); .append(INCLUDE_FILE.key(), NONE).append(INCLUDE_GENOTYPE.key(), true), new QueryOptions(QueryOptions.EXCLUDE, ANNOTATION.fieldName())); checkEqualDocuments(expected, projection);
public static List<String> getIncludeSamplesList(Query query, QueryOptions options) { return getIncludeSamplesList(query, VariantField.getIncludeFields(options)); }
public SelectVariantElements(StudyConfiguration studyConfiguration, List<Integer> samples, List<Integer> files) { this.fields = VariantField.getIncludeFields(null); this.studies = Collections.singletonList(studyConfiguration.getStudyId()); this.studyConfigurations = Collections.singletonMap(studyConfiguration.getStudyId(), studyConfiguration); this.samples = Collections.singletonMap(studyConfiguration.getStudyId(), samples); this.files = Collections.singletonMap(studyConfiguration.getStudyId(), files); }
public static Map<String, List<String>> getSamplesMetadata(Query query, QueryOptions options, StudyConfigurationManager studyConfigurationManager) { if (VariantField.getIncludeFields(options).contains(VariantField.STUDIES)) { List<Integer> includeStudies = getIncludeStudies(query, options, studyConfigurationManager); Function<Integer, StudyConfiguration> studyProvider = studyId -> studyConfigurationManager.getStudyConfiguration(studyId, options).first(); return getIncludeSamples(query, options, includeStudies, studyProvider, (sc, s) -> s, StudyConfiguration::getStudyName); } else { return Collections.emptyMap(); } }
public static boolean isOutputMultiStudy(Query query, QueryOptions options, Collection<?> studies) { Set<VariantField> fields = VariantField.getIncludeFields(options); if (!fields.contains(VariantField.STUDIES)) { return false; } else if (isValidParam(query, INCLUDE_STUDY)) { String includeStudy = query.getString(VariantQueryParam.INCLUDE_STUDY.key()); if (NONE.equals(includeStudy)) { return false; } else if (ALL.equals(includeStudy)) { return studies.size() > 1; } else { return query.getAsList(VariantQueryParam.INCLUDE_STUDY.key()).size() > 1; } } else if (isValidParam(query, STUDY)) { String value = query.getString(VariantQueryParam.STUDY.key()); long numStudies = splitValue(value, checkOperator(value)).stream().filter(s -> !isNegated(s)).count(); return numStudies > 1; } else { return studies.size() > 1; } }