public VariantSearchManager getVariantSearchManager() throws StorageEngineException { if (variantSearchManager.get() == null) { synchronized (variantSearchManager) { if (variantSearchManager.get() == null) { // TODO One day we should use reflection here reading from storage-configuration.yml variantSearchManager.set(new VariantSearchManager(getStudyConfigurationManager(), configuration)); } } } return variantSearchManager.get(); }
if (count % insertBatchSize == 0 || !variantDBIterator.hasNext()) { try { delete(collection, variantList); } catch (SolrServerException e) { throw new VariantSearchException("Error deleting variants.", e);
/** * Fetch facet (i.e., counts) resulting of executing the query in the database. * * @param query Query to be executed in the database to filter variants * @param options Query modifiers, accepted values are: facet fields and facet ranges * @return A FacetedQueryResult with the result of the query */ public FacetQueryResult facet(Query query, QueryOptions options) { if (query == null) { query = new Query(); } if (options == null) { options = new QueryOptions(); } FacetQueryResult facetedQueryResult; try { facetedQueryResult = getVariantSearchManager().facetedQuery(dbName, query, options); } catch (IOException | SolrException | StorageEngineException | VariantSearchException e) { throw Throwables.propagate(e); } return facetedQueryResult; }
VariantSearchManager variantSearchManager = new VariantSearchManager(variantStorageEngine.getStudyConfigurationManager(), variantStorageEngine.getConfiguration()); boolean querying = true; variantSearchManager.create(dbName, searchOptions.solrConfig); querying = false; if (querying) { if ("cloud".equals(mode)) { if (!variantSearchManager.existsCollection(dbName)) { throw new IllegalArgumentException("Search " + mode + " '" + dbName + "' does not exists"); if (!variantSearchManager.existsCore(dbName)) { throw new IllegalArgumentException("Search " + mode + " '" + dbName + "' does not exists"); FacetQueryResult facetedQueryResult = variantSearchManager.facetedQuery(dbName, query, queryOptions); if (facetedQueryResult.getResults() != null && CollectionUtils.isNotEmpty(facetedQueryResult.getResults())) { System.out.println("Faceted fields (" + facetedQueryResult.getResults().size() + "):"); queryOptions.put(QueryOptions.SKIP, 0); VariantSolrIterator iterator = variantSearchManager.iterator(dbName, query, queryOptions); System.out.print("["); while (iterator.hasNext()) {
try { if (iterator) { return getVariantSearchManager().iterator(specificSearchIndexSamples, query, options); } else { return getVariantSearchManager().query(specificSearchIndexSamples, query, options); try { if (iterator) { return getVariantSearchManager().iterator(dbName, query, options); } else { return getVariantSearchManager().query(dbName, query, options);
public VariantSearchLoadResult searchIndex(Query inputQuery, QueryOptions inputQueryOptions, boolean overwrite) throws StorageEngineException, IOException, VariantSearchException { Query query = inputQuery == null ? new Query() : new Query(inputQuery); QueryOptions queryOptions = inputQueryOptions == null ? new QueryOptions() : new QueryOptions(inputQueryOptions); VariantDBAdaptor dbAdaptor = getDBAdaptor(); VariantSearchManager variantSearchManager = getVariantSearchManager(); // first, create the collection it it does not exist variantSearchManager.create(dbName); if (!configuration.getSearch().isActive() || !variantSearchManager.isAlive(dbName)) { throw new StorageEngineException("Solr is not alive!"); } // then, load variants queryOptions.put(QueryOptions.EXCLUDE, Arrays.asList(VariantField.STUDIES_SAMPLES_DATA, VariantField.STUDIES_FILES)); try (VariantDBIterator iterator = getVariantsToIndex(overwrite, query, queryOptions, dbAdaptor)) { ProgressLogger progressLogger = new ProgressLogger("Variants loaded in Solr:", () -> dbAdaptor.count(query).first(), 200); VariantSearchLoadResult load = variantSearchManager.load(dbName, iterator, progressLogger, newVariantSearchLoadListener()); long value = System.currentTimeMillis(); getStudyConfigurationManager().lockAndUpdateProject(projectMetadata -> { projectMetadata.getAttributes().put(SEARCH_INDEX_LAST_TIMESTAMP.key(), value); return projectMetadata; }); return load; } catch (StorageEngineException | IOException | RuntimeException e) { throw e; } catch (Exception e) { throw new StorageEngineException("Exception closing VariantDBIterator", e); } }
variantSearchManager.createCore(collection, VariantSearchManager.CONF_SET); variantSearchManager.insert(collection, variants); VariantQueryResult<Variant> results = variantSearchManager.query(collection, new Query(), new QueryOptions(QueryOptions.LIMIT, limit));
public void checkVariantSearchIndex(VariantDBAdaptor dbAdaptor) throws IOException, VariantSearchException, StorageEngineException { QueryOptions queryOptions = new QueryOptions(QueryOptions.LIMIT, 1000); Query query = new Query(); TreeSet<Variant> variantsFromSearch = new TreeSet<>(Comparator.comparing(Variant::toString)); TreeSet<Variant> variantsFromDB = new TreeSet<>(Comparator.comparing(Variant::toString)); variantsFromSearch.addAll(variantStorageEngine.getVariantSearchManager().query(DB_NAME, query, queryOptions).getResult()); variantsFromDB.addAll(dbAdaptor.get(query, queryOptions).getResult()); assertEquals(variantsFromDB.size(), variantsFromSearch.size()); assertEquals(variantsFromDB.size(), variantStorageEngine.getVariantSearchManager().count(DB_NAME, query)); Iterator<Variant> variantsFromSearchIterator = variantsFromSearch.iterator(); Iterator<Variant> variantsFromDBIterator = variantsFromDB.iterator(); for (int i = 0; i < variantsFromDB.size(); i++) { Variant variantFromDB = variantsFromSearchIterator.next(); Set<String> studiesFromDB = variantFromDB.getStudies().stream().map(StudyEntry::getStudyId).collect(Collectors.toSet()); Variant variantFromSearch = variantsFromDBIterator.next(); Set<String> studiesFromSearch = variantFromSearch.getStudies().stream().map(StudyEntry::getStudyId).collect(Collectors.toSet()); assertEquals(variantFromDB.toString(), variantFromSearch.toString()); assertEquals(variantFromDB.toString(), studiesFromDB, studiesFromSearch); } }
long timeStamp = System.currentTimeMillis(); if (configuration.getSearch().isActive() && variantSearchManager.isAlive(dbName)) { deletedVariants = variantSearchManager.delete(dbName, removedVariants, progressLogger); getDBAdaptor().cleanTrash(timeStamp); } catch (StorageEngineException | IOException | RuntimeException e) {
if (doQuerySearchManager(query, new QueryOptions(QueryOptions.COUNT, true))) { approxCount = false; count = getVariantSearchManager().count(dbName, query); } else { sampling = options.getInt(APPROXIMATE_COUNT_SAMPLING_SIZE.key(), .nativeQuery(dbName, searchEngineQuery, queryOptions); List<String> variantIds = nativeResult.getResult().stream().map(VariantSearchModel::getId).collect(Collectors.toList());
protected boolean searchActiveAndAlive() throws StorageEngineException { return configuration.getSearch().isActive() && getVariantSearchManager() != null && getVariantSearchManager().isAlive(dbName); }
public QueryResult<Long> count(Query query) throws StorageEngineException { query = preProcessQuery(query, null); if (!doQuerySearchManager(query, new QueryOptions(QueryOptions.COUNT, true))) { return getDBAdaptor().count(query); } else { try { StopWatch watch = StopWatch.createStarted(); long count = getVariantSearchManager().count(dbName, query); int time = (int) watch.getTime(TimeUnit.MILLISECONDS); return new QueryResult<>("count", time, 1, 1, "", "", Collections.singletonList(count)); } catch (IOException | VariantSearchException e) { throw new VariantQueryException("Error querying Solr", e); } } }
protected VariantQueryResult<Variant> query(Query query, QueryOptions options) { try { query = preProcessQuery(query, options); StudyConfigurationManager scm = dbAdaptor.getStudyConfigurationManager(); String collection = VariantSearchUtils.inferSpecificSearchIndexSamplesCollection(query, options, scm, DB_NAME); // Do not execute this test if the query is not covered by the specific search index collection Assume.assumeThat(query.toJson(), collection, CoreMatchers.notNullValue()); if (options.getInt(QueryOptions.LIMIT, 0) <= 0) { options = new QueryOptions(options); options.put(QueryOptions.LIMIT, 100000); } return variantStorageEngine.getVariantSearchManager().query(collection, query, options); } catch (StorageEngineException | VariantSearchException | IOException e) { e.printStackTrace(); Assert.fail(e.getMessage()); } return null; }
variantSearchManager.create(collectionName); if (configuration.getSearch().isActive() && variantSearchManager.isAlive(collectionName)) { variantSearchManager.load(collectionName, iterator, progressLogger, VariantSearchLoadListener.empty()); } else { throw new StorageEngineException("Solr is not alive!");
private void loadAvro(String collection, Path path) throws StorageEngineException, IOException, SolrServerException { // reader VariantReader reader = VariantReaderUtils.getVariantReader(path); // TODO: get the buffer size from configuration file int bufferSize = 10000; List<Variant> variants; do { variants = reader.read(bufferSize); insert(collection, variants); } while (CollectionUtils.isNotEmpty(variants)); reader.close(); }
@Test @Ignore public void insertVariantIntoSolr() throws Exception { clearDB(DB_NAME); ObjectMap params = new ObjectMap(); StudyConfiguration studyConfiguration = newStudyConfiguration(); params.put(VariantStorageEngine.Options.STUDY.key(), studyConfiguration.getStudyName()); params.put(VariantStorageEngine.Options.TRANSFORM_FORMAT.key(), "json"); params.put(VariantStorageEngine.Options.COMPRESS_METHOD.key(), "gZiP"); params.put(VariantStorageEngine.Options.TRANSFORM_THREADS.key(), 1); params.put(VariantStorageEngine.Options.LOAD_THREADS.key(), 1); params.put(VariantStorageEngine.Options.ANNOTATE.key(), true); runETL(variantStorageEngine, params, true, true, true); VariantDBAdaptor dbAdaptor = getVariantStorageEngine().getDBAdaptor(); VariantSearchManager variantSearchManager = new VariantSearchManager(null, variantStorageEngine.getConfiguration()); // FIXME Collection is not in the configuration any more // variantSearchManager.load(variantStorageEngine.getConfiguration().getSearch().getCollection(), dbAdaptor.iterator()); }
numLoadedVariants += variantList.size(); try { insert(collection, variantList); } catch (SolrServerException e) { throw new VariantSearchException("Error inserting variant.", e); numLoadedVariants += variantList.size(); try { insert(collection, variantList); } catch (SolrServerException e) { throw new VariantSearchException("Error inserting variant.", e);