public CatalogSolrManager(CatalogManager catalogManager) { this.catalogManager = catalogManager; SearchConfiguration searchConfiguration = catalogManager.getConfiguration().getCatalog().getSearch(); this.solrManager = new SolrManager(searchConfiguration.getHosts(), searchConfiguration.getMode(), searchConfiguration.getTimeout()); insertBatchSize = searchConfiguration.getInsertBatchSize() > 0 ? searchConfiguration.getInsertBatchSize() : DEFAULT_INSERT_BATCH_SIZE; DATABASE_PREFIX = catalogManager.getConfiguration().getDatabasePrefix() + "_"; populateConfigCollectionMap(); logger = LoggerFactory.getLogger(CatalogSolrManager.class); }
public StorageConfiguration(String defaultStorageEngineId, List<StorageEngineConfiguration> storageEngines) { this.defaultStorageEngineId = defaultStorageEngineId; this.storageEngines = storageEngines; this.cellbase = new CellBaseConfiguration(); this.server = new ServerConfiguration(); this.cache = new CacheConfiguration(); this.search = new SearchConfiguration(); this.clinical = new SearchConfiguration(); }
private ClinicalVariantEngine getClinicalStorageEngine() throws ClassNotFoundException, IllegalAccessException, InstantiationException { String clazz = this.storageConfiguration.getClinical().getManager(); ClinicalVariantEngine storageEngine = (ClinicalVariantEngine) Class.forName(clazz).newInstance(); storageEngine.setStorageConfiguration(this.storageConfiguration); return storageEngine; }
public VariantSearchManager configure(VariantStorageEngine variantStorageEngine) throws StorageEngineException { variantStorageEngine.getConfiguration().getSearch().setMode("core"); variantStorageEngine.getConfiguration().getSearch().setActive(true); VariantSearchManager variantSearchManager = variantStorageEngine.getVariantSearchManager(); variantSearchManager.setSolrManager(new SolrManager(solrClient, "localhost", "core", variantStorageEngine.getConfiguration().getSearch().getTimeout())); return variantSearchManager; }
break; case "OPENCGA_CATALOG_SEARCH_HOST": configuration.getCatalog().getSearch().setHost(envVariables.get(variable)); break; case "OPENCGA_CATALOG_SEARCH_TIMEOUT": configuration.getCatalog().getSearch().setTimeout(Integer.parseInt(envVariables.get(variable))); break; case "OPENCGA_CATALOG_SEARCH_BATCH": configuration.getCatalog().getSearch().setInsertBatchSize(Integer.parseInt(envVariables.get(variable))); break; case "OPENCGA_SERVER_REST_PORT":
@Deprecated public String getHost() { return String.join(",", getHosts()); }
protected boolean searchActiveAndAlive() throws StorageEngineException { return configuration.getSearch().isActive() && getVariantSearchManager() != null && getVariantSearchManager().isAlive(dbName); }
public void createSolrCollections() { if (catalogManager.getConfiguration().getCatalog().getSearch().getMode().equals("cloud")) { createCatalogSolrCollections(); } else { createCatalogSolrCores(); } }
@Test public void testFailSearchNotAvailable() throws StorageEngineException { VariantQueryException exception = new VariantQueryException("Unable to use search index. SearchEngine is not available"); thrown.expect(exception.getClass()); thrown.expectMessage(exception.getMessage()); variantStorageEngine.getConfiguration().getSearch().setActive(false); variantStorageEngine.doIntersectWithSearch(new Query(ANNOT_TRAIT.key(), "myTrait"), new QueryOptions(USE_SEARCH_INDEX, VariantStorageEngine.UseSearchIndex.YES)); }
if (configuration.getSearch().isActive() && variantSearchManager.isAlive(collectionName)) {
public VariantSearchManager(StudyConfigurationManager studyConfigurationManager, StorageConfiguration storageConfiguration) { this.storageConfiguration = storageConfiguration; this.solrQueryParser = new SolrQueryParser(studyConfigurationManager); this.cellBaseClient = new CellBaseClient(storageConfiguration.getCellbase().toClientConfiguration()); this.variantSearchToVariantConverter = new VariantSearchToVariantConverter(); this.solrManager = new SolrManager(storageConfiguration.getSearch().getHosts(), storageConfiguration.getSearch().getMode(), storageConfiguration.getSearch().getTimeout()); // Set internal insert batch size from configuration and default value insertBatchSize = storageConfiguration.getSearch().getInsertBatchSize() > 0 ? storageConfiguration.getSearch().getInsertBatchSize() : DEFAULT_INSERT_BATCH_SIZE; logger = LoggerFactory.getLogger(VariantSearchManager.class); }
long timeStamp = System.currentTimeMillis(); if (configuration.getSearch().isActive() && variantSearchManager.isAlive(dbName)) {
public VariantSearchLoadResult searchIndex(Query inputQuery, QueryOptions inputQueryOptions, boolean overwrite) throws StorageEngineException, IOException, VariantSearchException { Query query = inputQuery == null ? new Query() : new Query(inputQuery); QueryOptions queryOptions = inputQueryOptions == null ? new QueryOptions() : new QueryOptions(inputQueryOptions); VariantDBAdaptor dbAdaptor = getDBAdaptor(); VariantSearchManager variantSearchManager = getVariantSearchManager(); // first, create the collection it it does not exist variantSearchManager.create(dbName); if (!configuration.getSearch().isActive() || !variantSearchManager.isAlive(dbName)) { throw new StorageEngineException("Solr is not alive!"); } // then, load variants queryOptions.put(QueryOptions.EXCLUDE, Arrays.asList(VariantField.STUDIES_SAMPLES_DATA, VariantField.STUDIES_FILES)); try (VariantDBIterator iterator = getVariantsToIndex(overwrite, query, queryOptions, dbAdaptor)) { ProgressLogger progressLogger = new ProgressLogger("Variants loaded in Solr:", () -> dbAdaptor.count(query).first(), 200); VariantSearchLoadResult load = variantSearchManager.load(dbName, iterator, progressLogger, newVariantSearchLoadListener()); long value = System.currentTimeMillis(); getStudyConfigurationManager().lockAndUpdateProject(projectMetadata -> { projectMetadata.getAttributes().put(SEARCH_INDEX_LAST_TIMESTAMP.key(), value); return projectMetadata; }); return load; } catch (StorageEngineException | IOException | RuntimeException e) { throw e; } catch (Exception e) { throw new StorageEngineException("Exception closing VariantDBIterator", e); } }