/** * Provide a new VariantAnnotationManager for creating and loading annotations. * * @param annotator VariantAnnotator to use for creating the new annotations * @return A new instance of VariantAnnotationManager * @throws StorageEngineException if there is an error creating the VariantAnnotationManager */ protected VariantAnnotationManager newVariantAnnotationManager(VariantAnnotator annotator) throws StorageEngineException { return new DefaultVariantAnnotationManager(annotator, getDBAdaptor()); }
public VariantSearchLoadResult searchIndex() throws StorageEngineException, IOException, VariantSearchException { return searchIndex(new Query(), new QueryOptions(), false); }
@Override public void teardownTest(JavaSamplerContext javaSamplerContext) { logger.debug("Closing variant engine"); try { variantStorageEngine.close(); } catch (Exception e) { Throwables.propagate(e); } } }
public VariantQueryResult<Variant> getPhased(String variant, String studyName, String sampleName, QueryOptions options, int windowsSize) throws StorageEngineException { setDefaultTimeout(options); return getDBAdaptor().getPhased(variant, studyName, sampleName, options, windowsSize); }
/** * Index the given input files. By default, executes the steps in {@link VariantStoragePipeline}. * * Will create a {@link #newStoragePipeline} for each input file. * * @param inputFiles Input files to index * @param outdirUri Output directory for possible intermediate files * @param doExtract Execute extract step {@link VariantStoragePipeline#extract} * @param doTransform Execute transform step {@link VariantStoragePipeline#transform} * @param doLoad Execute load step {@link VariantStoragePipeline#load} * @return List of {@link StoragePipelineResult}, one for each input file. * @throws StorageEngineException If there is any problem related with the StorageEngine */ @Override public List<StoragePipelineResult> index(List<URI> inputFiles, URI outdirUri, boolean doExtract, boolean doTransform, boolean doLoad) throws StorageEngineException { List<StoragePipelineResult> results = super.index(inputFiles, outdirUri, doExtract, doTransform, doLoad); if (doLoad) { annotateLoadedFiles(outdirUri, inputFiles, results, getOptions()); calculateStatsForLoadedFiles(outdirUri, inputFiles, results, getOptions()); searchIndexLoadedFiles(inputFiles, getOptions()); } return results; }
/** * Exports the result of the given query and the associated metadata. * @param outputFile Optional output file. If null or empty, will print into the Standard output. Won't export any metadata. * @param outputFormat Variant output format * @param query Query with the variants to export * @param queryOptions Query options * @throws IOException If there is any IO error * @throws StorageEngineException If there is any error exporting variants */ public void exportData(URI outputFile, VariantOutputFormat outputFormat, Query query, QueryOptions queryOptions) throws IOException, StorageEngineException { exportData(outputFile, outputFormat, new VariantMetadataFactory(getStudyConfigurationManager() ), query, queryOptions); }
@Test public void loadFromSTDIN() throws Exception { clearDB(DB_NAME); StudyConfiguration studyConfiguration = newStudyConfiguration(); StoragePipelineResult etlResult = runDefaultETL(smallInputUri, variantStorageEngine, studyConfiguration, new ObjectMap(VariantStorageEngine.Options.TRANSFORM_FORMAT.key(), "avro"), true, false); VariantFileMetadata fileMetadata = checkTransformedVariants(etlResult.getTransformResult(), studyConfiguration); Path tempFile = Paths.get(outputUri).resolve("temp_file"); Files.move(Paths.get(etlResult.getTransformResult()), tempFile); assertFalse(Files.exists(Paths.get(etlResult.getTransformResult()))); InputStream in = System.in; try (InputStream is = new FileInputStream(tempFile.toFile())) { System.setIn(is); variantStorageEngine.getConfiguration() .getStorageEngine(variantStorageEngine.getStorageEngineId()).getVariant().getOptions() .put(VariantStorageEngine.Options.STDIN.key(), true); variantStorageEngine.index(Collections.singletonList(etlResult.getTransformResult()), outputUri, false, false, true); } finally { System.setIn(in); } studyConfiguration = variantStorageEngine.getStudyConfigurationManager().getStudyConfiguration(STUDY_NAME, null).first(); assertEquals(1, studyConfiguration.getIndexedFiles().size()); checkLoadedVariants(variantStorageEngine.getDBAdaptor(), studyConfiguration, true, false, true, getExpectedNumLoadedVariants(fileMetadata)); }
@Test public void testSimpleImport() throws Exception { URI outputFile = newOutputUri().resolve("export.avro"); System.out.println("outputFile = " + outputFile); variantStorageEngine.exportData(outputFile, VariantOutputFormat.AVRO, new Query(), new QueryOptions()); clearDB(DB_NAME); variantStorageEngine.importData(outputFile, new ObjectMap()); for (Variant variant : variantStorageEngine.getDBAdaptor()) { assertEquals(4, variant.getStudies().get(0).getSamplesData().size()); } }
@Before public void before() throws Exception { clearDB(DB_NAME); dbAdaptor = getVariantStorageEngine().getDBAdaptor(); VariantStorageEngine storageEngine = getVariantStorageEngine(); List<URI> inputFiles = new ArrayList<>(); // 12877, 12889 and 12890 have an overlapping at M:16184:C:A , M:16184:C:- , M:16184:CC:- // for (int i : new int[]{0, 12, 13}) { for (int fileId = 12877; fileId < 12877 + 17; fileId++) { String fileName = "1K.end.platinum-genomes-vcf-NA" + fileId + "_S1.genome.vcf.gz"; inputFiles.add(getResourceUri("platinum/" + fileName)); } storageEngine.getOptions().put(VariantStorageEngine.Options.STUDY.key(), STUDY_NAME); storageEngine.getOptions().put(VariantStorageEngine.Options.CALCULATE_STATS.key(), false); // storageEngine.index(inputFiles.subList(0, inputFiles.size()/2), outputUri, true, true, true); // storageEngine.index(inputFiles.subList(inputFiles.size()/2, inputFiles.size()), outputUri, true, true, true); storageEngine.index(inputFiles, outputUri, true, true, true); studyConfiguration = storageEngine.getStudyConfigurationManager().getStudyConfiguration(STUDY_ID, null).first(); }
List<String> studyNames = variantStorageEngine.getStudyConfigurationManager().getStudyNames(new QueryOptions()); QueryResult<Long> result = variantStorageEngine.count(query); System.out.println("Num. results\t" + result.getResult().get(0)); } else if (StringUtils.isNotEmpty(variantQueryCommandOptions.rank)) { } else if (StringUtils.isNotEmpty(variantQueryCommandOptions.groupBy)) { ObjectMapper objectMapper = new ObjectMapper(); QueryResult groupBy = variantStorageEngine.groupBy(query, variantQueryCommandOptions.groupBy, options); System.out.println("groupBy = " + objectMapper.writerWithDefaultPrettyPrinter().writeValueAsString(groupBy)); } else { variantStorageEngine.exportData(uri, of, query, options);
updateProjectMetadata(variantStorageEngine.getStudyConfigurationManager(), studyInfo.getOrganism(), release); variantStorageEngine.getOptions().putAll(options); boolean calculateStats = options.getBoolean(VariantStorageEngine.Options.CALCULATE_STATS.key()) && (step.equals(Type.LOAD) || step.equals(Type.INDEX)); storagePipelineResults = variantStorageEngine.index(fileUris, outdir.toUri(), false, transform, load); } catch (StoragePipelineException e) { logger.error("Error executing " + step, e); variantStorageEngine.close();
@Test public void testIncrementalIndex() throws Exception { VariantDBAdaptor dbAdaptor = variantStorageEngine.getDBAdaptor(); dbAdaptor.getStudyConfigurationManager().updateStudyConfiguration(studyConfiguration, null); options.put(VariantStorageEngine.Options.STUDY.key(), studyId); storageEngine.getOptions().putAll(options); storageEngine.getOptions().put(VariantStorageEngine.Options.RELEASE.key(), release++); storageEngine.index(inputFiles.subList(0, 2), outputUri, true, true, true); storageEngine.getOptions().putAll(options); storageEngine.getOptions().put(VariantStorageEngine.Options.RELEASE.key(), release++); storageEngine.index(inputFiles.subList(2, 4), outputUri, true, true, true); .append(DefaultVariantStatisticsManager.OUTPUT, outputUri) .append(DefaultVariantStatisticsManager.OUTPUT_FILE_NAME, "stats"); storageEngine.calculateStats(studyConfiguration.getStudyName(), Collections.singletonList("ALL"), statsOptions);
public void testRemoveFiles(boolean searchIndexBeforeRemove) throws Exception { VariantDBAdaptor dbAdaptor = variantStorageEngine.getDBAdaptor(); storageEngine.getOptions().putAll(options); storageEngine.index(inputFiles, outputUri, true, true, true); storageEngine.removeFiles(studyConfiguration.getStudyName(), Collections.singletonList(fileNames.get(0))); expected = 0; storageEngine.removeFiles(studyConfiguration.getStudyName(), Collections.singletonList(fileNames.get(0))); expected = dbAdaptor.count(query).first();
@Test public void transformToJsonSTDOUT() throws Exception { URI outputUri = newOutputUri(); VariantStorageEngine variantStorageManager = getVariantStorageEngine(); File outputFile = Paths.get(outputUri.resolve("outputFile.json")).toFile(); try (PrintStream os = new PrintStream(new FileOutputStream(outputFile))) { System.setOut(os); ObjectMap options = variantStorageManager.getConfiguration() .getStorageEngine(variantStorageManager.getStorageEngineId()).getVariant().getOptions(); options.append(VariantStorageEngine.Options.STDOUT.key(), true); options.append(VariantStorageEngine.Options.TRANSFORM_FORMAT.key(), "json"); variantStorageManager.index(Collections.singletonList(smallInputUri), outputUri, true, true, false).get(0); } finally { System.setOut(out); } assertEquals(999, countLines(outputFile)); }
@Test public void testMultiAnnotations() throws Exception { VariantStorageEngine variantStorageEngine = getVariantStorageEngine(); runDefaultETL(smallInputUri, variantStorageEngine, newStudyConfiguration(), new ObjectMap(VariantStorageEngine.Options.ANNOTATE.key(), false)); variantStorageEngine.getOptions() .append(VARIANT_ANNOTATOR_CLASSNAME, TestAnnotator.class.getName()) .append(ANNOTATOR, VariantAnnotatorFactory.AnnotationSource.OTHER); variantStorageEngine.saveAnnotation("v0", new ObjectMap()); variantStorageEngine.annotate(new Query(), new ObjectMap(TestAnnotator.ANNOT_KEY, "v1").append(OVERWRITE_ANNOTATIONS, true)); variantStorageEngine.saveAnnotation("v1", new ObjectMap()); variantStorageEngine.annotate(new Query(), new ObjectMap(TestAnnotator.ANNOT_KEY, "v2").append(OVERWRITE_ANNOTATIONS, true)); variantStorageEngine.saveAnnotation("v2", new ObjectMap()); variantStorageEngine.annotate(new Query(VariantQueryParam.REGION.key(), "1"), new ObjectMap(TestAnnotator.ANNOT_KEY, "v3").append(OVERWRITE_ANNOTATIONS, true)); assertEquals(0, variantStorageEngine.getAnnotation("v0", null, null).getResult().size()); checkAnnotationSnapshot(variantStorageEngine, "v1", "v1"); checkAnnotationSnapshot(variantStorageEngine, "v2", "v2"); checkAnnotationSnapshot(variantStorageEngine, VariantAnnotationManager.CURRENT, VariantAnnotationManager.CURRENT, "v3", new Query(VariantQueryParam.REGION.key(), "1")); checkAnnotationSnapshot(variantStorageEngine, VariantAnnotationManager.CURRENT, "v2", "v2", new Query(VariantQueryParam.REGION.key(), "2")); variantStorageEngine.deleteAnnotation("v1", new ObjectMap()); testQueries(variantStorageEngine); }
public void importData(StudyInfo studyInfo, URI inputUri, String sessionId) throws IOException, StorageEngineException { VariantMetadataImporter variantMetadataImporter; variantMetadataImporter = new CatalogVariantMetadataImporter(studyInfo.getStudyFQN(), inputUri, sessionId); try { DataStore dataStore = studyInfo.getDataStores().get(File.Bioformat.VARIANT); VariantStorageEngine variantStorageEngine = getVariantStorageEngine(dataStore); ObjectMap options = variantStorageEngine.getOptions(); VariantMetadata metadata; StudyConfiguration studyConfiguration; try (StudyConfigurationManager scm = variantStorageEngine.getStudyConfigurationManager()) { metadata = variantMetadataImporter.importMetaData(inputUri, scm); studyConfiguration = scm.getStudyConfiguration(((int) studyInfo.getStudyUid()), null).first(); } variantStorageEngine.importData(inputUri, metadata, Collections.singletonList(studyConfiguration), options); } catch (Exception e) { logger.error("Error importing data"); throw e; } }
String dbName = (searchOptions.dbName == null ? "variants" : searchOptions.dbName); variantStorageEngine.getConfiguration().getSearch().setHost(solrUrl); VariantSearchManager variantSearchManager = new VariantSearchManager(variantStorageEngine.getStudyConfigurationManager(), variantStorageEngine.getConfiguration()); boolean querying = true; QueryOptions options = new QueryOptions(); querying = false; Query query = VariantQueryCommandUtils.parseQuery(searchOptions, new Query()); variantStorageEngine.searchIndex(query, options, searchOptions.overwrite); String mode = variantStorageEngine.getConfiguration().getSearch().getMode(); if (querying) { if ("cloud".equals(mode)) {
@GET @Path("/query") @Produces("application/json") public Response fetch(@QueryParam("storageEngine") String storageEngine, @QueryParam("dbName") String dbName, @QueryParam("region") String regionsCVS, @QueryParam("histogram") @DefaultValue("false") boolean histogram, @QueryParam("histogram_interval") @DefaultValue("2000") int interval ) { try { Query query = VariantStorageManager.getVariantQuery(params); VariantQueryResult<Variant> queryResult = StorageEngineFactory.get().getVariantStorageEngine(storageEngine, dbName).get(query, queryOptions); return createOkResponse(queryResult); } catch (Exception e) { e.printStackTrace(); return createErrorResponse(e.toString()); } }
@Test public void testChangeAnnotator() throws Exception { VariantStorageEngine variantStorageEngine = getVariantStorageEngine(); runDefaultETL(smallInputUri, variantStorageEngine, newStudyConfiguration(), new ObjectMap(VariantStorageEngine.Options.ANNOTATE.key(), false)); variantStorageEngine.getOptions() .append(VARIANT_ANNOTATOR_CLASSNAME, TestAnnotator.class.getName()) .append(ANNOTATOR, VariantAnnotatorFactory.AnnotationSource.OTHER); // First annotation. Should run ok. variantStorageEngine.annotate(new Query(), new ObjectMap(TestAnnotator.ANNOT_KEY, "v1")); assertEquals("v1", variantStorageEngine.getStudyConfigurationManager().getProjectMetadata().first().getAnnotation().getCurrent().getAnnotator().getVersion()); // Second annotation. New annotator. Overwrite. Should run ok. variantStorageEngine.annotate(new Query(), new ObjectMap(TestAnnotator.ANNOT_KEY, "v2").append(OVERWRITE_ANNOTATIONS, true)); assertEquals("v2", variantStorageEngine.getStudyConfigurationManager().getProjectMetadata().first().getAnnotation().getCurrent().getAnnotator().getVersion()); // Third annotation. New annotator. Do not overwrite. Should fail. thrown.expect(VariantAnnotatorException.class); variantStorageEngine.annotate(new Query(), new ObjectMap(TestAnnotator.ANNOT_KEY, "v3").append(OVERWRITE_ANNOTATIONS, false)); }
fileMetadata = variantStorageEngine.getVariantReaderUtils().readVariantFileMetadata(Paths.get(etlResult.getTransformResult().getPath()).toUri()); Integer indexedFileId = studyConfiguration.getIndexedFiles().iterator().next(); variantStorageEngine.calculateStats(studyConfiguration.getStudyName(), new ArrayList<>(cohortIds.keySet()), options); variantStorageEngine.searchIndex();