/** * Provide a new VariantAnnotationManager for creating and loading annotations. * * @param annotator VariantAnnotator to use for creating the new annotations * @return A new instance of VariantAnnotationManager * @throws StorageEngineException if there is an error creating the VariantAnnotationManager */ protected VariantAnnotationManager newVariantAnnotationManager(VariantAnnotator annotator) throws StorageEngineException { return new DefaultVariantAnnotationManager(annotator, getDBAdaptor()); }
public VariantExporter(VariantStorageEngine engine, VariantMetadataFactory metadataFactory) throws StorageEngineException { this.engine = engine; variantWriterFactory = new VariantWriterFactory(engine.getDBAdaptor()); this.metadataFactory = metadataFactory; }
/** * Provide a new VariantStatisticsManager for creating and loading statistics. * * @return A new instance of VariantStatisticsManager * @throws StorageEngineException if there is an error creating the VariantStatisticsManager */ public VariantStatisticsManager newVariantStatisticsManager() throws StorageEngineException { return new DefaultVariantStatisticsManager(getDBAdaptor()); }
public VariantQueryResult<Variant> getPhased(String variant, String studyName, String sampleName, QueryOptions options, int windowsSize) throws StorageEngineException { setDefaultTimeout(options); return getDBAdaptor().getPhased(variant, studyName, sampleName, options, windowsSize); }
private void init(String storageEngine) throws IllegalAccessException, InstantiationException, ClassNotFoundException, StorageEngineException { StorageEngineFactory storageEngineFactory = StorageEngineFactory.get(storageConfiguration); VariantStorageEngine variantStorageEngine = storageEngineFactory.getVariantStorageEngine(storageEngine, storageConfiguration.getBenchmark().getDatabaseName()); variantDBAdaptor = variantStorageEngine.getDBAdaptor(); }
public QueryResult<VariantMetadata> getMetadata(Query query, QueryOptions queryOptions, String sessionId) throws CatalogException, IOException, StorageEngineException { return secure(query, queryOptions, sessionId, engine -> { StopWatch watch = StopWatch.createStarted(); VariantMetadataFactory metadataFactory = new CatalogVariantMetadataFactory(catalogManager, engine.getDBAdaptor(), sessionId); VariantMetadata metadata = metadataFactory.makeVariantMetadata(query, queryOptions); return new QueryResult<>("getMetadata", ((int) watch.getTime()), 1, 1, "", "", Collections.singletonList(metadata)); }); }
@Override public void before() throws Exception { boolean wasLoaded = loaded; super.before(); if (loaded && !wasLoaded) { VariantHbaseTestUtils.printVariants(((VariantHadoopDBAdaptor) variantStorageEngine.getDBAdaptor()), newOutputUri(getClass().getSimpleName())); // for (String study : variantStorageEngine.getDBAdaptor().getStudyConfigurationManager().getStudies(null).keySet()) { // variantStorageEngine.fillMissing(study, new ObjectMap(), false); // } // VariantHbaseTestUtils.printVariants(((VariantHadoopDBAdaptor) variantStorageEngine.getDBAdaptor()), newOutputUri(getClass().getSimpleName())); } }
@Test public void checkCount() throws Exception { int expected = 24 + 7; int count = variantStorageEngine.getDBAdaptor().count(null).first().intValue(); assertEquals(expected, count); }
public static void checkCalculatedAggregatedStats(Set<String> cohortNames, String dbName) throws Exception { VariantDBAdaptor dbAdaptor = StorageEngineFactory.get().getVariantStorageEngine(null, dbName).getDBAdaptor(); for (Variant variant : dbAdaptor) { for (StudyEntry sourceEntry : variant.getStudies()) { assertEquals(cohortNames, sourceEntry.getStats().keySet()); for (Map.Entry<String, VariantStats> entry : sourceEntry.getStats().entrySet()) { assertTrue(cohortNames.contains(entry.getKey())); } } } } /**
@Before public void before() throws Exception { dbAdaptor = variantStorageEngine.getDBAdaptor(); if (!loaded) { super.before(); load(); loaded = true; } }
@Test public void checkAllAnnotated() throws Exception { for (Variant variant : variantStorageEngine.getDBAdaptor()) { assertNotNull(variant.toString(), variant.getAnnotation()); } }
@Test public void testSimpleImport() throws Exception { URI outputFile = newOutputUri().resolve("export.avro"); System.out.println("outputFile = " + outputFile); variantStorageEngine.exportData(outputFile, VariantOutputFormat.AVRO, new Query(), new QueryOptions()); clearDB(DB_NAME); variantStorageEngine.importData(outputFile, new ObjectMap()); for (Variant variant : variantStorageEngine.getDBAdaptor()) { assertEquals(4, variant.getStudies().get(0).getSamplesData().size()); } }
@Test public void testImportEmptySamples() throws Exception { URI outputFile = newOutputUri().resolve("export.avro"); System.out.println("outputFile = " + outputFile); Query query = new Query(VariantQueryParam.INCLUDE_SAMPLE.key(), "."); QueryOptions queryOptions = new QueryOptions(); variantStorageEngine.exportData(outputFile, VariantOutputFormat.AVRO, query, queryOptions); clearDB(DB_NAME); variantStorageEngine.importData(outputFile, new ObjectMap()); for (Variant variant : variantStorageEngine.getDBAdaptor()) { assertEquals(0, variant.getStudies().get(0).getSamplesData().size()); } }
@Test public void avroBasicIndex() throws Exception { clearDB(DB_NAME); StudyConfiguration studyConfiguration = newStudyConfiguration(); StoragePipelineResult etlResult = runDefaultETL(smallInputUri, variantStorageEngine, studyConfiguration, new ObjectMap(VariantStorageEngine.Options.TRANSFORM_FORMAT.key(), "avro")); assertTrue("Incorrect transform file extension " + etlResult.getTransformResult() + ". Expected 'variants.avro.gz'", Paths.get(etlResult.getTransformResult()).toFile().getName().endsWith("variants.avro.gz")); assertEquals(1, studyConfiguration.getIndexedFiles().size()); VariantFileMetadata fileMetadata = checkTransformedVariants(etlResult.getTransformResult(), studyConfiguration); checkLoadedVariants(variantStorageEngine.getDBAdaptor(), studyConfiguration, true, false, true, getExpectedNumLoadedVariants (fileMetadata)); }
@Test public void basicIndex() throws Exception { clearDB(DB_NAME); StudyConfiguration studyConfiguration = newStudyConfiguration(); StoragePipelineResult etlResult = runDefaultETL(smallInputUri, variantStorageEngine, studyConfiguration, new ObjectMap(VariantStorageEngine.Options.TRANSFORM_FORMAT.key(), "json")); assertTrue("Incorrect transform file extension " + etlResult.getTransformResult() + ". Expected 'variants.json.gz'", Paths.get(etlResult.getTransformResult()).toFile().getName().endsWith("variants.json.gz")); VariantFileMetadata fileMetadata = variantStorageEngine.getVariantReaderUtils().readVariantFileMetadata(etlResult.getTransformResult()); assertEquals(1, studyConfiguration.getIndexedFiles().size()); checkTransformedVariants(etlResult.getTransformResult(), studyConfiguration); checkLoadedVariants(variantStorageEngine.getDBAdaptor(), studyConfiguration, true, false, true, getExpectedNumLoadedVariants(fileMetadata)); }
@Override @Before public void before() throws Exception { studyConfiguration = newStudyConfiguration(); clearDB(DB_NAME); runDefaultETL(inputUri, getVariantStorageEngine(), studyConfiguration, new ObjectMap(VariantStorageEngine.Options.ANNOTATE.key(), false)); dbAdaptor = getVariantStorageEngine().getDBAdaptor(); }
@Before public void setUp() throws Exception { clearDB(DB_NAME); VariantStorageEngine variantStorageManager = getVariantStorageEngine(); ObjectMap options = new ObjectMap(VariantStorageEngine.Options.ANNOTATE.key(), false) .append(VariantStorageEngine.Options.CALCULATE_STATS.key(), false) .append(VariantStorageEngine.Options.EXTRA_GENOTYPE_FIELDS.key(), "DP,PS"); runDefaultETL(getResourceUri("variant-test-phased.vcf"), variantStorageManager, newStudyConfiguration(), options); VariantDBAdaptor dbAdaptor = variantStorageManager.getDBAdaptor(); for (Variant variant : dbAdaptor) { System.out.println("variant = " + variant.toJson()); } }
@Override @Before public void before() throws Exception { studyConfiguration = newStudyConfiguration(); studyConfiguration.setAggregation(getAggregationType()); clearDB(DB_NAME); inputUri = getInputUri(); runDefaultETL(inputUri, getVariantStorageEngine(), studyConfiguration, new ObjectMap(VariantStorageEngine.Options.ANNOTATE.key(), false) .append(VariantStorageEngine.Options.CALCULATE_STATS.key(), false)); dbAdaptor = getVariantStorageEngine().getDBAdaptor(); }
@Test public void stageAlreadyStagedFileTest() throws Exception { StudyConfiguration studyConfiguration = createStudyConfiguration(); StoragePipelineResult storagePipelineResult = runDefaultETL(smallInputUri, variantStorageEngine, studyConfiguration, new ObjectMap() .append(MongoDBVariantOptions.STAGE.key(), true) .append(MongoDBVariantOptions.MERGE.key(), false)); long count = variantStorageEngine.getDBAdaptor().count(null).first(); assertEquals(0L, count); runETL(variantStorageEngine, storagePipelineResult.getTransformResult(), outputUri, new ObjectMap() .append(VariantStorageEngine.Options.ANNOTATE.key(), false) .append(MongoDBVariantOptions.STAGE.key(), true) .append(MongoDBVariantOptions.MERGE.key(), false), false, false, true); }
@Test public void mergeAlreadyStagedFileTest() throws Exception { StudyConfiguration studyConfiguration = createStudyConfiguration(); StoragePipelineResult storagePipelineResult = runDefaultETL(smallInputUri, variantStorageEngine, studyConfiguration, new ObjectMap() .append(MongoDBVariantOptions.STAGE.key(), true) .append(MongoDBVariantOptions.MERGE.key(), false)); runETL(variantStorageEngine, storagePipelineResult.getTransformResult(), outputUri, new ObjectMap() .append(VariantStorageEngine.Options.ANNOTATE.key(), false) .append(MongoDBVariantOptions.STAGE.key(), false) .append(MongoDBVariantOptions.MERGE.key(), true), false, false, true); Long count = variantStorageEngine.getDBAdaptor().count(null).first(); assertTrue(count > 0); }