public static URI getResourceUri(String resourceName) throws IOException { return getResourceUri(resourceName, resourceName); }
@Override protected void before() throws Throwable { super.before(); Path rootDir = getTmpRootDir(); String confFolder = VariantSearchManager.CONF_SET; // Copy configuration getResourceUri("configsets/variantsCollection/solrconfig.xml", "configsets/" + confFolder + "/solrconfig.xml"); getResourceUri("solr/variant/managed-schema", "configsets/" + confFolder + "/managed-schema"); getResourceUri("configsets/variantsCollection/params.json", "configsets/" + confFolder + "/params.json"); getResourceUri("configsets/variantsCollection/protwords.txt", "configsets/" + confFolder + "/protwords.txt"); getResourceUri("configsets/variantsCollection/stopwords.txt", "configsets/" + confFolder + "/stopwords.txt"); getResourceUri("configsets/variantsCollection/synonyms.txt", "configsets/" + confFolder + "/synonyms.txt"); getResourceUri("configsets/variantsCollection/lang/stopwords_en.txt", "configsets/" + confFolder + "/lang/stopwords_en.txt"); String solrHome = rootDir.resolve("solr").toString(); if (embeded) { solrClient = create(solrHome, rootDir.resolve("configsets").toString(), coreName); } else { String host = "http://localhost:8983/solr"; String configSet = VariantSearchManager.CONF_SET; int timeout = 5000; SolrManager solrManager = new SolrManager(host, "core", timeout); if (!solrManager.existsCore(coreName)) { solrManager.createCore(coreName, configSet); } this.solrClient = solrManager.getSolrClient(); } }
public static StoragePipelineResult runDefaultETL(VariantStorageEngine variantStorageManager, StudyConfiguration studyConfiguration) throws URISyntaxException, IOException, FileFormatException, StorageEngineException { return runDefaultETL(inputUri, variantStorageManager, studyConfiguration); }
@Before public final void _before() throws Exception { printActiveThreadsNumber(); variantStorageEngine = getVariantStorageEngine(); }
@BeforeClass public static void beforeClass() throws Exception { HadoopVariantStorageEngine variantStorageManager = externalResource.getVariantStorageEngine(); externalResource.clearDB(variantStorageManager.getVariantTableName()); externalResource.clearDB(variantStorageManager.getArchiveTableName(STUDY_ID)); // URI inputUri = VariantStorageBaseTest.getResourceUri("sample1.genome.vcf"); URI inputUri = VariantStorageBaseTest.getResourceUri("platinum/1K.end.platinum-genomes-vcf-NA12877_S1.genome.vcf.gz"); // URI inputUri = VariantStorageManagerTestUtils.getResourceUri("variant-test-file.vcf.gz"); studyConfiguration = VariantStorageBaseTest.newStudyConfiguration(); etlResult = VariantStorageBaseTest.runDefaultETL(inputUri, variantStorageManager, studyConfiguration, new ObjectMap(Options.TRANSFORM_FORMAT.key(), "avro") .append(Options.ANNOTATE.key(), true) .append(Options.CALCULATE_STATS.key(), false) ); fileMetadata = variantStorageManager.readVariantFileMetadata(etlResult.getTransformResult()); VariantSetStats stats = fileMetadata.getStats(); Assert.assertNotNull(stats); try (VariantHadoopDBAdaptor dbAdaptor = variantStorageManager.getDBAdaptor()) { VariantHbaseTestUtils.printVariantsFromVariantsTable(dbAdaptor); VariantHbaseTestUtils.printVariantsFromArchiveTable(dbAdaptor, studyConfiguration); } }
@BeforeClass public static void beforeClass() throws Exception { variantStorageEngine = externalResource.getVariantStorageEngine(); // URI inputUri = VariantStorageBaseTest.getResourceUri("sample1.genome.vcf"); URI inputUri = VariantStorageBaseTest.getResourceUri("platinum/1K.end.platinum-genomes-vcf-NA12877_S1.genome.vcf.gz"); VariantStorageBaseTest.runDefaultETL(inputUri, variantStorageEngine, new StudyConfiguration(0, study1), new ObjectMap(VariantStorageEngine.Options.ANNOTATE.key(), true) .append(VariantStorageEngine.Options.CALCULATE_STATS.key(), false) ); inputUri = VariantStorageBaseTest.getResourceUri("platinum/1K.end.platinum-genomes-vcf-NA12878_S1.genome.vcf.gz"); VariantStorageBaseTest.runDefaultETL(inputUri, variantStorageEngine, new StudyConfiguration(0, study1), new ObjectMap(VariantStorageEngine.Options.ANNOTATE.key(), true) .append(VariantStorageEngine.Options.CALCULATE_STATS.key(), false) ); inputUri = VariantStorageBaseTest.getResourceUri("platinum/1K.end.platinum-genomes-vcf-NA12878_S1.genome.vcf.gz"); VariantStorageBaseTest.runDefaultETL(inputUri, variantStorageEngine, new StudyConfiguration(0, study2), new ObjectMap(VariantStorageEngine.Options.ANNOTATE.key(), true) .append(VariantStorageEngine.Options.CALCULATE_STATS.key(), false) ); VariantHbaseTestUtils.printVariants(variantStorageEngine.getDBAdaptor(), newOutputUri()); }
public static VariantFileMetadata loadFile( HadoopVariantStorageEngine variantStorageManager, String dbName, URI outputUri, String resourceName, StudyConfiguration studyConfiguration, Map<? extends String, ?> otherParams, boolean doTransform, boolean loadArchive, boolean loadVariant) throws Exception { URI fileInputUri = VariantStorageBaseTest.getResourceUri(resourceName); ObjectMap params = new ObjectMap(VariantStorageEngine.Options.TRANSFORM_FORMAT.key(), "proto") .append(VariantStorageEngine.Options.STUDY.key(), studyConfiguration.getStudyName()) .append(VariantStorageEngine.Options.DB_NAME.key(), dbName).append(VariantStorageEngine.Options.ANNOTATE.key(), false) .append(VariantAnnotationManager.SPECIES, "hsapiens").append(VariantAnnotationManager.ASSEMBLY, "GRch37") .append(VariantStorageEngine.Options.CALCULATE_STATS.key(), false) .append(HadoopVariantStorageEngine.HADOOP_LOAD_DIRECT, true) .append(HadoopVariantStorageEngine.HADOOP_LOAD_ARCHIVE, loadArchive) .append(HadoopVariantStorageEngine.HADOOP_LOAD_VARIANT, loadVariant); if (otherParams != null) { params.putAll(otherParams); } // if (fileId > 0) { // params.append(VariantStorageEngine.Options.FILE_ID.key(), fileId); // } StoragePipelineResult etlResult = VariantStorageBaseTest.runETL(variantStorageManager, fileInputUri, outputUri, params, doTransform, doTransform, loadArchive || loadVariant); StudyConfiguration updatedStudyConfiguration = variantStorageManager.getDBAdaptor().getStudyConfigurationManager().getStudyConfiguration(studyConfiguration.getStudyId(), null).first(); if (updatedStudyConfiguration != null) { studyConfiguration.copy(updatedStudyConfiguration); } return variantStorageManager.readVariantFileMetadata(doTransform ? etlResult.getTransformResult() : etlResult.getInput()); }
public static StoragePipelineResult runDefaultETL(URI inputUri, VariantStorageEngine variantStorageManager, StudyConfiguration studyConfiguration, ObjectMap params, boolean doTransform, boolean doLoad) throws URISyntaxException, IOException, FileFormatException, StorageEngineException { ObjectMap newParams = new ObjectMap(params); // newParams.put(VariantStorageEngine.Options.STUDY_CONFIGURATION.key(), studyConfiguration); newParams.putIfAbsent(VariantStorageEngine.Options.AGGREGATED_TYPE.key(), studyConfiguration.getAggregation()); // newParams.putIfAbsent(VariantStorageEngine.Options.STUDY_ID.key(), studyConfiguration.getStudyId()); newParams.putIfAbsent(VariantStorageEngine.Options.STUDY.key(), studyConfiguration.getStudyName()); // newParams.putIfAbsent(VariantStorageEngine.Options.FILE_ID.key(), FILE_ID); // Default value is already avro // newParams.putIfAbsent(VariantStorageEngine.Options.TRANSFORM_FORMAT.key(), "avro"); newParams.putIfAbsent(VariantStorageEngine.Options.ANNOTATE.key(), true); newParams.putIfAbsent(VariantAnnotationManager.SPECIES, "hsapiens"); newParams.putIfAbsent(VariantAnnotationManager.ASSEMBLY, "GRch37"); newParams.putIfAbsent(VariantStorageEngine.Options.CALCULATE_STATS.key(), true); StoragePipelineResult storagePipelineResult = runETL(variantStorageManager, inputUri, outputUri, newParams, true, doTransform, doLoad); try (VariantDBAdaptor dbAdaptor = variantStorageManager.getDBAdaptor()) { StudyConfiguration newStudyConfiguration = dbAdaptor.getStudyConfigurationManager().getStudyConfiguration(studyConfiguration.getStudyName(), null).first(); if (newStudyConfiguration != null) { studyConfiguration.copy(newStudyConfiguration); } } return storagePipelineResult; }
public static URI getResourceUri(String resourceName, String targetName) throws IOException { Path rootDir = getTmpRootDir(); Path resourcePath = rootDir.resolve(targetName); if (!resourcePath.getParent().toFile().exists()) { Files.createDirectories(resourcePath.getParent()); } if (!resourcePath.toFile().exists()) { InputStream stream = VariantStorageEngineTest.class.getClassLoader().getResourceAsStream(resourceName); Files.copy(stream, resourcePath, StandardCopyOption.REPLACE_EXISTING); } return resourcePath.toUri(); }
public static StoragePipelineResult runETL(VariantStorageEngine variantStorageManager, ObjectMap options, boolean doExtract, boolean doTransform, boolean doLoad) throws IOException, FileFormatException, StorageEngineException { return runETL(variantStorageManager, inputUri, outputUri, options, doExtract, doTransform, doLoad); }
@After public final void _after() throws Exception { close(); }
@Before public void before() throws Exception { clearDB(DB_NAME); }
public static StoragePipelineResult runETL(VariantStorageEngine variantStorageManager, URI inputUri, URI outputUri, ObjectMap params, boolean doExtract, boolean doTransform, boolean doLoad) throws IOException, FileFormatException, StorageEngineException { variantStorageManager.getConfiguration() .getStorageEngine(variantStorageManager.getStorageEngineId()).getVariant().getOptions().putAll(params); StoragePipelineResult storagePipelineResult = variantStorageManager.index(Collections.singletonList(inputUri), outputUri, doExtract, doTransform, doLoad).get(0); checkFileExists(storagePipelineResult.getExtractResult()); checkFileExists(storagePipelineResult.getPreTransformResult()); checkFileExists(storagePipelineResult.getTransformResult()); checkFileExists(storagePipelineResult.getPostTransformResult()); checkFileExists(storagePipelineResult.getPreLoadResult()); checkFileExists(storagePipelineResult.getLoadResult()); checkFileExists(storagePipelineResult.getPostLoadResult()); return storagePipelineResult; }
@Before public void before() throws Exception { dbAdaptor = variantStorageEngine.getDBAdaptor(); if (!loaded) { super.before(); load(); loaded = true; } }
public static void printVariantsFromVariantsTable(VariantHadoopDBAdaptor dbAdaptor) throws IOException { printVariantsFromVariantsTable(dbAdaptor, getTmpRootDir()); }
public static URI getPlatinumFile(int fileId) throws IOException { String fileName; if (fileId < 17) { fileName = "1K.end.platinum-genomes-vcf-NA" + (fileId + 12877) + "_S1.genome.vcf.gz"; } else if (fileId >= 12877 && fileId <= 12893){ fileName = "1K.end.platinum-genomes-vcf-NA" + fileId + "_S1.genome.vcf.gz"; } else { throw new IllegalArgumentException("Unknown platinum file " + fileId); } return getResourceUri("platinum/" + fileName); }
public static StoragePipelineResult runDefaultETL(URI inputUri, VariantStorageEngine variantStorageManager, StudyConfiguration studyConfiguration) throws URISyntaxException, IOException, FileFormatException, StorageEngineException { return runDefaultETL(inputUri, variantStorageManager, studyConfiguration, new ObjectMap()); }
protected File create(String resourceName) throws IOException, CatalogException { return create(studyId, getResourceUri(resourceName)); }
@Before public void before() throws Exception { inputFile1 = create(studyId, getResourceUri("platinum/1K.end.platinum-genomes-vcf-NA12877_S1.genome.vcf.gz", "platinum_1/1K.end.platinum-genomes-vcf-NA12877_S1.genome.vcf.gz"), "data/platinum_1/"); inputFile2 = create(studyId, getResourceUri("platinum/1K.end.platinum-genomes-vcf-NA12877_S1.genome.vcf.gz", "platinum_2/1K.end.platinum-genomes-vcf-NA12877_S1.genome.vcf.gz"), "data/platinum_2/"); System.out.println("inputFile1 = " + inputFile1.getUid()); System.out.println("inputFile2 = " + inputFile2.getUid()); }
@Test public void testCalculateNonAggregatedWithAggregationMappingFile() throws Exception { before(); String tagMap = getResourceUri("exac-tag-mapping.properties").getPath(); QueryOptions options = new QueryOptions(VariantStorageEngine.Options.AGGREGATION_MAPPING_PROPERTIES.key(), tagMap); thrown.expectMessage(VariantStatsStorageOperation.nonAggregatedWithMappingFile().getMessage()); calculateStats(options, Arrays.asList("ALL")); }