private void checkAlignmentFormat(File file) throws CatalogException { if (!file.getFormat().equals(File.Format.BAM)) { throw new CatalogException("File " + file.getName() + " not supported. Expecting a BAM file."); } }
private void checkAlignmentBioformat(File file) throws CatalogException { if (!file.getBioformat().equals(File.Bioformat.ALIGNMENT)) { throw new CatalogException("File " + file.getName() + " not supported. Expecting an alignment file."); } }
public static VariantFileMetadata readVariantFileMetadata(File file, URI fileUri) throws IOException { File.Format format = file.getFormat(); File.Format detectFormat = FileUtils.detectFormat(fileUri); if (format == File.Format.VCF || format == File.Format.GVCF || format == File.Format.BCF || detectFormat == File.Format.VCF || detectFormat == File.Format.GVCF || detectFormat == File.Format.BCF) { VariantFileMetadata metadata = new VariantFileMetadata(String.valueOf(file.getUid()), file.getName()); metadata.setId(String.valueOf(file.getUid())); return VariantMetadataUtils.readVariantFileMetadata(Paths.get(fileUri.getPath()), metadata); } else { return null; } }
@Override protected List<String> validate(String defaultStudyStr, List<String> values, Integer release, VariantQueryParam param, String sessionId) throws CatalogException { if (release == null) { AbstractManager.MyResources<File> uids = catalogManager.getFileManager().getUids(values, defaultStudyStr, sessionId); return uids.getResourceList().stream().map(File::getName).collect(Collectors.toList()); } else { return validate(defaultStudyStr, values, release, param, catalogManager.getFileManager(), File::getName, file -> ((int) file.getIndex().getRelease()), file -> { if (file.getIndex() == null || file.getIndex().getStatus() == null || file.getIndex().getStatus().getName() == null || !file.getIndex().getStatus().getName().equals(Status.READY)) { throw new VariantQueryException("File '" + file.getName() + "' is not indexed"); } }, sessionId); } } }
@Test public void parseQuery() throws Exception { assertEquals("user@p1:s1", parseValue(STUDY, "s1")); assertEquals("user@p1:s1,user@p1:s2", parseValue(STUDY, "s1,s2")); assertEquals("!user@p1:s1,user@p1:s2", parseValue(STUDY, "!s1,s2")); assertEquals("user@p1:s2;!user@p1:s1;user@p1:s3", parseValue(STUDY, "user@p1:s2;!s1;p1:s3")); assertEquals(file1.getName(), parseValue("s1", FILE, file1.getName())); assertEquals(file1.getName(), parseValue("s1", FILE, file1.getId())); assertEquals(file1.getName(), parseValue("s1", FILE, file1.getPath())); assertEquals(file1.getName() + "," + file2.getName(), parseValue("s1", FILE, file1.getPath() + "," + file2.getPath())); assertEquals(file1.getName() + ";" + file2.getName(), parseValue("s1", FILE, file1.getPath() + ";" + file2.getPath())); // assertEquals("file1.vcf", parseValue("s1", FILE, String.valueOf(file1.getUid()))); assertEquals("sample1:HOM_ALT;sample2:HET_REF", parseValue("s1", GENOTYPE, "sample1:HOM_ALT;sample2:HET_REF")); assertEquals("sample1:HOM_ALT,sample2:HET_REF", parseValue("s1", GENOTYPE, "sample1:HOM_ALT,sample2:HET_REF")); }
@Test public void includeFields() throws CatalogDBException { QueryResult<File> fileQueryResult = catalogFileDBAdaptor.get(7, new QueryOptions(QueryOptions.INCLUDE, FileDBAdaptor.QueryParams.PATH.key())); List<File> files = fileQueryResult.getResult(); assertEquals("Include path does not work.", "data/file.vcf", files.get(0).getPath()); assertEquals("Include not working.", null, files.get(0).getName()); }
@Test public void testIndex() throws Exception { URI uri = getClass().getResource("/biofiles/variant-test-file.vcf.gz").toURI(); File file = fileManager.link(studyFqn, uri, "", null, sessionIdUser).first(); assertEquals(4, file.getSamples().size()); assertEquals(File.Format.VCF, file.getFormat()); assertEquals(File.Bioformat.VARIANT, file.getBioformat()); Job job = fileManager.index(studyFqn, Collections.singletonList(file.getName()), "VCF", null, sessionIdUser).first(); assertEquals(file.getUid(), job.getInput().get(0).getUid()); }
@Test public void getAllFilesTest() throws CatalogDBException { long studyId = user3.getProjects().get(0).getStudies().get(0).getUid(); QueryResult<File> allFiles = catalogFileDBAdaptor.getAllInStudy(studyId, null); List<File> files = allFiles.getResult(); List<File> expectedFiles = user3.getProjects().get(0).getStudies().get(0).getFiles(); assertEquals(expectedFiles.size(), files.size()); for (File expectedFile : expectedFiles) { boolean found = false; for (File fileResult : allFiles.getResult()) { if (fileResult.getUid() == expectedFile.getUid()) found = true; } if (!found) { throw new CatalogDBException("The file " + expectedFile.getName() + " could not be found."); } } }
@Override public Document convertToStorageType(File file, List<VariableSet> variableSetList) { Document document = super.convertToStorageType(file, variableSetList); document.remove(SampleDBAdaptor.QueryParams.ANNOTATION_SETS.key()); document.put("uid", file.getUid()); document.put("studyUid", file.getStudyUid()); long jobId = file.getJob() != null ? (file.getJob().getUid() == 0 ? -1L : file.getJob().getUid()) : -1L; document.put("job", new Document("uid", jobId)); document.put("samples", convertSamples(file.getSamples())); document.put(FileMongoDBAdaptor.REVERSE_NAME, StringUtils.reverse(file.getName())); return document; }
@Test public void testIndexWithTransformError() throws Exception { QueryOptions queryOptions = new QueryOptions(VariantStorageEngine.Options.ANNOTATE.key(), false) .append(VariantStorageEngine.Options.CALCULATE_STATS.key(), false); DummyVariantStoragePipeline storageETL = mockVariantStorageETL(); List<File> files = Arrays.asList(getFile(0), getFile(1)); StorageEngineException transformException = StorageEngineException.unableToExecute("transform", 0, ""); Mockito.doThrow(transformException).when(storageETL) .transform(ArgumentMatchers.argThat(argument -> argument.toString().contains(files.get(1).getName())), Mockito.any(), Mockito.any()); try { indexFiles(files, queryOptions, outputId); } catch (StoragePipelineException exception) { assertEquals(files.size(), exception.getResults().size()); assertTrue(exception.getResults().get(0).isTransformExecuted()); assertNull(exception.getResults().get(0).getTransformError()); assertTrue(exception.getResults().get(1).isTransformExecuted()); assertSame(transformException, exception.getResults().get(1).getTransformError()); for (int i = files.size(); i > 0; i--) { assertFalse(exception.getResults().get(1).isLoadExecuted()); assertNull(exception.getResults().get(1).getLoadError()); } } mockVariantStorageETL(); // File 0 already transformed. // Expecting to transform and load only file 1 indexFiles(files, singletonList(files.get(1)), queryOptions, outputId); }
@Test public void testIndexWithLoadErrorExternalOutputFolder() throws Exception { QueryOptions queryOptions = new QueryOptions(VariantStorageEngine.Options.ANNOTATE.key(), false) .append(VariantStorageEngine.Options.CALCULATE_STATS.key(), false); List<File> files = Arrays.asList(getFile(0), getFile(1)); queryOptions.put(DummyVariantStoragePipeline.VARIANTS_LOAD_FAIL, files.get(1).getName() + ".variants.avro.gz"); List<String> fileIds = files.stream().map(File::getId).collect(Collectors.toList()); try { String outdir = opencga.createTmpOutdir(studyId, "_INDEX_", sessionId); variantManager.index(studyId, fileIds, outdir, queryOptions, sessionId); fail(); } catch (StoragePipelineException exception) { assertEquals(files.size(), exception.getResults().size()); for (int i = 0; i < files.size(); i++) { assertTrue(exception.getResults().get(i).isTransformExecuted()); assertNull(exception.getResults().get(i).getTransformError()); } assertTrue(exception.getResults().get(0).isLoadExecuted()); assertNull(exception.getResults().get(0).getLoadError()); assertTrue(exception.getResults().get(1).isLoadExecuted()); } queryOptions.put(DummyVariantStoragePipeline.VARIANTS_LOAD_FAIL, false); indexFiles(files, singletonList(files.get(1)), queryOptions, outputId); }
private void checkStudyConfiguration(Study study, StudyConfiguration studyConfiguration) throws CatalogException { assertEquals("user@p1:s1", studyConfiguration.getStudyName()); assertEquals(study.getUid(), studyConfiguration.getStudyId()); assertTrue(studyConfiguration.getInvalidStats().isEmpty()); for (Map.Entry<String, Integer> entry : studyConfiguration.getFileIds().entrySet()) { File file = catalogManager.getFileManager().get(studyConfiguration.getStudyName(), studyConfiguration.getFileIds().inverse().get(entry.getValue()), null, sessionId).first(); assertEquals(file.getName(), entry.getKey()); int id = (int) file.getUid(); assertEquals(file.getSamples().stream().map(Sample::getUid).map(Long::intValue).collect(Collectors.toSet()), studyConfiguration.getSamplesInFiles().get((id))); if (file.getIndex() == null || file.getIndex().getStatus() == null || file.getIndex().getStatus().getName() == null || !file.getIndex().getStatus().getName().equals(FileIndex.IndexStatus.READY)) { assertFalse(studyConfiguration.getIndexedFiles().contains(id)); // assertFalse("Should not contain header for file " + file.getId(), studyConfiguration.getHeaders().containsKey(id)); } // else { // assertTrue(studyConfiguration.getIndexedFiles().contains(id)); // assertTrue("Missing header for file " + file.getId(), studyConfiguration.getHeaders().containsKey(id)); // assertTrue("Missing header for file " + file.getId(), !studyConfiguration.getHeaders().get(id).isEmpty()); // } } }
@Test public void testIndexWithLoadError() throws Exception { QueryOptions queryOptions = new QueryOptions(VariantStorageEngine.Options.ANNOTATE.key(), false) .append(VariantStorageEngine.Options.CALCULATE_STATS.key(), false); List<File> files = Arrays.asList(getFile(0), getFile(1)); queryOptions.put(DummyVariantStoragePipeline.VARIANTS_LOAD_FAIL, files.get(1).getName() + ".variants.avro.gz"); try { indexFiles(files, queryOptions, outputId); fail(); } catch (StoragePipelineException exception) { assertEquals(files.size(), exception.getResults().size()); for (int i = 0; i < files.size(); i++) { assertTrue(exception.getResults().get(i).isTransformExecuted()); assertNull(exception.getResults().get(i).getTransformError()); } assertTrue(exception.getResults().get(0).isLoadExecuted()); assertNull(exception.getResults().get(0).getLoadError()); assertTrue(exception.getResults().get(1).isLoadExecuted()); } queryOptions.put(DummyVariantStoragePipeline.VARIANTS_LOAD_FAIL, false); // File 0 already loaded. // Expecting to load only file 1 loadFiles(files, singletonList(files.get(1)), queryOptions, outputId); }
@Test public void renameFileTest() throws CatalogDBException { String newName = "newFile.bam"; String parentPath = "data/"; long fileId = catalogFileDBAdaptor.getId(user3.getProjects().get(0).getStudies().get(0).getUid(), "data/file.vcf"); System.out.println(catalogFileDBAdaptor.rename(fileId, parentPath + newName, "", null)); File file = catalogFileDBAdaptor.get(fileId, null).first(); assertEquals(file.getName(), newName); assertEquals(file.getPath(), parentPath + newName); try { catalogFileDBAdaptor.rename(-1, "noFile", "", null); fail("error: expected \"file not found\"exception"); } catch (CatalogDBException e) { System.out.println("correct exception: " + e); } long folderId = catalogFileDBAdaptor.getId(user3.getProjects().get(0).getStudies().get(0).getUid(), "data/"); String folderName = "folderName"; catalogFileDBAdaptor.rename(folderId, folderName, "", null); assertTrue(catalogFileDBAdaptor.get(fileId, null).first().getPath().equals(folderName + "/" + newName)); }
@Before public void setUp() throws Exception { scm = new StudyConfigurationManager(new DummyProjectMetadataAdaptor(), new DummyStudyConfigurationAdaptor(), new DummyVariantFileMetadataDBAdaptor()); Study study = catalogManager.getStudyManager().get(studyId, null, sessionId).first(); StudyConfiguration studyConfigurationToReturn = new StudyConfiguration((int) study.getUid(), "user@p1:s1"); studyConfigurationToReturn.setIndexedFiles(indexedFiles); for (File file : files) { if (indexedFiles.contains(((int) file.getUid()))) { studyConfigurationToReturn.getFileIds().put(file.getName(), ((int) file.getUid())); } } scm.updateStudyConfiguration(studyConfigurationToReturn, null); // checkStudyConfiguration(study, studyConfigurationToReturn); }
@Test public void testDeleteTransformedFile() throws Exception { QueryOptions queryOptions = new QueryOptions(VariantStorageEngine.Options.ANNOTATE.key(), false) .append(VariantStorageEngine.Options.CALCULATE_STATS.key(), false); File inputFile = getFile(0); File transformedFile = transformFile(inputFile, queryOptions); catalogManager.getFileManager().delete(studyFqn, new Query(FileDBAdaptor.QueryParams.NAME.key(), transformedFile.getName()), new ObjectMap(FileManager.SKIP_TRASH, true), sessionId); catalogManager.getFileManager().delete(studyFqn, new Query(FileDBAdaptor.QueryParams.NAME.key(), VariantReaderUtils.getMetaFromTransformedFile(transformedFile.getName())), new ObjectMap(FileManager.SKIP_TRASH, true), sessionId); indexFile(inputFile, queryOptions, outputId); }
@Test public void testGetFileParents3() throws CatalogException { long fileId; QueryResult<File> fileParents; fileId = catalogManager.getFileManager().getUid("data/test/", studyFqn, sessionIdUser).getResource().getUid(); fileParents = catalogManager.getFileManager().getParents(fileId, new QueryOptions("include", "projects.studies.files.path," + "projects.studies.files.id"), sessionIdUser); assertEquals(3, fileParents.getNumResults()); assertEquals("", fileParents.getResult().get(0).getPath()); assertEquals("data/", fileParents.getResult().get(1).getPath()); assertEquals("data/test/", fileParents.getResult().get(2).getPath()); fileParents.getResult().forEach(f -> { assertNull(f.getName()); assertNotNull(f.getPath()); assertTrue(f.getUid() != 0); }); }
@Test public void testUnlinkFolder() throws CatalogException, IOException { URI uri = Paths.get(getStudyURI()).resolve("data").toUri(); link(uri, "myDirectory", studyFqn, new ObjectMap("parents", true), sessionIdUser); CatalogIOManager ioManager = catalogManager.getCatalogIOManagerFactory().get(uri); Query query = new Query() .append(FileDBAdaptor.QueryParams.STUDY_UID.key(), studyUid) .append(FileDBAdaptor.QueryParams.PATH.key(), "~myDirectory/*") .append(FileDBAdaptor.QueryParams.STATUS_NAME.key(), File.FileStatus.READY); QueryResult<File> fileQueryResultLinked = catalogManager.getFileManager().get(studyFqn, query, null, sessionIdUser); System.out.println("Number of files/folders linked = " + fileQueryResultLinked.getNumResults()); // Now we try to unlink them catalogManager.getFileManager().unlink(studyFqn, "myDirectory/data/", sessionIdUser); fileQueryResultLinked = catalogManager.getFileManager().get(studyFqn, query, null, sessionIdUser); assertEquals(1, fileQueryResultLinked.getNumResults()); query = new Query() .append(FileDBAdaptor.QueryParams.STUDY_UID.key(), studyUid) .append(FileDBAdaptor.QueryParams.PATH.key(), "~myDirectory/*") .append(FileDBAdaptor.QueryParams.STATUS_NAME.key(), File.FileStatus.REMOVED); QueryResult<File> fileQueryResultUnlinked = catalogManager.getFileManager().get(studyFqn, query, null, sessionIdUser); assertEquals(6, fileQueryResultUnlinked.getNumResults()); String myPath = "myDirectory/data" + AbstractManager.INTERNAL_DELIMITER + "REMOVED"; for (File file : fileQueryResultUnlinked.getResult()) { assertTrue("File name should have been modified", file.getPath().contains(myPath)); assertEquals("Status should be to REMOVED", File.FileStatus.REMOVED, file.getStatus().getName()); assertEquals("Name should not have changed", file.getName(), file.getName()); assertTrue("File uri: " + file.getUri() + " should exist", ioManager.exists(file.getUri())); } }
@Test public void testDeleteFile() throws CatalogException, IOException { List<File> result = catalogManager.getFileManager().get(studyFqn, new Query(FileDBAdaptor.QueryParams.TYPE.key(), "FILE"), new QueryOptions(), sessionIdUser).getResult(); for (File file : result) { catalogManager.getFileManager().delete(studyFqn, new Query(FileDBAdaptor.QueryParams.UID.key(), file.getUid()), null, sessionIdUser); } // CatalogFileUtils catalogFileUtils = new CatalogFileUtils(catalogManager); catalogManager.getFileManager().get(studyFqn, new Query(FileDBAdaptor.QueryParams.TYPE.key(), "FILE"), new QueryOptions(), sessionIdUser).getResult().forEach(f -> { assertEquals(f.getStatus().getName(), File.FileStatus.TRASHED); assertTrue(f.getName().startsWith(".deleted")); }); result = catalogManager.getFileManager().get(studyFqn2, new Query(FileDBAdaptor.QueryParams.TYPE.key(), "FILE"), new QueryOptions(), sessionIdUser).getResult(); for (File file : result) { catalogManager.getFileManager().delete(studyFqn2, new Query(FileDBAdaptor.QueryParams.UID.key(), file.getUid()), null, sessionIdUser); } catalogManager.getFileManager().get(studyFqn, new Query(FileDBAdaptor.QueryParams.TYPE.key(), "FILE"), new QueryOptions(), sessionIdUser).getResult().forEach(f -> { assertEquals(f.getStatus().getName(), File.FileStatus.TRASHED); assertTrue(f.getName().startsWith(".deleted")); }); }
@Test public void deleteFolderTest2() throws CatalogException, IOException { List<File> folderFiles = new LinkedList<>(); File folder = createBasicDirectoryFileTestEnvironment(folderFiles); CatalogIOManager ioManager = catalogManager.getCatalogIOManagerFactory().get(catalogManager.getFileManager().getUri(folder)); for (File file : folderFiles) { assertTrue(ioManager.exists(catalogManager.getFileManager().getUri(file))); } catalogManager.getFileManager().delete(studyFqn, new Query(FileDBAdaptor.QueryParams.UID.key(), folder.getUid()), null, sessionIdUser); Query query = new Query() .append(FileDBAdaptor.QueryParams.UID.key(), folder.getUid()) .append(FileDBAdaptor.QueryParams.STATUS_NAME.key(), File.FileStatus.TRASHED); File fileTmp = catalogManager.getFileManager().get(studyFqn, query, QueryOptions.empty(), sessionIdUser).first(); assertEquals("Folder name should not be modified", folder.getPath(), fileTmp.getPath()); assertEquals("Status should be to TRASHED", File.FileStatus.TRASHED, fileTmp.getStatus().getName()); assertEquals("Name should not have changed", folder.getName(), fileTmp.getName()); assertTrue(ioManager.exists(fileTmp.getUri())); for (File file : folderFiles) { query.put(FileDBAdaptor.QueryParams.UID.key(), file.getUid()); fileTmp = fileManager.get(studyFqn, query, QueryOptions.empty(), sessionIdUser).first(); assertEquals("Folder name should not be modified", file.getPath(), fileTmp.getPath()); assertEquals("Status should be to TRASHED", File.FileStatus.TRASHED, fileTmp.getStatus().getName()); assertEquals("Name should not have changed", file.getName(), fileTmp.getName()); assertTrue("File uri: " + fileTmp.getUri() + " should exist", ioManager.exists(fileTmp.getUri())); } }