/** * Check status. To upload a fileObject, the catalog file entry must be in status "uploading". * Set "ignoreStatus = true" to ignore the actual file status and replace the content. * * @throws org.opencb.opencga.catalog.exceptions.CatalogIOException */ private void checkStatus(File file) throws CatalogIOException { if (!file.getStatus().getName().equals(File.FileStatus.STAGE)) { throw new CatalogIOException("File status is already uploaded and ready! " + "file:{ path: '" + file.getPath() + "', id:" + file.getPath() + ", status: '" + file.getStatus().getName() + "' } " + "Needs 'ignoreStatus = true' for continue."); } }
@Deprecated private void checkCanLinkFile(File file, boolean relink) throws CatalogException { String fileStatusEnum = file.getStatus().getName(); if (relink) { if (!fileStatusEnum.equals(File.FileStatus.READY)) { throw new CatalogIOException("Unable to relink a file with status : " + fileStatusEnum); } if (!file.isExternal()) { throw new CatalogIOException("Unable to relink a non linked file"); } } else { if (!fileStatusEnum.equals(File.FileStatus.STAGE) && !fileStatusEnum.equals(File.FileStatus.MISSING)) { throw new CatalogIOException("Unable to link a file with status : " + fileStatusEnum); } } }
@Test public void testScanStudyURI() throws IOException, CatalogException { CatalogManagerTest.createDebugFile(directory.resolve("file1.txt").toString()); FileScanner fileScanner = new FileScanner(catalogManager); List<File> files = fileScanner.scan(folder, directory.toUri(), FileScanner.FileScannerPolicy.REPLACE, true, true, sessionIdUser); assertEquals(1, files.size()); URI studyUri = study.getUri(); CatalogManagerTest.createDebugFile(studyUri.resolve("data/test/folder/").resolve("file2.txt").getPath()); File root = catalogManager.getFileManager().get(study.getFqn(), new Query("name", "."), null, sessionIdUser).first(); files = fileScanner.scan(root, studyUri, FileScanner.FileScannerPolicy.REPLACE, true, true, sessionIdUser); assertEquals(1, files.size()); files.forEach((f) -> assertTrue(f.getSize() > 0)); files.forEach((f) -> assertEquals(f.getStatus().getName(), File.FileStatus.READY)); files.forEach((f) -> assertTrue(StringUtils.isNotEmpty(f.getChecksum()))); }
@Test public void testComplexAdd() throws IOException, CatalogException, URISyntaxException { CatalogIOManager ioManager = catalogManager.getCatalogIOManagerFactory().getDefault(); URI fileUri = getClass().getResource("/biofiles/variant-test-file.vcf.gz").toURI(); ioManager.copyFile(fileUri, directory.resolve("file1.vcf.gz").toUri()); CatalogManagerTest.createDebugFile(directory.resolve("file1.vcf.variants.json").toString()); CatalogManagerTest.createDebugFile(directory.resolve("file1.vcf.variants.json.gz").toString()); CatalogManagerTest.createDebugFile(directory.resolve("file1.vcf.variants.json.snappy").toString()); CatalogManagerTest.createDebugFile(directory.resolve("file2.bam").toString()); CatalogManagerTest.createDebugFile(directory.resolve("file2.sam.gz").toString()); FileScanner fileScanner = new FileScanner(catalogManager); List<File> files = fileScanner.scan(folder, directory.toUri(), FileScanner.FileScannerPolicy.REPLACE, true, true, sessionIdUser); Map<String, File> map = files.stream().collect(Collectors.toMap(File::getName, (f) -> f)); assertEquals(6, files.size()); files.forEach((file) -> assertEquals(File.FileStatus.READY, file.getStatus().getName())); assertEquals(File.Bioformat.VARIANT, map.get("file1.vcf.gz").getBioformat()); assertEquals(File.Bioformat.VARIANT, map.get("file1.vcf.variants.json").getBioformat()); assertEquals(File.Bioformat.VARIANT, map.get("file1.vcf.variants.json.gz").getBioformat()); assertEquals(File.Bioformat.VARIANT, map.get("file1.vcf.variants.json.snappy").getBioformat()); assertEquals(File.Bioformat.ALIGNMENT, map.get("file2.bam").getBioformat()); assertEquals(File.Bioformat.ALIGNMENT, map.get("file2.sam.gz").getBioformat()); assertEquals(File.Format.VCF, map.get("file1.vcf.gz").getFormat()); assertEquals(File.Format.JSON, map.get("file1.vcf.variants.json").getFormat()); assertEquals(File.Format.JSON, map.get("file1.vcf.variants.json.gz").getFormat()); assertEquals(File.Format.JSON, map.get("file1.vcf.variants.json.snappy").getFormat()); assertEquals(File.Format.BAM, map.get("file2.bam").getFormat()); assertEquals(File.Format.SAM, map.get("file2.sam.gz").getFormat()); }
@Test public void testDeleteExisting() throws IOException, CatalogException { QueryResult<File> queryResult = catalogManager.getFileManager().upload(study.getFqn(), CatalogManagerTest.createDebugFile().toURI(), new File().setPath(folder.getPath() + "file1.txt"), false, false, sessionIdUser); File file = queryResult.first(); CatalogManagerTest.createDebugFile(directory.resolve("file1.txt").toString()); List<File> files = new FileScanner(catalogManager).scan(folder, directory.toUri(), FileScanner.FileScannerPolicy.DELETE, false, true, sessionIdUser); files.forEach((File f) -> assertFalse(f.getAttributes().containsKey("checksum"))); assertEquals(File.FileStatus.DELETED, getFile(file.getUid()).getStatus().getName()); }
@Test public void testGetMetadataFromVcf() throws CatalogException, FileNotFoundException { File file = catalogManager.getFileManager().upload(study.getFqn(), vcfFileUri, new File().setPath(folder.getPath() + VCF_FILE_NAME), false, false, sessionIdUser).first(); assertTrue(file.getSize() > 0); file = FileMetadataReader.get(catalogManager). setMetadataInformation(file, null, null, sessionIdUser, false); assertEquals(File.FileStatus.READY, file.getStatus().getName()); assertEquals(File.Format.VCF, file.getFormat()); assertEquals(File.Bioformat.VARIANT, file.getBioformat()); assertNotNull(file.getAttributes().get(VARIANT_FILE_METADATA)); assertEquals(4, file.getSamples().size()); assertEquals(expectedSampleNames, ((Map<String, Object>) file.getAttributes().get(VARIANT_FILE_METADATA)).get("sampleIds")); catalogManager.getSampleManager().get(study.getFqn(), new Query(SampleDBAdaptor.QueryParams.ID.key(), file.getSamples().stream().map(Sample::getId).collect(Collectors.toList())), new QueryOptions(), sessionIdUser).getResult(); assertTrue(expectedSampleNames.containsAll(file.getSamples().stream().map(Sample::getId).collect(Collectors.toSet()))); }
@Test public void modifyFileTest() throws CatalogDBException, IOException { File file = user3.getProjects().get(0).getStudies().get(0).getFiles().get(0); long fileId = file.getUid(); Document stats = new Document("stat1", 1).append("stat2", true).append("stat3", "ok" + StringUtils.randomString(20)); ObjectMap parameters = new ObjectMap(); parameters.put("status.name", File.FileStatus.READY); parameters.put("stats", stats); System.out.println(catalogFileDBAdaptor.update(fileId, parameters, QueryOptions.empty())); file = catalogFileDBAdaptor.get(fileId, null).first(); assertEquals(file.getStatus().getName(), File.FileStatus.READY); assertEquals(file.getStats(), stats); parameters = new ObjectMap(); parameters.put("stats", "{}"); System.out.println(catalogFileDBAdaptor.update(fileId, parameters, QueryOptions.empty())); file = catalogFileDBAdaptor.get(fileId, null).first(); assertEquals(file.getStats(), new LinkedHashMap<String, Object>()); }
@Test public void testCreate() throws CatalogException { QueryResult<File> fileQueryResult = FileMetadataReader.get(catalogManager). create(study.getFqn(), vcfFileUri, folder.getPath() + VCF_FILE_NAME, "", false, null, sessionIdUser); File file = fileQueryResult.first(); assertEquals(File.FileStatus.STAGE, file.getStatus().getName()); assertEquals(File.Format.VCF, file.getFormat()); assertEquals(File.Bioformat.VARIANT, file.getBioformat()); assertNotNull(file.getAttributes().get(VARIANT_FILE_METADATA)); assertEquals(4, file.getSamples().size()); assertEquals(21499, file.getSize()); new FileUtils(catalogManager).upload(vcfFileUri, file, null, sessionIdUser, false, false, true, true, Integer.MAX_VALUE); file = catalogManager.getFileManager().get(study.getFqn(), file.getPath(), null, sessionIdUser).first(); assertEquals(File.FileStatus.READY, file.getStatus().getName()); assertEquals(File.Format.VCF, file.getFormat()); assertEquals(File.Bioformat.VARIANT, file.getBioformat()); assertNotNull(file.getAttributes().get(VARIANT_FILE_METADATA)); assertNotNull(((Map) file.getAttributes().get(VARIANT_FILE_METADATA)).get("sampleIds")); assertEquals(4, ((List) ((Map) file.getAttributes().get(VARIANT_FILE_METADATA)).get("sampleIds")).size()); assertNotNull(((Map) file.getAttributes().get(VARIANT_FILE_METADATA)).get("header")); assertEquals(4, file.getSamples().size()); assertEquals(21499, file.getSize()); }
assertEquals(localDate.getDayOfWeek().toString(), fileSolrModel.getCreationDayOfWeek()); assert (fileSolrModel.getStatus().equals(file.getStatus().getName())); assert (fileSolrModel.isExternal() == file.isExternal()); assert (fileSolrModel.getSize() == file.getSize());
@Test public void testReplaceExisting() throws IOException, CatalogException { // Create and register file1.txt and s/file2.txt File file = catalogManager.getFileManager().upload(study.getFqn(), CatalogManagerTest.createDebugFile().toURI(), new File().setPath(folder.getPath() + "file1.txt"), false, false, true, true, sessionIdUser).first(); catalogManager.getFileManager().upload(study.getFqn(), CatalogManagerTest.createDebugFile().toURI(), new File().setPath(folder.getPath() + "s/file2.txt"), false, true, true, true, sessionIdUser).first(); // Create same file structure, and replace CatalogManagerTest.createDebugFile(directory.resolve("file1.txt").toString()); Files.createDirectory(directory.resolve("s/")); CatalogManagerTest.createDebugFile(directory.resolve("s/file2.txt").toString()); FileScanner fileScanner = new FileScanner(catalogManager); fileScanner.scan(folder, directory.toUri(), FileScanner.FileScannerPolicy.REPLACE, true, true, sessionIdUser); File replacedFile = catalogManager.getFileManager().get(study.getFqn(), file.getPath(), null, sessionIdUser).first(); assertEquals(File.FileStatus.READY, replacedFile.getStatus().getName()); assertEquals(file.getUid(), replacedFile.getUid()); assertNotEquals(replacedFile.getChecksum(), file.getChecksum()); assertEquals(replacedFile.getChecksum(), catalogManager.getCatalogIOManagerFactory().getDefault().calculateChecksum(replacedFile.getUri())); }
@Test public void testGetMetadataFromBam() throws CatalogException, FileNotFoundException { File file = catalogManager.getFileManager().upload(study.getFqn(), bamFileUri, new File().setPath(folder.getPath() + BAM_FILE_NAME), false, false, sessionIdUser).first(); assertTrue(file.getSize() > 0); file = FileMetadataReader.get(catalogManager). setMetadataInformation(file, null, null, sessionIdUser, false); assertEquals(File.FileStatus.READY, file.getStatus().getName()); // assertEquals(File.Format.GZIP, file.getFormat()); assertEquals(File.Bioformat.ALIGNMENT, file.getBioformat()); assertNotNull(file.getAttributes().get("alignmentHeader")); assertEquals(1, file.getSamples().size()); assertEquals("HG00096", catalogManager.getSampleManager().get(study.getFqn(), file.getSamples().get(0).getId(), null, sessionIdUser).first().getId()); }
@Test public void testDoNotOverwriteSampleIds() throws CatalogException, FileNotFoundException { File file = catalogManager.getFileManager().upload(study.getFqn(), vcfFileUri, new File().setPath(folder.getPath() + VCF_FILE_NAME), false, false, sessionIdUser).first(); assertEquals(File.FileStatus.READY, file.getStatus().getName()); assertEquals(File.Format.VCF, file.getFormat()); assertEquals(File.Bioformat.VARIANT, file.getBioformat()); assertNotNull(file.getAttributes().get(VARIANT_FILE_METADATA)); assertEquals(4, file.getSamples().size()); //Add a sampleId String sampleId = catalogManager.getSampleManager().create(study.getFqn(), new Sample().setId("Bad_Sample"), null, sessionIdUser) .first().getId(); catalogManager.getFileManager().update(study.getFqn(), file.getPath(), new ObjectMap(FileDBAdaptor.QueryParams.SAMPLES.key(), Collections.singletonList(sampleId)), new QueryOptions(), sessionIdUser); file = catalogManager.getFileManager().get(study.getFqn(), file.getPath(), null, sessionIdUser).first(); assertEquals(5, file.getSamples().size()); assertEquals(sampleId, file.getSamples().get(4).getId()); }
@Test public void testDeleteFile() throws CatalogException, IOException { List<File> result = catalogManager.getFileManager().get(studyFqn, new Query(FileDBAdaptor.QueryParams.TYPE.key(), "FILE"), new QueryOptions(), sessionIdUser).getResult(); for (File file : result) { catalogManager.getFileManager().delete(studyFqn, new Query(FileDBAdaptor.QueryParams.UID.key(), file.getUid()), null, sessionIdUser); } // CatalogFileUtils catalogFileUtils = new CatalogFileUtils(catalogManager); catalogManager.getFileManager().get(studyFqn, new Query(FileDBAdaptor.QueryParams.TYPE.key(), "FILE"), new QueryOptions(), sessionIdUser).getResult().forEach(f -> { assertEquals(f.getStatus().getName(), File.FileStatus.TRASHED); assertTrue(f.getName().startsWith(".deleted")); }); result = catalogManager.getFileManager().get(studyFqn2, new Query(FileDBAdaptor.QueryParams.TYPE.key(), "FILE"), new QueryOptions(), sessionIdUser).getResult(); for (File file : result) { catalogManager.getFileManager().delete(studyFqn2, new Query(FileDBAdaptor.QueryParams.UID.key(), file.getUid()), null, sessionIdUser); } catalogManager.getFileManager().get(studyFqn, new Query(FileDBAdaptor.QueryParams.TYPE.key(), "FILE"), new QueryOptions(), sessionIdUser).getResult().forEach(f -> { assertEquals(f.getStatus().getName(), File.FileStatus.TRASHED); assertTrue(f.getName().startsWith(".deleted")); }); }
@Test public void deleteFolderTest3() throws CatalogException, IOException { List<File> folderFiles = new LinkedList<>(); File folder = createBasicDirectoryFileTestEnvironment(folderFiles); CatalogIOManager ioManager = catalogManager.getCatalogIOManagerFactory().get(catalogManager.getFileManager().getUri(folder)); for (File file : folderFiles) { assertTrue(ioManager.exists(catalogManager.getFileManager().getUri(file))); } catalogManager.getFileManager().delete(studyFqn, new Query(FileDBAdaptor.QueryParams.UID.key(), folder.getUid()), new ObjectMap(FileManager.SKIP_TRASH, true), sessionIdUser); Query query = new Query() .append(FileDBAdaptor.QueryParams.UID.key(), folder.getUid()) .append(FileDBAdaptor.QueryParams.STATUS_NAME.key(), File.FileStatus.PENDING_DELETE); File fileTmp = fileManager.get(studyFqn, query, QueryOptions.empty(), sessionIdUser).first(); String myPath = Paths.get(folder.getPath()) + AbstractManager.INTERNAL_DELIMITER + "DELETED"; assertTrue("Folder name should have been modified", fileTmp.getPath().contains(myPath)); assertEquals("Status should be to PENDING_DELETE", File.FileStatus.PENDING_DELETE, fileTmp.getStatus().getName()); assertEquals("Name should not have changed", folder.getName(), fileTmp.getName()); assertTrue(ioManager.exists(fileTmp.getUri())); for (File file : folderFiles) { query.put(FileDBAdaptor.QueryParams.UID.key(), file.getUid()); fileTmp = fileManager.get(studyFqn, query, QueryOptions.empty(), sessionIdUser).first(); assertTrue("Folder name should have been modified", fileTmp.getPath().contains(myPath)); assertEquals("Status should be to PENDING_DELETE", File.FileStatus.PENDING_DELETE, fileTmp.getStatus().getName()); assertEquals("Name should not have changed", file.getName(), fileTmp.getName()); assertTrue("File uri: " + fileTmp.getUri() + " should exist", ioManager.exists(fileTmp.getUri())); } }
@Test public void testUnlinkFolder() throws CatalogException, IOException { URI uri = Paths.get(getStudyURI()).resolve("data").toUri(); link(uri, "myDirectory", studyFqn, new ObjectMap("parents", true), sessionIdUser); CatalogIOManager ioManager = catalogManager.getCatalogIOManagerFactory().get(uri); Query query = new Query() .append(FileDBAdaptor.QueryParams.STUDY_UID.key(), studyUid) .append(FileDBAdaptor.QueryParams.PATH.key(), "~myDirectory/*") .append(FileDBAdaptor.QueryParams.STATUS_NAME.key(), File.FileStatus.READY); QueryResult<File> fileQueryResultLinked = catalogManager.getFileManager().get(studyFqn, query, null, sessionIdUser); System.out.println("Number of files/folders linked = " + fileQueryResultLinked.getNumResults()); // Now we try to unlink them catalogManager.getFileManager().unlink(studyFqn, "myDirectory/data/", sessionIdUser); fileQueryResultLinked = catalogManager.getFileManager().get(studyFqn, query, null, sessionIdUser); assertEquals(1, fileQueryResultLinked.getNumResults()); query = new Query() .append(FileDBAdaptor.QueryParams.STUDY_UID.key(), studyUid) .append(FileDBAdaptor.QueryParams.PATH.key(), "~myDirectory/*") .append(FileDBAdaptor.QueryParams.STATUS_NAME.key(), File.FileStatus.REMOVED); QueryResult<File> fileQueryResultUnlinked = catalogManager.getFileManager().get(studyFqn, query, null, sessionIdUser); assertEquals(6, fileQueryResultUnlinked.getNumResults()); String myPath = "myDirectory/data" + AbstractManager.INTERNAL_DELIMITER + "REMOVED"; for (File file : fileQueryResultUnlinked.getResult()) { assertTrue("File name should have been modified", file.getPath().contains(myPath)); assertEquals("Status should be to REMOVED", File.FileStatus.REMOVED, file.getStatus().getName()); assertEquals("Name should not have changed", file.getName(), file.getName()); assertTrue("File uri: " + file.getUri() + " should exist", ioManager.exists(file.getUri())); } }
assertEquals("Status should be to DELETED", File.FileStatus.DELETED, fileTmp.getStatus().getName()); assertEquals("Name should not have changed", folder.getName(), fileTmp.getName()); assertTrue(!ioManager.exists(fileTmp.getUri())); fileTmp = fileManager.get(studyFqn, query, QueryOptions.empty(), sessionIdUser).first(); assertTrue("Folder name should have been modified", fileTmp.getPath().contains(myPath)); assertEquals("Status should be to DELETED", File.FileStatus.DELETED, fileTmp.getStatus().getName()); assertEquals("Name should not have changed", file.getName(), fileTmp.getName()); assertTrue("File uri: " + fileTmp.getUri() + " should not exist", !ioManager.exists(fileTmp.getUri()));
assertEquals("Status should be DELETED", File.FileStatus.DELETED, fileTmp.getStatus().getName()); assertEquals("Name should not have changed", folder.getName(), fileTmp.getName()); assertTrue(!ioManager.exists(fileTmp.getUri())); fileTmp = fileManager.get(studyFqn, query, QueryOptions.empty(), sessionIdUser).first(); assertTrue("Folder name should have been modified", fileTmp.getPath().contains(myPath)); assertEquals("Status should be to DELETED", File.FileStatus.DELETED, fileTmp.getStatus().getName()); assertEquals("Name should not have changed", file.getName(), fileTmp.getName()); assertTrue("File uri: " + fileTmp.getUri() + " should not exist", !ioManager.exists(fileTmp.getUri()));
private void deleteFolderAndCheck(File deletable) throws CatalogException, IOException { List<File> allFilesInFolder; Study study = catalogManager.getFileManager().getStudy(deletable, sessionIdUser); catalogManager.getFileManager().delete(study.getFqn(), new Query(FileDBAdaptor.QueryParams.PATH.key(), deletable.getPath()), null, sessionIdUser); Query query = new Query() .append(FileDBAdaptor.QueryParams.PATH.key(), deletable.getPath()) .append(FileDBAdaptor.QueryParams.STATUS_NAME.key(), File.FileStatus.TRASHED); QueryOptions options = new QueryOptions(QueryOptions.INCLUDE, FileDBAdaptor.QueryParams.PATH.key()); QueryResult<File> fileQueryResult = catalogManager.getFileManager().get(study.getFqn(), query, options, sessionIdUser); assertEquals(1, fileQueryResult.getNumResults()); // allFilesInFolder = catalogManager.getAllFilesInFolder(deletable, null, sessionIdUser).getResult(); query = new Query() .append(FileDBAdaptor.QueryParams.DIRECTORY.key(), fileQueryResult.first().getPath() + ".*") .append(FileDBAdaptor.QueryParams.STATUS_NAME.key(), File.FileStatus.TRASHED); allFilesInFolder = catalogManager.getFileManager().get(study.getFqn(), query, null, sessionIdUser).getResult(); for (File subFile : allFilesInFolder) { assertTrue(subFile.getStatus().getName().equals(File.FileStatus.TRASHED)); } }
@Test public void testDeleteTrashed() throws IOException, CatalogException { QueryResult<File> queryResult = catalogManager.getFileManager().upload(study.getFqn(), CatalogManagerTest.createDebugFile().toURI(), new File().setPath(folder.getPath() + "file1.txt"), false, false, sessionIdUser); File file = queryResult.first(); catalogManager.getFileManager().delete(study.getFqn(), new Query(FileDBAdaptor.QueryParams.UID.key(), file.getUid()), new QueryOptions(), sessionIdUser); QueryResult<File> fileQueryResult = catalogManager.getFileManager().get(study.getFqn(), new Query() .append(FileDBAdaptor.QueryParams.UID.key(), file.getUid()) .append(FileDBAdaptor.QueryParams.STATUS_NAME.key(), "!=EMPTY"), new QueryOptions(), sessionIdUser); file = fileQueryResult.first(); assertEquals(File.FileStatus.TRASHED, file.getStatus().getName()); }
@Test public void deleteFolderTest2() throws CatalogException, IOException { List<File> folderFiles = new LinkedList<>(); File folder = createBasicDirectoryFileTestEnvironment(folderFiles); CatalogIOManager ioManager = catalogManager.getCatalogIOManagerFactory().get(catalogManager.getFileManager().getUri(folder)); for (File file : folderFiles) { assertTrue(ioManager.exists(catalogManager.getFileManager().getUri(file))); } catalogManager.getFileManager().delete(studyFqn, new Query(FileDBAdaptor.QueryParams.UID.key(), folder.getUid()), null, sessionIdUser); Query query = new Query() .append(FileDBAdaptor.QueryParams.UID.key(), folder.getUid()) .append(FileDBAdaptor.QueryParams.STATUS_NAME.key(), File.FileStatus.TRASHED); File fileTmp = catalogManager.getFileManager().get(studyFqn, query, QueryOptions.empty(), sessionIdUser).first(); assertEquals("Folder name should not be modified", folder.getPath(), fileTmp.getPath()); assertEquals("Status should be to TRASHED", File.FileStatus.TRASHED, fileTmp.getStatus().getName()); assertEquals("Name should not have changed", folder.getName(), fileTmp.getName()); assertTrue(ioManager.exists(fileTmp.getUri())); for (File file : folderFiles) { query.put(FileDBAdaptor.QueryParams.UID.key(), file.getUid()); fileTmp = fileManager.get(studyFqn, query, QueryOptions.empty(), sessionIdUser).first(); assertEquals("Folder name should not be modified", file.getPath(), fileTmp.getPath()); assertEquals("Status should be to TRASHED", File.FileStatus.TRASHED, fileTmp.getStatus().getName()); assertEquals("Name should not have changed", file.getName(), fileTmp.getName()); assertTrue("File uri: " + fileTmp.getUri() + " should exist", ioManager.exists(fileTmp.getUri())); } }