public URI getUri(File file) throws CatalogException { ParamUtils.checkObj(file, "File"); if (file.getUri() != null) { return file.getUri(); } else { QueryResult<File> fileQueryResult = fileDBAdaptor.get(file.getUid(), INCLUDE_STUDY_URI); if (fileQueryResult.getNumResults() == 0) { throw new CatalogException("File " + file.getUid() + " not found"); } return fileQueryResult.first().getUri(); } }
/** * Get the URI where a file should be in Catalog, given a study and a path. * * @param studyId Study identifier * @param path Path to locate * @param directory Boolean indicating if the file is a directory * @return URI where the file should be placed * @throws CatalogException CatalogException */ private URI getFileUri(long studyId, String path, boolean directory) throws CatalogException, URISyntaxException { // Get the closest existing parent. If parents == true, may happen that the parent is not registered in catalog yet. File existingParent = getParents(false, null, path, studyId).first(); //Relative path to the existing parent String relativePath = Paths.get(existingParent.getPath()).relativize(Paths.get(path)).toString(); if (path.endsWith("/") && !relativePath.endsWith("/")) { relativePath += "/"; } String uriStr = Paths.get(existingParent.getUri().getPath()).resolve(relativePath).toString(); if (directory) { return UriUtils.createDirectoryUri(uriStr); } else { return UriUtils.createUri(uriStr); } }
logger.info("Scanning files from {} to move to {}", tmpOutdirPath, outDir.getUri());
@Deprecated public URI getUri(long studyId, String filePath) throws CatalogException { ParamUtils.checkObj(filePath, "filePath"); List<File> parents = getParents(false, INCLUDE_FILE_URI_PATH, filePath, studyId).getResult(); for (File parent : parents) { if (parent.getUri() != null) { if (parent.isExternal()) { throw new CatalogException("Cannot upload files to an external folder"); } String relativePath = filePath.replaceFirst(parent.getPath(), ""); return Paths.get(parent.getUri()).resolve(relativePath).toUri(); } } URI studyUri = getStudyUri(studyId); return filePath.isEmpty() ? studyUri : catalogIOManagerFactory.get(studyUri).getFileUri(studyUri, filePath); }
@Deprecated private Path getFilePath(long fileId, String sessionId) throws CatalogException, IOException { QueryOptions fileOptions = new QueryOptions(QueryOptions.INCLUDE, Arrays.asList(FileDBAdaptor.QueryParams.URI.key(), FileDBAdaptor.QueryParams.NAME.key())); QueryResult<File> fileQueryResult = catalogManager.getFileManager().get(fileId, fileOptions, sessionId); if (fileQueryResult.getNumResults() != 1) { logger.error("Critical error: File {} not found in catalog.", fileId); throw new CatalogException("Critical error: File " + fileId + " not found in catalog"); } Path path = Paths.get(fileQueryResult.first().getUri().getRawPath()); FileUtils.checkFile(path); return path; }
private QueryResult<File> create(Study study, File file, boolean parents, String content, QueryOptions options, String sessionId) throws CatalogException { QueryResult<File> queryResult = register(study, file, parents, options, sessionId); if (file.getType() == File.Type.FILE && StringUtils.isNotEmpty(content)) { CatalogIOManager ioManager = catalogIOManagerFactory.getDefault(); // We set parents to true because the file has been successfully registered, which means the directories are already registered // in catalog ioManager.createDirectory(Paths.get(file.getUri()).getParent().toUri(), true); InputStream inputStream = new ByteArrayInputStream(content.getBytes(StandardCharsets.UTF_8)); ioManager.createFile(file.getUri(), inputStream); // Update file parameters ObjectMap params = new ObjectMap() .append(FileDBAdaptor.QueryParams.STATUS_NAME.key(), File.FileStatus.READY) .append(FileDBAdaptor.QueryParams.SIZE.key(), ioManager.getFileSize(file.getUri())); queryResult = fileDBAdaptor.update(file.getUid(), params, QueryOptions.empty()); } return queryResult; }
@Test public void testLinkNormalizedUris() throws CatalogException, IOException, URISyntaxException { Path path = Paths.get(getStudyURI().resolve("data")); URI uri = new URI("file://" + path.toString() + "/../data"); ObjectMap params = new ObjectMap("parents", true); QueryResult<File> allFiles = link(uri, "test/myLinkedFolder/", studyFqn, params, sessionIdUser); assertEquals(6, allFiles.getNumResults()); for (File file : allFiles.getResult()) { assertTrue(file.getUri().isAbsolute()); assertEquals(file.getUri().normalize(), file.getUri()); } }
@Test public void testRegisterFiles() throws IOException, CatalogException { Path file1 = directory.resolve("file1.txt"); Path file2 = directory.resolve("s/file2.txt"); Path folder = directory.resolve("s/"); Path file3 = directory.resolve("file3.txt"); CatalogManagerTest.createDebugFile(file1.toString()); Files.createDirectory(folder); CatalogManagerTest.createDebugFile(file2.toString()); CatalogManagerTest.createDebugFile(file3.toString()); List<Path> filePaths = new ArrayList<>(2); filePaths.add(file1); filePaths.add(file2); FileScanner fileScanner = new FileScanner(catalogManager); // List<File> files = fileScanner.registerFiles(this.folder, filePaths, FileScanner.FileScannerPolicy.DELETE, true, false, sessionIdUser); Predicate<URI> uriPredicate = uri -> uri.getPath().endsWith("file1.txt") || uri.getPath().endsWith("file2.txt"); List<File> files = fileScanner.scan(this.folder, directory.toUri(), FileScanner.FileScannerPolicy.DELETE, true, false, uriPredicate, -1, sessionIdUser); assertEquals(2, files.size()); for (File file : files) { assertTrue(Paths.get(file.getUri()).toFile().exists()); } for (Path filePath : filePaths) { assertTrue(filePath.toFile().exists()); } assertTrue(file3.toFile().exists()); }
@Test public void testCreateFileInLinkedFolder() throws Exception { // Create an empty folder Path dir = catalogManagerResource.getOpencgaHome().resolve("folder_to_link"); Files.createDirectory(dir); URI uri = dir.toUri(); // Link the folder in the root link(uri, "", studyFqn, new ObjectMap(), sessionIdUser); File file = catalogManager.getFileManager().create(studyFqn, File.Type.FILE, File.Format.PLAIN, File.Bioformat .NONE, "folder_to_link/file.txt", null, "", null, 0, -1, null, (long) -1, null, null, false, null, null, sessionIdUser).first(); assertEquals(uri.resolve("file.txt"), file.getUri()); }
public QueryResult<RegionCoverage> getLowCoverageRegions(String studyIdStr, String fileIdStr, Region region, int minCoverage, String sessionId) throws Exception { QueryResult<File> fileQueryResult = catalogManager.getFileManager().get(studyIdStr, fileIdStr, new QueryOptions(QueryOptions.INCLUDE, Arrays.asList(FileDBAdaptor.QueryParams.URI.key(), FileDBAdaptor.QueryParams.BIOFORMAT.key(), FileDBAdaptor.QueryParams.FORMAT.key())), sessionId); if (fileQueryResult.getNumResults() == 0) { throw new CatalogException("File " + fileIdStr + " not found"); } checkAlignmentBioformat(fileQueryResult.first()); checkAlignmentFormat(fileQueryResult.first()); return alignmentStorageEngine.getDBAdaptor() .getLowCoverageRegions(Paths.get(fileQueryResult.first().getUri()), region, minCoverage); }
@Test public void testLinkFolder4() throws CatalogException, IOException { URI uri = Paths.get(getStudyURI()).resolve("data").toUri(); ObjectMap params = new ObjectMap("parents", true); QueryResult<File> allFiles = link(uri, "test/myLinkedFolder/", studyFqn, params, sessionIdUser); assertEquals(6, allFiles.getNumResults()); QueryResult<File> sameAllFiles = link(uri, "test/myLinkedFolder/", studyFqn, params, sessionIdUser); assertEquals(allFiles.getNumResults(), sameAllFiles.getNumResults()); List<File> result = allFiles.getResult(); for (int i = 0; i < result.size(); i++) { assertEquals(allFiles.getResult().get(i).getUid(), sameAllFiles.getResult().get(i).getUid()); assertEquals(allFiles.getResult().get(i).getPath(), sameAllFiles.getResult().get(i).getPath()); assertEquals(allFiles.getResult().get(i).getUri(), sameAllFiles.getResult().get(i).getUri()); } thrown.expect(CatalogException.class); thrown.expectMessage("already linked"); link(uri, "data", studyFqn, new ObjectMap(), sessionIdUser); }
@Test public void testCreateFile() throws CatalogException, IOException { String content = "This is the content\tof the file"; try { fileManager.create(studyFqn3, File.Type.FILE, File.Format.UNKNOWN, File.Bioformat.UNKNOWN, "data/test/myTest/myFile.txt", null, null, new File.FileStatus(File.FileStatus.READY), 0, -1, null, -1, null, null, false, "This is the content\tof the file", null, sessionIdUser2); fail("An error should be raised because parents is false"); } catch (CatalogException e) { System.out.println("Correct"); } QueryResult<File> fileQueryResult = fileManager.create(studyFqn3, File.Type.FILE, File.Format.UNKNOWN, File.Bioformat.UNKNOWN, "data/test/myTest/myFile.txt", null, null, new File.FileStatus(File.FileStatus.READY), 0, -1, null, -1, null, null, true, content, null, sessionIdUser2); CatalogIOManager ioManager = catalogManager.getCatalogIOManagerFactory().get(fileQueryResult.first().getUri()); assertTrue(ioManager.exists(fileQueryResult.first().getUri())); DataInputStream fileObject = ioManager.getFileObject(fileQueryResult.first().getUri(), -1, -1); assertEquals(content, fileObject.readLine()); }
@Test public void deleteFilesTest2() throws CatalogException, IOException { QueryResult<File> queryResult = catalogManager.getFileManager().create(studyFqn, File.Type.FILE, File.Format.PLAIN, File.Bioformat.NONE, "my.txt", null, "", new File.FileStatus(File.FileStatus.STAGE), 0, -1, null, -1, null, null, false, null, null, userSessionId); new FileUtils(catalogManager).upload(new ByteArrayInputStream(StringUtils.randomString(200) .getBytes()), queryResult.first(), userSessionId, false, false, true); File file = catalogManager.getFileManager().get(studyFqn, queryResult.first().getPath(), null, userSessionId).first(); CatalogIOManager ioManager = catalogManager.getCatalogIOManagerFactory().get(file.getUri()); assertTrue(ioManager.exists(file.getUri())); catalogManager.getFileManager().delete(studyFqn, new Query(FileDBAdaptor.QueryParams.UID.key(), file.getUid()), null, userSessionId); assertTrue(ioManager.exists(file.getUri())); // catalogFileUtils.delete(file.getId(), userSessionId); // assertTrue(!ioManager.exists(catalogManager.getFileUri(catalogManager.getFile(file.getId(), userSessionId).first()))); }
@Test public void testReplaceExisting() throws IOException, CatalogException { // Create and register file1.txt and s/file2.txt File file = catalogManager.getFileManager().upload(study.getFqn(), CatalogManagerTest.createDebugFile().toURI(), new File().setPath(folder.getPath() + "file1.txt"), false, false, true, true, sessionIdUser).first(); catalogManager.getFileManager().upload(study.getFqn(), CatalogManagerTest.createDebugFile().toURI(), new File().setPath(folder.getPath() + "s/file2.txt"), false, true, true, true, sessionIdUser).first(); // Create same file structure, and replace CatalogManagerTest.createDebugFile(directory.resolve("file1.txt").toString()); Files.createDirectory(directory.resolve("s/")); CatalogManagerTest.createDebugFile(directory.resolve("s/file2.txt").toString()); FileScanner fileScanner = new FileScanner(catalogManager); fileScanner.scan(folder, directory.toUri(), FileScanner.FileScannerPolicy.REPLACE, true, true, sessionIdUser); File replacedFile = catalogManager.getFileManager().get(study.getFqn(), file.getPath(), null, sessionIdUser).first(); assertEquals(File.FileStatus.READY, replacedFile.getStatus().getName()); assertEquals(file.getUid(), replacedFile.getUid()); assertNotEquals(replacedFile.getChecksum(), file.getChecksum()); assertEquals(replacedFile.getChecksum(), catalogManager.getCatalogIOManagerFactory().getDefault().calculateChecksum(replacedFile.getUri())); }
@Test public void deleteFoldersTest() throws CatalogException, IOException { List<File> folderFiles = new LinkedList<>(); File folder = prepareFiles(folderFiles); CatalogIOManager ioManager = catalogManager.getCatalogIOManagerFactory().get(catalogManager.getFileManager().getUri(folder)); for (File file : folderFiles) { assertTrue(ioManager.exists(catalogManager.getFileManager().getUri(file))); } catalogManager.getFileManager().delete(studyFqn, new Query(FileDBAdaptor.QueryParams.UID.key(), folder.getUid()), null, userSessionId); Query query = new Query() .append(FileDBAdaptor.QueryParams.UID.key(), folder.getUid()) .append(FileDBAdaptor.QueryParams.STATUS_NAME.key(), File.FileStatus.TRASHED); QueryResult<File> fileQueryResult = catalogManager.getFileManager().get(studyFqn, query, QueryOptions.empty(), userSessionId); assertTrue(ioManager.exists(fileQueryResult.first().getUri())); for (File file : folderFiles) { assertTrue("File uri: " + file.getUri() + " should exist", ioManager.exists(file.getUri())); } }
@Test public void deleteFolderTest() throws CatalogException, IOException { List<File> folderFiles = new LinkedList<>(); File folder = createBasicDirectoryFileTestEnvironment(folderFiles); CatalogIOManager ioManager = catalogManager.getCatalogIOManagerFactory().get(catalogManager.getFileManager().getUri(folder)); for (File file : folderFiles) { assertTrue(ioManager.exists(catalogManager.getFileManager().getUri(file))); } catalogManager.getFileManager().create(studyFqn, File.Type.FILE, File.Format.PLAIN, File.Bioformat.NONE, "folder/subfolder/subsubfolder/my_staged.txt", null, null, new File.FileStatus(File.FileStatus.STAGE), (long) 0, (long) -1, null, (long) -1, null, null, true, null, null, sessionIdUser).first(); WriteResult deleteResult = catalogManager.getFileManager().delete(studyFqn, new Query(FileDBAdaptor.QueryParams.UID.key(), folder.getUid()), null, sessionIdUser); assertEquals(0, deleteResult.getNumModified()); File fileTmp = catalogManager.getFileManager().get(studyFqn, folder.getPath(), null, sessionIdUser).first(); assertEquals("Folder name should not be modified", folder.getPath(), fileTmp.getPath()); assertTrue(ioManager.exists(fileTmp.getUri())); for (File file : folderFiles) { fileTmp = catalogManager.getFileManager().get(studyFqn, file.getPath(), null, sessionIdUser).first(); assertEquals("File name should not be modified", file.getPath(), fileTmp.getPath()); assertTrue("File uri: " + fileTmp.getUri() + " should exist", ioManager.exists(fileTmp.getUri())); } }
@Test public void testUnlinkFolder() throws CatalogException, IOException { URI uri = Paths.get(getStudyURI()).resolve("data").toUri(); link(uri, "myDirectory", studyFqn, new ObjectMap("parents", true), sessionIdUser); CatalogIOManager ioManager = catalogManager.getCatalogIOManagerFactory().get(uri); Query query = new Query() .append(FileDBAdaptor.QueryParams.STUDY_UID.key(), studyUid) .append(FileDBAdaptor.QueryParams.PATH.key(), "~myDirectory/*") .append(FileDBAdaptor.QueryParams.STATUS_NAME.key(), File.FileStatus.READY); QueryResult<File> fileQueryResultLinked = catalogManager.getFileManager().get(studyFqn, query, null, sessionIdUser); System.out.println("Number of files/folders linked = " + fileQueryResultLinked.getNumResults()); // Now we try to unlink them catalogManager.getFileManager().unlink(studyFqn, "myDirectory/data/", sessionIdUser); fileQueryResultLinked = catalogManager.getFileManager().get(studyFqn, query, null, sessionIdUser); assertEquals(1, fileQueryResultLinked.getNumResults()); query = new Query() .append(FileDBAdaptor.QueryParams.STUDY_UID.key(), studyUid) .append(FileDBAdaptor.QueryParams.PATH.key(), "~myDirectory/*") .append(FileDBAdaptor.QueryParams.STATUS_NAME.key(), File.FileStatus.REMOVED); QueryResult<File> fileQueryResultUnlinked = catalogManager.getFileManager().get(studyFqn, query, null, sessionIdUser); assertEquals(6, fileQueryResultUnlinked.getNumResults()); String myPath = "myDirectory/data" + AbstractManager.INTERNAL_DELIMITER + "REMOVED"; for (File file : fileQueryResultUnlinked.getResult()) { assertTrue("File name should have been modified", file.getPath().contains(myPath)); assertEquals("Status should be to REMOVED", File.FileStatus.REMOVED, file.getStatus().getName()); assertEquals("Name should not have changed", file.getName(), file.getName()); assertTrue("File uri: " + file.getUri() + " should exist", ioManager.exists(file.getUri())); } }
@Test public void deleteFolderTest3() throws CatalogException, IOException { List<File> folderFiles = new LinkedList<>(); File folder = createBasicDirectoryFileTestEnvironment(folderFiles); CatalogIOManager ioManager = catalogManager.getCatalogIOManagerFactory().get(catalogManager.getFileManager().getUri(folder)); for (File file : folderFiles) { assertTrue(ioManager.exists(catalogManager.getFileManager().getUri(file))); } catalogManager.getFileManager().delete(studyFqn, new Query(FileDBAdaptor.QueryParams.UID.key(), folder.getUid()), new ObjectMap(FileManager.SKIP_TRASH, true), sessionIdUser); Query query = new Query() .append(FileDBAdaptor.QueryParams.UID.key(), folder.getUid()) .append(FileDBAdaptor.QueryParams.STATUS_NAME.key(), File.FileStatus.PENDING_DELETE); File fileTmp = fileManager.get(studyFqn, query, QueryOptions.empty(), sessionIdUser).first(); String myPath = Paths.get(folder.getPath()) + AbstractManager.INTERNAL_DELIMITER + "DELETED"; assertTrue("Folder name should have been modified", fileTmp.getPath().contains(myPath)); assertEquals("Status should be to PENDING_DELETE", File.FileStatus.PENDING_DELETE, fileTmp.getStatus().getName()); assertEquals("Name should not have changed", folder.getName(), fileTmp.getName()); assertTrue(ioManager.exists(fileTmp.getUri())); for (File file : folderFiles) { query.put(FileDBAdaptor.QueryParams.UID.key(), file.getUid()); fileTmp = fileManager.get(studyFqn, query, QueryOptions.empty(), sessionIdUser).first(); assertTrue("Folder name should have been modified", fileTmp.getPath().contains(myPath)); assertEquals("Status should be to PENDING_DELETE", File.FileStatus.PENDING_DELETE, fileTmp.getStatus().getName()); assertEquals("Name should not have changed", file.getName(), fileTmp.getName()); assertTrue("File uri: " + fileTmp.getUri() + " should exist", ioManager.exists(fileTmp.getUri())); } }
assertEquals("Status should be to DELETED", File.FileStatus.DELETED, fileTmp.getStatus().getName()); assertEquals("Name should not have changed", folder.getName(), fileTmp.getName()); assertTrue(!ioManager.exists(fileTmp.getUri())); assertEquals("Status should be to DELETED", File.FileStatus.DELETED, fileTmp.getStatus().getName()); assertEquals("Name should not have changed", file.getName(), fileTmp.getName()); assertTrue("File uri: " + fileTmp.getUri() + " should not exist", !ioManager.exists(fileTmp.getUri()));
@Test public void deleteFolderTest2() throws CatalogException, IOException { List<File> folderFiles = new LinkedList<>(); File folder = createBasicDirectoryFileTestEnvironment(folderFiles); CatalogIOManager ioManager = catalogManager.getCatalogIOManagerFactory().get(catalogManager.getFileManager().getUri(folder)); for (File file : folderFiles) { assertTrue(ioManager.exists(catalogManager.getFileManager().getUri(file))); } catalogManager.getFileManager().delete(studyFqn, new Query(FileDBAdaptor.QueryParams.UID.key(), folder.getUid()), null, sessionIdUser); Query query = new Query() .append(FileDBAdaptor.QueryParams.UID.key(), folder.getUid()) .append(FileDBAdaptor.QueryParams.STATUS_NAME.key(), File.FileStatus.TRASHED); File fileTmp = catalogManager.getFileManager().get(studyFqn, query, QueryOptions.empty(), sessionIdUser).first(); assertEquals("Folder name should not be modified", folder.getPath(), fileTmp.getPath()); assertEquals("Status should be to TRASHED", File.FileStatus.TRASHED, fileTmp.getStatus().getName()); assertEquals("Name should not have changed", folder.getName(), fileTmp.getName()); assertTrue(ioManager.exists(fileTmp.getUri())); for (File file : folderFiles) { query.put(FileDBAdaptor.QueryParams.UID.key(), file.getUid()); fileTmp = fileManager.get(studyFqn, query, QueryOptions.empty(), sessionIdUser).first(); assertEquals("Folder name should not be modified", file.getPath(), fileTmp.getPath()); assertEquals("Status should be to TRASHED", File.FileStatus.TRASHED, fileTmp.getStatus().getName()); assertEquals("Name should not have changed", file.getName(), fileTmp.getName()); assertTrue("File uri: " + fileTmp.getUri() + " should exist", ioManager.exists(fileTmp.getUri())); } }