public CatalogIOManager getDefault() throws CatalogIOException { return get(defaultCatalogScheme); }
public CatalogIOManager get(URI uri) throws CatalogIOException { return get(uri.getScheme()); }
public ExecutionOutputRecorder(CatalogManager catalogManager, String sessionId) throws CatalogIOException { this.catalogManager = catalogManager; this.ioManager = catalogManager.getCatalogIOManagerFactory().get("file"); this.sessionId = sessionId; }
protected void outdirMustBeEmpty(Path outdir, ObjectMap options) throws CatalogIOException, StorageEngineException { if (!isCatalogPathDefined(options)) { // This restriction is only necessary if the output files are going to be moved to Catalog. // If CATALOG_PATH is NOT defined, output does not need to be empty. return; } List<URI> uris = catalogManager.getCatalogIOManagerFactory().get(outdir.toUri()).listFiles(outdir.toUri()); if (!uris.isEmpty()) { // Only allow stdout and stderr files for (URI uri : uris) { // Obtain the extension int i = uri.toString().lastIndexOf("."); if (i <= 0) { throw new StorageEngineException("Unable to execute storage operation. Outdir '" + outdir + "' must be empty!"); } String extension = uri.toString().substring(i); // If the extension is not one of the ones created by the daemons, throw the exception. if (!ERR_LOG_EXTENSION.equalsIgnoreCase(extension) && !OUT_LOG_EXTENSION.equalsIgnoreCase(extension)) { throw new StorageEngineException("Unable to execute storage operation. Outdir '" + outdir + "' must be empty!"); } } } }
public IndexDaemon(int interval, String sessionId, CatalogManager catalogManager, String appHome) throws URISyntaxException, CatalogIOException, CatalogDBException { super(interval, sessionId, catalogManager); this.binHome = appHome + "/bin/"; URI uri = UriUtils.createUri(catalogManager.getConfiguration().getTempJobsDir()); this.tempJobFolder = Paths.get(uri.getPath()); this.catalogIOManager = catalogManager.getCatalogIOManagerFactory().get("file"); this.jobDBAdaptor = dbAdaptorFactory.getCatalogJobDBAdaptor(); // this.variantIndexOutputRecorder = new VariantIndexOutputRecorder(catalogManager, catalogIOManager, sessionId); }
public ExecutionDaemon(int interval, String sessionId, CatalogManager catalogManager, String appHome) throws CatalogDBException, URISyntaxException, CatalogIOException { super(interval, sessionId, catalogManager); URI uri = UriUtils.createUri(catalogManager.getConfiguration().getTempJobsDir()); this.tempJobFolder = Paths.get(uri.getPath()); this.catalogIOManager = catalogManager.getCatalogIOManagerFactory().get("file"); this.binAnalysis = appHome + "/bin/opencga-analysis.sh"; this.jobDBAdaptor = dbAdaptorFactory.getCatalogJobDBAdaptor(); }
@Deprecated public URI getUri(long studyId, String filePath) throws CatalogException { ParamUtils.checkObj(filePath, "filePath"); List<File> parents = getParents(false, INCLUDE_FILE_URI_PATH, filePath, studyId).getResult(); for (File parent : parents) { if (parent.getUri() != null) { if (parent.isExternal()) { throw new CatalogException("Cannot upload files to an external folder"); } String relativePath = filePath.replaceFirst(parent.getPath(), ""); return Paths.get(parent.getUri()).resolve(relativePath).toUri(); } } URI studyUri = getStudyUri(studyId); return filePath.isEmpty() ? studyUri : catalogIOManagerFactory.get(studyUri).getFileUri(studyUri, filePath); }
/** * Get a ObjectMap with some fields if they have been modified. * size * modificationDate * checksum * uri * * @param file file * @param fileUri If null, calls to getFileUri() * <p> * TODO: Lazy checksum: Only calculate checksum if the size has changed. * @param calculateChecksum Calculate checksum to check if have changed * @return ObjectMap ObjectMap * @throws CatalogException CatalogException */ public ObjectMap getModifiedFileAttributes(File file, URI fileUri, boolean calculateChecksum) throws CatalogException { if (fileUri == null) { fileUri = catalogManager.getFileManager().getUri(file); } String checksum = null; if (calculateChecksum) { checksum = catalogManager.getCatalogIOManagerFactory().get(fileUri).calculateChecksum(fileUri); } return getModifiedFileAttributes(file, checksum, fileUri, null); }
public DataInputStream download(String studyStr, String fileStr, int start, int limit, QueryOptions options, String sessionId) throws CatalogException { MyResource<File> resource = getUid(fileStr, studyStr, sessionId); authorizationManager.checkFilePermission(resource.getStudy().getUid(), resource.getResource().getUid(), resource.getUser(), FileAclEntry.FilePermissions.DOWNLOAD); URI fileUri = getUri(resource.getResource()); return catalogIOManagerFactory.get(fileUri).getFileObject(fileUri, start, limit); }
public DataInputStream grep(String studyStr, String fileStr, String pattern, QueryOptions options, String sessionId) throws CatalogException { MyResource<File> resource = getUid(fileStr, studyStr, sessionId); authorizationManager.checkFilePermission(resource.getStudy().getUid(), resource.getResource().getUid(), resource.getUser(), FileAclEntry.FilePermissions.VIEW); URI fileUri = getUri(resource.getResource()); boolean ignoreCase = options.getBoolean("ignoreCase"); boolean multi = options.getBoolean("multi"); return catalogIOManagerFactory.get(fileUri).getGrepFileObject(fileUri, pattern, ignoreCase, multi); }
@Deprecated private File linkFile(File file, boolean calculateChecksum, URI externalUri, String sessionId) throws CatalogException { String checksum = null; CatalogIOManager ioManager = catalogManager.getCatalogIOManagerFactory().get(externalUri); if (ioManager.isDirectory(externalUri)) { throw new CatalogIOException("Can't link file '" + file.getPath() + "' with a folder uri " + externalUri); } if (calculateChecksum) { try { checksum = ioManager.calculateChecksum(externalUri); } catch (CatalogIOException e) { e.printStackTrace(); } } ObjectMap objectMap = new ObjectMap("uri", externalUri); objectMap.put(FileDBAdaptor.QueryParams.STATUS_NAME.key(), File.FileStatus.READY); updateFileAttributes(file, checksum, externalUri, objectMap, sessionId); return catalogManager.getFileManager().get(file.getUid(), null, sessionId).first(); }
public void installCatalogDB(String secretKey, String password) throws CatalogException { if (existsCatalogDB()) { throw new CatalogException("Nothing to install. There already exists a catalog database"); } ParamUtils.checkParameter(secretKey, "secretKey"); ParamUtils.checkParameter(password, "password"); configuration.getAdmin().setPassword(password); configuration.getAdmin().setSecretKey(secretKey); // Check jobs folder is empty URI jobsURI; try { jobsURI = UriUtils.createDirectoryUri(configuration.getTempJobsDir()); } catch (URISyntaxException e) { throw new CatalogException("Failed to create a directory URI from " + configuration.getTempJobsDir()); } CatalogIOManager ioManager = getCatalogIOManagerFactory().get(jobsURI); if (!ioManager.isDirectory(jobsURI) || CollectionUtils.isNotEmpty(ioManager.listFiles(jobsURI))) { throw new CatalogException("Cannot install openCGA. Jobs folder is not empty.\nPlease, empty it first."); } catalogDBAdaptorFactory.installCatalogDB(configuration); }
@Test public void deleteFilesTest2() throws CatalogException, IOException { QueryResult<File> queryResult = catalogManager.getFileManager().create(studyFqn, File.Type.FILE, File.Format.PLAIN, File.Bioformat.NONE, "my.txt", null, "", new File.FileStatus(File.FileStatus.STAGE), 0, -1, null, -1, null, null, false, null, null, userSessionId); new FileUtils(catalogManager).upload(new ByteArrayInputStream(StringUtils.randomString(200) .getBytes()), queryResult.first(), userSessionId, false, false, true); File file = catalogManager.getFileManager().get(studyFqn, queryResult.first().getPath(), null, userSessionId).first(); CatalogIOManager ioManager = catalogManager.getCatalogIOManagerFactory().get(file.getUri()); assertTrue(ioManager.exists(file.getUri())); catalogManager.getFileManager().delete(studyFqn, new Query(FileDBAdaptor.QueryParams.UID.key(), file.getUid()), null, userSessionId); assertTrue(ioManager.exists(file.getUri())); // catalogFileUtils.delete(file.getId(), userSessionId); // assertTrue(!ioManager.exists(catalogManager.getFileUri(catalogManager.getFile(file.getId(), userSessionId).first()))); }
@Test public void testCreateFile() throws CatalogException, IOException { String content = "This is the content\tof the file"; try { fileManager.create(studyFqn3, File.Type.FILE, File.Format.UNKNOWN, File.Bioformat.UNKNOWN, "data/test/myTest/myFile.txt", null, null, new File.FileStatus(File.FileStatus.READY), 0, -1, null, -1, null, null, false, "This is the content\tof the file", null, sessionIdUser2); fail("An error should be raised because parents is false"); } catch (CatalogException e) { System.out.println("Correct"); } QueryResult<File> fileQueryResult = fileManager.create(studyFqn3, File.Type.FILE, File.Format.UNKNOWN, File.Bioformat.UNKNOWN, "data/test/myTest/myFile.txt", null, null, new File.FileStatus(File.FileStatus.READY), 0, -1, null, -1, null, null, true, content, null, sessionIdUser2); CatalogIOManager ioManager = catalogManager.getCatalogIOManagerFactory().get(fileQueryResult.first().getUri()); assertTrue(ioManager.exists(fileQueryResult.first().getUri())); DataInputStream fileObject = ioManager.getFileObject(fileQueryResult.first().getUri(), -1, -1); assertEquals(content, fileObject.readLine()); }
@Test public void testUnlinkFolder() throws CatalogException, IOException { URI uri = Paths.get(getStudyURI()).resolve("data").toUri(); link(uri, "myDirectory", studyFqn, new ObjectMap("parents", true), sessionIdUser); CatalogIOManager ioManager = catalogManager.getCatalogIOManagerFactory().get(uri); Query query = new Query() .append(FileDBAdaptor.QueryParams.STUDY_UID.key(), studyUid) .append(FileDBAdaptor.QueryParams.PATH.key(), "~myDirectory/*") .append(FileDBAdaptor.QueryParams.STATUS_NAME.key(), File.FileStatus.READY); QueryResult<File> fileQueryResultLinked = catalogManager.getFileManager().get(studyFqn, query, null, sessionIdUser); System.out.println("Number of files/folders linked = " + fileQueryResultLinked.getNumResults()); // Now we try to unlink them catalogManager.getFileManager().unlink(studyFqn, "myDirectory/data/", sessionIdUser); fileQueryResultLinked = catalogManager.getFileManager().get(studyFqn, query, null, sessionIdUser); assertEquals(1, fileQueryResultLinked.getNumResults()); query = new Query() .append(FileDBAdaptor.QueryParams.STUDY_UID.key(), studyUid) .append(FileDBAdaptor.QueryParams.PATH.key(), "~myDirectory/*") .append(FileDBAdaptor.QueryParams.STATUS_NAME.key(), File.FileStatus.REMOVED); QueryResult<File> fileQueryResultUnlinked = catalogManager.getFileManager().get(studyFqn, query, null, sessionIdUser); assertEquals(6, fileQueryResultUnlinked.getNumResults()); String myPath = "myDirectory/data" + AbstractManager.INTERNAL_DELIMITER + "REMOVED"; for (File file : fileQueryResultUnlinked.getResult()) { assertTrue("File name should have been modified", file.getPath().contains(myPath)); assertEquals("Status should be to REMOVED", File.FileStatus.REMOVED, file.getStatus().getName()); assertEquals("Name should not have changed", file.getName(), file.getName()); assertTrue("File uri: " + file.getUri() + " should exist", ioManager.exists(file.getUri())); } }
@Test public void deleteFoldersTest() throws CatalogException, IOException { List<File> folderFiles = new LinkedList<>(); File folder = prepareFiles(folderFiles); CatalogIOManager ioManager = catalogManager.getCatalogIOManagerFactory().get(catalogManager.getFileManager().getUri(folder)); for (File file : folderFiles) { assertTrue(ioManager.exists(catalogManager.getFileManager().getUri(file))); } catalogManager.getFileManager().delete(studyFqn, new Query(FileDBAdaptor.QueryParams.UID.key(), folder.getUid()), null, userSessionId); Query query = new Query() .append(FileDBAdaptor.QueryParams.UID.key(), folder.getUid()) .append(FileDBAdaptor.QueryParams.STATUS_NAME.key(), File.FileStatus.TRASHED); QueryResult<File> fileQueryResult = catalogManager.getFileManager().get(studyFqn, query, QueryOptions.empty(), userSessionId); assertTrue(ioManager.exists(fileQueryResult.first().getUri())); for (File file : folderFiles) { assertTrue("File uri: " + file.getUri() + " should exist", ioManager.exists(file.getUri())); } }
@Test public void deleteFolderTest3() throws CatalogException, IOException { List<File> folderFiles = new LinkedList<>(); File folder = createBasicDirectoryFileTestEnvironment(folderFiles); CatalogIOManager ioManager = catalogManager.getCatalogIOManagerFactory().get(catalogManager.getFileManager().getUri(folder)); for (File file : folderFiles) { assertTrue(ioManager.exists(catalogManager.getFileManager().getUri(file))); } catalogManager.getFileManager().delete(studyFqn, new Query(FileDBAdaptor.QueryParams.UID.key(), folder.getUid()), new ObjectMap(FileManager.SKIP_TRASH, true), sessionIdUser); Query query = new Query() .append(FileDBAdaptor.QueryParams.UID.key(), folder.getUid()) .append(FileDBAdaptor.QueryParams.STATUS_NAME.key(), File.FileStatus.PENDING_DELETE); File fileTmp = fileManager.get(studyFqn, query, QueryOptions.empty(), sessionIdUser).first(); String myPath = Paths.get(folder.getPath()) + AbstractManager.INTERNAL_DELIMITER + "DELETED"; assertTrue("Folder name should have been modified", fileTmp.getPath().contains(myPath)); assertEquals("Status should be to PENDING_DELETE", File.FileStatus.PENDING_DELETE, fileTmp.getStatus().getName()); assertEquals("Name should not have changed", folder.getName(), fileTmp.getName()); assertTrue(ioManager.exists(fileTmp.getUri())); for (File file : folderFiles) { query.put(FileDBAdaptor.QueryParams.UID.key(), file.getUid()); fileTmp = fileManager.get(studyFqn, query, QueryOptions.empty(), sessionIdUser).first(); assertTrue("Folder name should have been modified", fileTmp.getPath().contains(myPath)); assertEquals("Status should be to PENDING_DELETE", File.FileStatus.PENDING_DELETE, fileTmp.getStatus().getName()); assertEquals("Name should not have changed", file.getName(), fileTmp.getName()); assertTrue("File uri: " + fileTmp.getUri() + " should exist", ioManager.exists(fileTmp.getUri())); } }
File folder = createBasicDirectoryFileTestEnvironment(folderFiles); CatalogIOManager ioManager = catalogManager.getCatalogIOManagerFactory().get(catalogManager.getFileManager().getUri(folder)); for (File file : folderFiles) { assertTrue(ioManager.exists(catalogManager.getFileManager().getUri(file)));
@Test public void deleteFolderTest() throws CatalogException, IOException { List<File> folderFiles = new LinkedList<>(); File folder = createBasicDirectoryFileTestEnvironment(folderFiles); CatalogIOManager ioManager = catalogManager.getCatalogIOManagerFactory().get(catalogManager.getFileManager().getUri(folder)); for (File file : folderFiles) { assertTrue(ioManager.exists(catalogManager.getFileManager().getUri(file))); } catalogManager.getFileManager().create(studyFqn, File.Type.FILE, File.Format.PLAIN, File.Bioformat.NONE, "folder/subfolder/subsubfolder/my_staged.txt", null, null, new File.FileStatus(File.FileStatus.STAGE), (long) 0, (long) -1, null, (long) -1, null, null, true, null, null, sessionIdUser).first(); WriteResult deleteResult = catalogManager.getFileManager().delete(studyFqn, new Query(FileDBAdaptor.QueryParams.UID.key(), folder.getUid()), null, sessionIdUser); assertEquals(0, deleteResult.getNumModified()); File fileTmp = catalogManager.getFileManager().get(studyFqn, folder.getPath(), null, sessionIdUser).first(); assertEquals("Folder name should not be modified", folder.getPath(), fileTmp.getPath()); assertTrue(ioManager.exists(fileTmp.getUri())); for (File file : folderFiles) { fileTmp = catalogManager.getFileManager().get(studyFqn, file.getPath(), null, sessionIdUser).first(); assertEquals("File name should not be modified", file.getPath(), fileTmp.getPath()); assertTrue("File uri: " + fileTmp.getUri() + " should exist", ioManager.exists(fileTmp.getUri())); } }
@Test public void deleteFolderTest2() throws CatalogException, IOException { List<File> folderFiles = new LinkedList<>(); File folder = createBasicDirectoryFileTestEnvironment(folderFiles); CatalogIOManager ioManager = catalogManager.getCatalogIOManagerFactory().get(catalogManager.getFileManager().getUri(folder)); for (File file : folderFiles) { assertTrue(ioManager.exists(catalogManager.getFileManager().getUri(file))); } catalogManager.getFileManager().delete(studyFqn, new Query(FileDBAdaptor.QueryParams.UID.key(), folder.getUid()), null, sessionIdUser); Query query = new Query() .append(FileDBAdaptor.QueryParams.UID.key(), folder.getUid()) .append(FileDBAdaptor.QueryParams.STATUS_NAME.key(), File.FileStatus.TRASHED); File fileTmp = catalogManager.getFileManager().get(studyFqn, query, QueryOptions.empty(), sessionIdUser).first(); assertEquals("Folder name should not be modified", folder.getPath(), fileTmp.getPath()); assertEquals("Status should be to TRASHED", File.FileStatus.TRASHED, fileTmp.getStatus().getName()); assertEquals("Name should not have changed", folder.getName(), fileTmp.getName()); assertTrue(ioManager.exists(fileTmp.getUri())); for (File file : folderFiles) { query.put(FileDBAdaptor.QueryParams.UID.key(), file.getUid()); fileTmp = fileManager.get(studyFqn, query, QueryOptions.empty(), sessionIdUser).first(); assertEquals("Folder name should not be modified", file.getPath(), fileTmp.getPath()); assertEquals("Status should be to TRASHED", File.FileStatus.TRASHED, fileTmp.getStatus().getName()); assertEquals("Name should not have changed", file.getName(), fileTmp.getName()); assertTrue("File uri: " + fileTmp.getUri() + " should exist", ioManager.exists(fileTmp.getUri())); } }