public CatalogIOManager getDefault() throws CatalogIOException { return get(defaultCatalogScheme); }
private void configureIOManager(Configuration properties) throws CatalogIOException { catalogIOManagerFactory = new CatalogIOManagerFactory(properties); }
private QueryResult<File> create(Study study, File file, boolean parents, String content, QueryOptions options, String sessionId) throws CatalogException { QueryResult<File> queryResult = register(study, file, parents, options, sessionId); if (file.getType() == File.Type.FILE && StringUtils.isNotEmpty(content)) { CatalogIOManager ioManager = catalogIOManagerFactory.getDefault(); // We set parents to true because the file has been successfully registered, which means the directories are already registered // in catalog ioManager.createDirectory(Paths.get(file.getUri()).getParent().toUri(), true); InputStream inputStream = new ByteArrayInputStream(content.getBytes(StandardCharsets.UTF_8)); ioManager.createFile(file.getUri(), inputStream); // Update file parameters ObjectMap params = new ObjectMap() .append(FileDBAdaptor.QueryParams.STATUS_NAME.key(), File.FileStatus.READY) .append(FileDBAdaptor.QueryParams.SIZE.key(), ioManager.getFileSize(file.getUri())); queryResult = fileDBAdaptor.update(file.getUid(), params, QueryOptions.empty()); } return queryResult; }
uriScheme = catalogIOManagerFactory.getDefaultCatalogScheme(); attributes = ParamUtils.defaultObject(attributes, HashMap<String, Object>::new); CatalogIOManager catalogIOManager = catalogIOManagerFactory.get(uriScheme);
@Test public void testComplexAdd() throws IOException, CatalogException, URISyntaxException { CatalogIOManager ioManager = catalogManager.getCatalogIOManagerFactory().getDefault(); URI fileUri = getClass().getResource("/biofiles/variant-test-file.vcf.gz").toURI(); ioManager.copyFile(fileUri, directory.resolve("file1.vcf.gz").toUri()); CatalogManagerTest.createDebugFile(directory.resolve("file1.vcf.variants.json").toString()); CatalogManagerTest.createDebugFile(directory.resolve("file1.vcf.variants.json.gz").toString()); CatalogManagerTest.createDebugFile(directory.resolve("file1.vcf.variants.json.snappy").toString()); CatalogManagerTest.createDebugFile(directory.resolve("file2.bam").toString()); CatalogManagerTest.createDebugFile(directory.resolve("file2.sam.gz").toString()); FileScanner fileScanner = new FileScanner(catalogManager); List<File> files = fileScanner.scan(folder, directory.toUri(), FileScanner.FileScannerPolicy.REPLACE, true, true, sessionIdUser); Map<String, File> map = files.stream().collect(Collectors.toMap(File::getName, (f) -> f)); assertEquals(6, files.size()); files.forEach((file) -> assertEquals(File.FileStatus.READY, file.getStatus().getName())); assertEquals(File.Bioformat.VARIANT, map.get("file1.vcf.gz").getBioformat()); assertEquals(File.Bioformat.VARIANT, map.get("file1.vcf.variants.json").getBioformat()); assertEquals(File.Bioformat.VARIANT, map.get("file1.vcf.variants.json.gz").getBioformat()); assertEquals(File.Bioformat.VARIANT, map.get("file1.vcf.variants.json.snappy").getBioformat()); assertEquals(File.Bioformat.ALIGNMENT, map.get("file2.bam").getBioformat()); assertEquals(File.Bioformat.ALIGNMENT, map.get("file2.sam.gz").getBioformat()); assertEquals(File.Format.VCF, map.get("file1.vcf.gz").getFormat()); assertEquals(File.Format.JSON, map.get("file1.vcf.variants.json").getFormat()); assertEquals(File.Format.JSON, map.get("file1.vcf.variants.json.gz").getFormat()); assertEquals(File.Format.JSON, map.get("file1.vcf.variants.json.snappy").getFormat()); assertEquals(File.Format.BAM, map.get("file2.bam").getFormat()); assertEquals(File.Format.SAM, map.get("file2.sam.gz").getFormat()); }
public CatalogIOManager get(URI uri) throws CatalogIOException { return get(uri.getScheme()); }
CatalogIOManager ioManager = catalogManager.getCatalogIOManagerFactory().getDefault(); URI tempDirectory = null; if (fileInputStream != null) {
public ExecutionOutputRecorder(CatalogManager catalogManager, String sessionId) throws CatalogIOException { this.catalogManager = catalogManager; this.ioManager = catalogManager.getCatalogIOManagerFactory().get("file"); this.sessionId = sessionId; }
catalogIOManagerFactory.getDefault().createUser(user.getId()); QueryResult<User> queryResult = userDBAdaptor.insert(user, options); auditManager.recordCreation(AuditRecord.Resource.user, id, id, queryResult.first(), null, null); if (!userDBAdaptor.exists(user.getId())) { logger.error("ERROR! DELETING USER! " + user.getId()); catalogIOManagerFactory.getDefault().deleteUser(user.getId());
protected void outdirMustBeEmpty(Path outdir, ObjectMap options) throws CatalogIOException, StorageEngineException { if (!isCatalogPathDefined(options)) { // This restriction is only necessary if the output files are going to be moved to Catalog. // If CATALOG_PATH is NOT defined, output does not need to be empty. return; } List<URI> uris = catalogManager.getCatalogIOManagerFactory().get(outdir.toUri()).listFiles(outdir.toUri()); if (!uris.isEmpty()) { // Only allow stdout and stderr files for (URI uri : uris) { // Obtain the extension int i = uri.toString().lastIndexOf("."); if (i <= 0) { throw new StorageEngineException("Unable to execute storage operation. Outdir '" + outdir + "' must be empty!"); } String extension = uri.toString().substring(i); // If the extension is not one of the ones created by the daemons, throw the exception. if (!ERR_LOG_EXTENSION.equalsIgnoreCase(extension) && !OUT_LOG_EXTENSION.equalsIgnoreCase(extension)) { throw new StorageEngineException("Unable to execute storage operation. Outdir '" + outdir + "' must be empty!"); } } } }
catalogIOManagerFactory.getDefault().createProject(userId, Long.toString(project.getUid())); } catch (CatalogIOException e) { try {
public IndexDaemon(int interval, String sessionId, CatalogManager catalogManager, String appHome) throws URISyntaxException, CatalogIOException, CatalogDBException { super(interval, sessionId, catalogManager); this.binHome = appHome + "/bin/"; URI uri = UriUtils.createUri(catalogManager.getConfiguration().getTempJobsDir()); this.tempJobFolder = Paths.get(uri.getPath()); this.catalogIOManager = catalogManager.getCatalogIOManagerFactory().get("file"); this.jobDBAdaptor = dbAdaptorFactory.getCatalogJobDBAdaptor(); // this.variantIndexOutputRecorder = new VariantIndexOutputRecorder(catalogManager, catalogIOManager, sessionId); }
catalogManager.getCatalogIOManagerFactory().getDefault().createDirectory(studyUri.resolve("data/test/folder/"), true); Path filePath = CatalogManagerTest.createDebugFile(studyUri.resolve("data/test/folder/").resolve("file_scanner_test_file.txt") .getPath()).toPath();
public ExecutionDaemon(int interval, String sessionId, CatalogManager catalogManager, String appHome) throws CatalogDBException, URISyntaxException, CatalogIOException { super(interval, sessionId, catalogManager); URI uri = UriUtils.createUri(catalogManager.getConfiguration().getTempJobsDir()); this.tempJobFolder = Paths.get(uri.getPath()); this.catalogIOManager = catalogManager.getCatalogIOManagerFactory().get("file"); this.binAnalysis = appHome + "/bin/opencga-analysis.sh"; this.jobDBAdaptor = dbAdaptorFactory.getCatalogJobDBAdaptor(); }
@Test public void testReplaceExisting() throws IOException, CatalogException { // Create and register file1.txt and s/file2.txt File file = catalogManager.getFileManager().upload(study.getFqn(), CatalogManagerTest.createDebugFile().toURI(), new File().setPath(folder.getPath() + "file1.txt"), false, false, true, true, sessionIdUser).first(); catalogManager.getFileManager().upload(study.getFqn(), CatalogManagerTest.createDebugFile().toURI(), new File().setPath(folder.getPath() + "s/file2.txt"), false, true, true, true, sessionIdUser).first(); // Create same file structure, and replace CatalogManagerTest.createDebugFile(directory.resolve("file1.txt").toString()); Files.createDirectory(directory.resolve("s/")); CatalogManagerTest.createDebugFile(directory.resolve("s/file2.txt").toString()); FileScanner fileScanner = new FileScanner(catalogManager); fileScanner.scan(folder, directory.toUri(), FileScanner.FileScannerPolicy.REPLACE, true, true, sessionIdUser); File replacedFile = catalogManager.getFileManager().get(study.getFqn(), file.getPath(), null, sessionIdUser).first(); assertEquals(File.FileStatus.READY, replacedFile.getStatus().getName()); assertEquals(file.getUid(), replacedFile.getUid()); assertNotEquals(replacedFile.getChecksum(), file.getChecksum()); assertEquals(replacedFile.getChecksum(), catalogManager.getCatalogIOManagerFactory().getDefault().calculateChecksum(replacedFile.getUri())); }
@Deprecated public URI getUri(long studyId, String filePath) throws CatalogException { ParamUtils.checkObj(filePath, "filePath"); List<File> parents = getParents(false, INCLUDE_FILE_URI_PATH, filePath, studyId).getResult(); for (File parent : parents) { if (parent.getUri() != null) { if (parent.isExternal()) { throw new CatalogException("Cannot upload files to an external folder"); } String relativePath = filePath.replaceFirst(parent.getPath(), ""); return Paths.get(parent.getUri()).resolve(relativePath).toUri(); } } URI studyUri = getStudyUri(studyId); return filePath.isEmpty() ? studyUri : catalogIOManagerFactory.get(studyUri).getFileUri(studyUri, filePath); }
URI studyUri = study.getUri(); CatalogIOManager ioManager = catalogManager.getCatalogIOManagerFactory().get(studyUri); Map<String, URI> linkedFolders = new HashMap<>(); linkedFolders.put("", studyUri);
CatalogIOManager ioManager = catalogManager.getCatalogIOManagerFactory().get("file");
/** * Get a ObjectMap with some fields if they have been modified. * size * modificationDate * checksum * uri * * @param file file * @param fileUri If null, calls to getFileUri() * <p> * TODO: Lazy checksum: Only calculate checksum if the size has changed. * @param calculateChecksum Calculate checksum to check if have changed * @return ObjectMap ObjectMap * @throws CatalogException CatalogException */ public ObjectMap getModifiedFileAttributes(File file, URI fileUri, boolean calculateChecksum) throws CatalogException { if (fileUri == null) { fileUri = catalogManager.getFileManager().getUri(file); } String checksum = null; if (calculateChecksum) { checksum = catalogManager.getCatalogIOManagerFactory().get(fileUri).calculateChecksum(fileUri); } return getModifiedFileAttributes(file, checksum, fileUri, null); }
fileUri = catalogManager.getFileManager().getUri(file); CatalogIOManager catalogIOManager = catalogManager.getCatalogIOManagerFactory().get(fileUri);