public List<Document> convertFilesToDocument(List<File> fileList) { if (fileList == null || fileList.isEmpty()) { return Collections.emptyList(); } List<Document> files = new ArrayList(fileList.size()); for (File file : fileList) { long fileId = file != null ? (file.getUid() == 0 ? -1L : file.getUid()) : -1L; if (fileId > 0) { files.add(new Document("uid", fileId)); } } return files; } }
@Deprecated public void setModificationDate(String studyStr, String fileId, String date, String sessionId) throws CatalogException { MyResource<File> resource = getUid(fileId, studyStr, sessionId); String userId = resource.getUser(); long studyId = resource.getStudy().getUid(); authorizationManager.checkFilePermission(studyId, resource.getResource().getUid(), userId, FileAclEntry.FilePermissions.WRITE); ObjectMap parameters = new ObjectMap(FileDBAdaptor.QueryParams.MODIFICATION_DATE.key(), date); fileDBAdaptor.update(resource.getResource().getUid(), parameters, QueryOptions.empty()); auditManager.recordUpdate(AuditRecord.Resource.file, resource.getResource().getUid(), userId, parameters, null, null); }
public URI getUri(File file) throws CatalogException { ParamUtils.checkObj(file, "File"); if (file.getUri() != null) { return file.getUri(); } else { QueryResult<File> fileQueryResult = fileDBAdaptor.get(file.getUid(), INCLUDE_STUDY_URI); if (fileQueryResult.getNumResults() == 0) { throw new CatalogException("File " + file.getUid() + " not found"); } return fileQueryResult.first().getUri(); } }
public void setUri(String studyStr, String fileId, String uri, String sessionId) throws CatalogException { MyResource<File> resource = getUid(fileId, studyStr, sessionId); String userId = resource.getUser(); long studyId = resource.getStudy().getUid(); authorizationManager.checkFilePermission(studyId, resource.getResource().getUid(), userId, FileAclEntry.FilePermissions.WRITE); ObjectMap parameters = new ObjectMap(FileDBAdaptor.QueryParams.URI.key(), uri); fileDBAdaptor.update(resource.getResource().getUid(), parameters, QueryOptions.empty()); auditManager.recordUpdate(AuditRecord.Resource.file, resource.getResource().getUid(), userId, parameters, null, null); }
public void setDiskUsage(String studyStr, String fileId, long size, String sessionId) throws CatalogException { MyResource<File> resource = getUid(fileId, studyStr, sessionId); authorizationManager.checkFilePermission(resource.getStudy().getUid(), resource.getResource().getUid(), resource.getUser(), FileAclEntry.FilePermissions.WRITE); ObjectMap parameters = new ObjectMap(FileDBAdaptor.QueryParams.SIZE.key(), size); fileDBAdaptor.update(resource.getResource().getUid(), parameters, QueryOptions.empty()); auditManager.recordUpdate(AuditRecord.Resource.file, resource.getResource().getUid(), resource.getUser(), parameters, null, null); }
public void setFileIndex(String studyStr, String fileId, FileIndex index, String sessionId) throws CatalogException { MyResource<File> resource = getUid(fileId, studyStr, sessionId); authorizationManager.checkFilePermission(resource.getStudy().getUid(), resource.getResource().getUid(), resource.getUser(), FileAclEntry.FilePermissions.WRITE); ObjectMap parameters = new ObjectMap(FileDBAdaptor.QueryParams.INDEX.key(), index); fileDBAdaptor.update(resource.getResource().getUid(), parameters, QueryOptions.empty()); auditManager.recordUpdate(AuditRecord.Resource.file, resource.getResource().getUid(), resource.getUser(), parameters, null, null); }
@Before public void before() throws Exception { inputFile1 = create(studyId, getResourceUri("platinum/1K.end.platinum-genomes-vcf-NA12877_S1.genome.vcf.gz", "platinum_1/1K.end.platinum-genomes-vcf-NA12877_S1.genome.vcf.gz"), "data/platinum_1/"); inputFile2 = create(studyId, getResourceUri("platinum/1K.end.platinum-genomes-vcf-NA12877_S1.genome.vcf.gz", "platinum_2/1K.end.platinum-genomes-vcf-NA12877_S1.genome.vcf.gz"), "data/platinum_2/"); System.out.println("inputFile1 = " + inputFile1.getUid()); System.out.println("inputFile2 = " + inputFile2.getUid()); }
private long getTransformedFileIdFromOriginal(File file) throws CatalogException { long transformedFile = file.getIndex() != null && file.getIndex().getTransformedFile() != null ? file.getIndex().getTransformedFile().getId() : -1; if (transformedFile == -1) { logger.error("This code should never be executed. Every vcf file containing the transformed status should have" + " a registered transformed file"); throw new CatalogException("Internal error. No transformed file could be found for file " + file.getUid()); } return transformedFile; }
@Test public void testIndexFromAvro() throws Exception { URI uri = getClass().getResource("/biofiles/variant-test-file.vcf.gz").toURI(); File file = fileManager.link(studyFqn, uri, "data", null, sessionIdUser).first(); fileManager.create(studyFqn, File.Type.FILE, File.Format.AVRO, null, "data/variant-test-file.vcf.gz.variants.avro.gz", "", "description", new File.FileStatus(File.FileStatus.READY), 0, -1, Collections.emptyList(), -1, Collections.emptyMap(), Collections.emptyMap(), true, "asdf", new QueryOptions(), sessionIdUser); fileManager.link(studyFqn, getClass().getResource("/biofiles/variant-test-file.vcf.gz.file.json.gz").toURI(), "data", null, sessionIdUser).first(); Job job = fileManager.index(studyFqn, Collections.singletonList("variant-test-file.vcf.gz.variants.avro.gz"), "VCF", null, sessionIdUser).first(); assertEquals(file.getUid(), job.getInput().get(0).getUid()); }
@Override public long getId(long studyId, String path) throws CatalogDBException { Query query = new Query(QueryParams.STUDY_UID.key(), studyId).append(QueryParams.PATH.key(), path); QueryOptions options = new QueryOptions(MongoDBCollection.INCLUDE, PRIVATE_UID); QueryResult<File> fileQueryResult = get(query, options); return fileQueryResult.getNumTotalResults() == 1 ? fileQueryResult.getResult().get(0).getUid() : -1; }
@Test public void getFileTest() throws CatalogDBException { File file = user3.getProjects().get(0).getStudies().get(0).getFiles().get(0); QueryResult<File> fileQueryResult = catalogFileDBAdaptor.get(file.getUid(), null); System.out.println(fileQueryResult); try { System.out.println(catalogFileDBAdaptor.get(-1, null)); fail("Expected \"FileId not found\" exception"); } catch (CatalogDBException e) { System.out.println(e); } }
@Test public void testIndex() throws Exception { URI uri = getClass().getResource("/biofiles/variant-test-file.vcf.gz").toURI(); File file = fileManager.link(studyFqn, uri, "", null, sessionIdUser).first(); assertEquals(4, file.getSamples().size()); assertEquals(File.Format.VCF, file.getFormat()); assertEquals(File.Bioformat.VARIANT, file.getBioformat()); Job job = fileManager.index(studyFqn, Collections.singletonList(file.getName()), "VCF", null, sessionIdUser).first(); assertEquals(file.getUid(), job.getInput().get(0).getUid()); }
public DataInputStream download(String studyStr, String fileStr, int start, int limit, QueryOptions options, String sessionId) throws CatalogException { MyResource<File> resource = getUid(fileStr, studyStr, sessionId); authorizationManager.checkFilePermission(resource.getStudy().getUid(), resource.getResource().getUid(), resource.getUser(), FileAclEntry.FilePermissions.DOWNLOAD); URI fileUri = getUri(resource.getResource()); return catalogIOManagerFactory.get(fileUri).getFileObject(fileUri, start, limit); }
@Test public void testDeleteExisting() throws IOException, CatalogException { QueryResult<File> queryResult = catalogManager.getFileManager().upload(study.getFqn(), CatalogManagerTest.createDebugFile().toURI(), new File().setPath(folder.getPath() + "file1.txt"), false, false, sessionIdUser); File file = queryResult.first(); CatalogManagerTest.createDebugFile(directory.resolve("file1.txt").toString()); List<File> files = new FileScanner(catalogManager).scan(folder, directory.toUri(), FileScanner.FileScannerPolicy.DELETE, false, true, sessionIdUser); files.forEach((File f) -> assertFalse(f.getAttributes().containsKey("checksum"))); assertEquals(File.FileStatus.DELETED, getFile(file.getUid()).getStatus().getName()); }
@Override public Document convertToStorageType(File file, List<VariableSet> variableSetList) { Document document = super.convertToStorageType(file, variableSetList); document.remove(SampleDBAdaptor.QueryParams.ANNOTATION_SETS.key()); document.put("uid", file.getUid()); document.put("studyUid", file.getStudyUid()); long jobId = file.getJob() != null ? (file.getJob().getUid() == 0 ? -1L : file.getJob().getUid()) : -1L; document.put("job", new Document("uid", jobId)); document.put("samples", convertSamples(file.getSamples())); document.put(FileMongoDBAdaptor.REVERSE_NAME, StringUtils.reverse(file.getName())); return document; }
public DataInputStream grep(String studyStr, String fileStr, String pattern, QueryOptions options, String sessionId) throws CatalogException { MyResource<File> resource = getUid(fileStr, studyStr, sessionId); authorizationManager.checkFilePermission(resource.getStudy().getUid(), resource.getResource().getUid(), resource.getUser(), FileAclEntry.FilePermissions.VIEW); URI fileUri = getUri(resource.getResource()); boolean ignoreCase = options.getBoolean("ignoreCase"); boolean multi = options.getBoolean("multi"); return catalogIOManagerFactory.get(fileUri).getGrepFileObject(fileUri, pattern, ignoreCase, multi); }
@Before public void setUp() throws Exception { scm = new StudyConfigurationManager(new DummyProjectMetadataAdaptor(), new DummyStudyConfigurationAdaptor(), new DummyVariantFileMetadataDBAdaptor()); Study study = catalogManager.getStudyManager().get(studyId, null, sessionId).first(); StudyConfiguration studyConfigurationToReturn = new StudyConfiguration((int) study.getUid(), "user@p1:s1"); studyConfigurationToReturn.setIndexedFiles(indexedFiles); for (File file : files) { if (indexedFiles.contains(((int) file.getUid()))) { studyConfigurationToReturn.getFileIds().put(file.getName(), ((int) file.getUid())); } } scm.updateStudyConfiguration(studyConfigurationToReturn, null); // checkStudyConfiguration(study, studyConfigurationToReturn); }
@Test public void testGetFileParents1() throws CatalogException { long fileId; QueryResult<File> fileParents; fileId = catalogManager.getFileManager().getUid("data/test/folder/", studyFqn, sessionIdUser).getResource().getUid(); fileParents = catalogManager.getFileManager().getParents(fileId, null, sessionIdUser); assertEquals(4, fileParents.getNumResults()); assertEquals("", fileParents.getResult().get(0).getPath()); assertEquals("data/", fileParents.getResult().get(1).getPath()); assertEquals("data/test/", fileParents.getResult().get(2).getPath()); assertEquals("data/test/folder/", fileParents.getResult().get(3).getPath()); }
@Test public void testGetFileParents2() throws CatalogException { long fileId; QueryResult<File> fileParents; fileId = catalogManager.getFileManager().getUid("data/test/folder/test_1K.txt.gz", studyFqn, sessionIdUser) .getResource().getUid(); fileParents = catalogManager.getFileManager().getParents(fileId, null, sessionIdUser); assertEquals(5, fileParents.getNumResults()); assertEquals("", fileParents.getResult().get(0).getPath()); assertEquals("data/", fileParents.getResult().get(1).getPath()); assertEquals("data/test/", fileParents.getResult().get(2).getPath()); assertEquals("data/test/folder/", fileParents.getResult().get(3).getPath()); assertEquals("data/test/folder/test_1K.txt.gz", fileParents.getResult().get(4).getPath()); }
@Test public void deleteFilesTest2() throws CatalogException, IOException { QueryResult<File> queryResult = catalogManager.getFileManager().create(studyFqn, File.Type.FILE, File.Format.PLAIN, File.Bioformat.NONE, "my.txt", null, "", new File.FileStatus(File.FileStatus.STAGE), 0, -1, null, -1, null, null, false, null, null, userSessionId); new FileUtils(catalogManager).upload(new ByteArrayInputStream(StringUtils.randomString(200) .getBytes()), queryResult.first(), userSessionId, false, false, true); File file = catalogManager.getFileManager().get(studyFqn, queryResult.first().getPath(), null, userSessionId).first(); CatalogIOManager ioManager = catalogManager.getCatalogIOManagerFactory().get(file.getUri()); assertTrue(ioManager.exists(file.getUri())); catalogManager.getFileManager().delete(studyFqn, new Query(FileDBAdaptor.QueryParams.UID.key(), file.getUid()), null, userSessionId); assertTrue(ioManager.exists(file.getUri())); // catalogFileUtils.delete(file.getId(), userSessionId); // assertTrue(!ioManager.exists(catalogManager.getFileUri(catalogManager.getFile(file.getId(), userSessionId).first()))); }