public QueryResult<File> create(String studyStr, File.Type type, File.Format format, File.Bioformat bioformat, String path, String creationDate, String description, File.FileStatus status, long size, long experimentId, List<Sample> samples, long jobId, Map<String, Object> stats, Map<String, Object> attributes, boolean parents, String content, QueryOptions options, String sessionId) throws CatalogException { File file = new File(type, format, bioformat, path, description, status, size, samples, jobId, null, stats, attributes); return create(studyStr, file, parents, content, options, sessionId); }
@Deprecated public QueryResult<Job> create(long studyId, String name, String toolName, String description, String executor, Map<String, String> params, String commandLine, URI tmpOutDirUri, long outDirId, List<File> inputFiles, List<File> outputFiles, Map<String, Object> attributes, Map<String, Object> resourceManagerAttributes, Job.JobStatus status, long startTime, long endTime, QueryOptions options, String sessionId) throws CatalogException { Job job = new Job(-1, null, name, "", toolName, null, "", description, startTime, endTime, executor, "", commandLine, false, status, -1, new File().setUid(outDirId), inputFiles, outputFiles, Collections.emptyList(), params, -1, attributes, resourceManagerAttributes); return create(String.valueOf(studyId), job, options, sessionId); }
public static void insertPedigreeFile(CatalogManager catalogManager, Path inputFile, String sessionId) throws CatalogException, FileNotFoundException { String path = "data/peds"; URI sourceUri = inputFile.toUri(); File file = catalogManager.getFileManager().upload("user1@default:study1", new FileInputStream(new java.io.File(sourceUri)), new File().setPath(Paths.get(path, inputFile.getFileName().toString()).toString()), false, true, sessionId).first(); // Load samples using the pedigree file CatalogSampleAnnotationsLoader catalogSampleAnnotationsLoader = new CatalogSampleAnnotationsLoader(catalogManager); catalogSampleAnnotationsLoader.loadSampleAnnotations(file, null, sessionId); } //
@Test public void getJobTest() throws CatalogException { long studyId = user3.getProjects().get(0).getStudies().get(0).getUid(); Job job = catalogJobDBAdaptor.insert(new Job("name", user3.getId(), "", "", "", new File().setUid(4), Collections.emptyList(), 1), studyId, null).first(); long jobId = job.getUid(); job = catalogJobDBAdaptor.get(jobId, null).first(); System.out.println(job); try { catalogJobDBAdaptor.get(-1, null); fail("error: expected exception"); } catch (CatalogDBException e) { System.out.println("correct exception: " + e); } }
@Test public void testDeleteExisting() throws IOException, CatalogException { QueryResult<File> queryResult = catalogManager.getFileManager().upload(study.getFqn(), CatalogManagerTest.createDebugFile().toURI(), new File().setPath(folder.getPath() + "file1.txt"), false, false, sessionIdUser); File file = queryResult.first(); CatalogManagerTest.createDebugFile(directory.resolve("file1.txt").toString()); List<File> files = new FileScanner(catalogManager).scan(folder, directory.toUri(), FileScanner.FileScannerPolicy.DELETE, false, true, sessionIdUser); files.forEach((File f) -> assertFalse(f.getAttributes().containsKey("checksum"))); assertEquals(File.FileStatus.DELETED, getFile(file.getUid()).getStatus().getName()); }
assertTrue(studyId >= 0); File file; file = new File("jobs/", File.Type.DIRECTORY, File.Format.PLAIN, File.Bioformat.NONE, "jobs/", "", new File.FileStatus(File.FileStatus.STAGE), 1000, 1); LinkedList<FileAclEntry> acl = new LinkedList<>(); acl.push(new FileAclEntry("jmmut", Collections.emptyList())); System.out.println(catalogFileDBAdaptor.insert(studyId, file, null, null)); file = new File("file.sam", File.Type.FILE, File.Format.PLAIN, File.Bioformat.ALIGNMENT, "data/file.sam", "", new File.FileStatus(File.FileStatus.STAGE), 1000, 1); System.out.println(catalogFileDBAdaptor.insert(studyId, file, null, null)); file = new File("file.bam", File.Type.FILE, File.Format.BINARY, File.Bioformat.ALIGNMENT, "data/file.bam", "", new File.FileStatus(File.FileStatus.STAGE), 1000, 1); System.out.println(catalogFileDBAdaptor.insert(studyId, file, null, null)); file = new File("file.vcf", File.Type.FILE, File.Format.PLAIN, File.Bioformat.VARIANT, "data/file2.vcf", "", new File.FileStatus(File.FileStatus.STAGE), 1000, 1);
@Test public void deleteJobTest() throws CatalogException { long studyId = user3.getProjects().get(0).getStudies().get(0).getUid(); Job job = catalogJobDBAdaptor.insert(new Job("name", user3.getId(), "", "", "", new File().setUid(4), Collections.emptyList(), 1), studyId, null).first(); long jobId = job.getUid(); assertEquals(Job.JobStatus.PREPARED, job.getStatus().getName()); catalogJobDBAdaptor.delete(jobId); thrown.expect(CatalogDBException.class); thrown.expectMessage("not exist"); catalogJobDBAdaptor.get(jobId, QueryOptions.empty()); }
private File prepareFiles(List<File> folderFiles) throws CatalogException { File folder = catalogManager.getFileManager().createFolder(studyFqn, Paths.get("folder").toString(), null, false, null, QueryOptions.empty(), userSessionId).first(); folderFiles.add( catalogManager.getFileManager().create(studyFqn, new File().setPath("folder/my.txt"), false, StringUtils.randomString(200), null, userSessionId).first() ); folderFiles.add( catalogManager.getFileManager().create(studyFqn, new File().setPath("folder/my2.txt"), false, StringUtils.randomString(200), null, userSessionId).first() ); folderFiles.add( catalogManager.getFileManager().create(studyFqn, new File().setPath("folder/my3.txt"), false, StringUtils.randomString(200), null, userSessionId).first() ); folderFiles.add( catalogManager.getFileManager().create(studyFqn, new File().setPath("folder/subfolder/my4.txt"), true, StringUtils.randomString(200), null, userSessionId).first() ); folderFiles.add( catalogManager.getFileManager().create(studyFqn, new File().setPath("folder/subfolder/my5.txt"), false, StringUtils.randomString(200), null, userSessionId).first() ); folderFiles.add( catalogManager.getFileManager().create(studyFqn, new File().setPath("folder/subfolder/subsubfolder/my6.txt"), true, StringUtils.randomString(200), null, userSessionId).first() ); return folder; }
private File createBasicDirectoryFileTestEnvironment(List<File> folderFiles) throws CatalogException { File folder = catalogManager.getFileManager().createFolder(studyFqn, Paths.get("folder").toString(), null, false, null, QueryOptions.empty(), sessionIdUser).first(); folderFiles.add( catalogManager.getFileManager().create(studyFqn, new File().setPath("folder/my.txt"), false, StringUtils.randomString(200), null, sessionIdUser).first() ); folderFiles.add( catalogManager.getFileManager().create(studyFqn, new File().setPath("folder/my2.txt"), false, StringUtils.randomString(200), null, sessionIdUser).first() ); folderFiles.add( catalogManager.getFileManager().create(studyFqn, new File().setPath("folder/my3.txt"), false, StringUtils.randomString(200), null, sessionIdUser).first() ); folderFiles.add( catalogManager.getFileManager().create(studyFqn, new File().setPath("folder/subfolder/my4.txt"), true, StringUtils.randomString(200), null, sessionIdUser).first() ); folderFiles.add( catalogManager.getFileManager().create(studyFqn, new File().setPath("folder/subfolder/my5.txt"), false, StringUtils.randomString(200), null, sessionIdUser).first() ); folderFiles.add( catalogManager.getFileManager().create(studyFqn, new File().setPath("folder/subfolder/subsubfolder/my6.txt"), true, StringUtils.randomString(200), null, sessionIdUser).first() ); return folder; }
@Test public void updateInputAndOutputFiles() throws Exception { Job job = new Job() .setOutDir(new File().setUid(5)) .setStatus(new Job.JobStatus()); long studyId = user3.getProjects().get(0).getStudies().get(0).getUid(); job.setName("jobName1"); QueryResult<Job> insert = catalogJobDBAdaptor.insert(job, studyId, null); List<File> fileInput = Arrays.asList( new File().setUid(5L).setName("file1").setStatus(new File.FileStatus()), new File().setUid(6L).setName("file2").setStatus(new File.FileStatus()), new File().setUid(7L).setName("file3").setStatus(new File.FileStatus()) ); List<File> fileOutput = Arrays.asList( new File().setUid(15L).setName("file1").setStatus(new File.FileStatus()), new File().setUid(16L).setName("file2").setStatus(new File.FileStatus()), new File().setUid(17L).setName("file3").setStatus(new File.FileStatus()) ); ObjectMap params = new ObjectMap() .append(JobDBAdaptor.QueryParams.INPUT.key(), fileInput) .append(JobDBAdaptor.QueryParams.OUTPUT.key(), fileOutput); QueryResult<Job> update = catalogJobDBAdaptor.update(insert.first().getUid(), params, QueryOptions.empty()); assertEquals(3, update.first().getInput().size()); assertEquals(3, update.first().getOutput().size()); assertTrue(Arrays.asList(5L, 6L, 7L).containsAll(update.first().getInput().stream().map(File::getUid).collect(Collectors.toList()))); assertTrue(Arrays.asList(15L, 16L, 17L) .containsAll(update.first().getOutput().stream().map(File::getUid).collect(Collectors.toList()))); }
@Test public void renameFileTest() throws CatalogException { QueryResult<File> queryResult1 = catalogManager.getFileManager().create(studyFqn, new File().setPath("data/file.txt"), true, StringUtils.randomString(200), null, sessionIdUser); assertEquals(1, queryResult1.getNumResults()); QueryResult<File> queryResult = catalogManager.getFileManager().create(studyFqn, new File().setPath("data/nested/folder/file2.txt"), true, StringUtils.randomString(200), null, sessionIdUser); assertEquals(1, queryResult.getNumResults()); catalogManager.getFileManager().rename(studyFqn, "data/nested/", "nested2", sessionIdUser); Set<String> paths = catalogManager.getFileManager().get(studyFqn, new Query(), new QueryOptions(), sessionIdUser) .getResult() .stream().map(File::getPath).collect(Collectors.toSet()); assertTrue(paths.contains("data/nested2/")); assertFalse(paths.contains("data/nested/")); assertTrue(paths.contains("data/nested2/folder/")); assertTrue(paths.contains("data/nested2/folder/file2.txt")); assertTrue(paths.contains("data/file.txt")); catalogManager.getFileManager().rename(studyFqn, "data/", "Data", sessionIdUser); paths = catalogManager.getFileManager().get(studyFqn, new Query(), new QueryOptions(), sessionIdUser).getResult() .stream().map(File::getPath).collect(Collectors.toSet()); assertTrue(paths.contains("Data/")); assertTrue(paths.contains("Data/file.txt")); assertTrue(paths.contains("Data/nested2/")); assertTrue(paths.contains("Data/nested2/folder/")); assertTrue(paths.contains("Data/nested2/folder/file2.txt")); }
@Test public void SetVisitedJob() throws CatalogException { long studyId = user3.getProjects().get(0).getStudies().get(0).getUid(); Job jobBefore = catalogJobDBAdaptor.insert(new Job("name", user3.getId(), "", "", "", new File().setUid(4), Collections.emptyList(), 1), studyId, null).first(); long jobId = jobBefore.getUid(); assertTrue(!jobBefore.isVisited()); ObjectMap params = new ObjectMap(JobDBAdaptor.QueryParams.VISITED.key(), true); catalogJobDBAdaptor.update(jobBefore.getUid(), params, QueryOptions.empty()); Job jobAfter = catalogJobDBAdaptor.get(jobId, null).first(); assertTrue(jobAfter.isVisited()); }
@Test public void testGetMetadataFromVcf() throws CatalogException, FileNotFoundException { File file = catalogManager.getFileManager().upload(study.getFqn(), vcfFileUri, new File().setPath(folder.getPath() + VCF_FILE_NAME), false, false, sessionIdUser).first(); assertTrue(file.getSize() > 0); file = FileMetadataReader.get(catalogManager). setMetadataInformation(file, null, null, sessionIdUser, false); assertEquals(File.FileStatus.READY, file.getStatus().getName()); assertEquals(File.Format.VCF, file.getFormat()); assertEquals(File.Bioformat.VARIANT, file.getBioformat()); assertNotNull(file.getAttributes().get(VARIANT_FILE_METADATA)); assertEquals(4, file.getSamples().size()); assertEquals(expectedSampleNames, ((Map<String, Object>) file.getAttributes().get(VARIANT_FILE_METADATA)).get("sampleIds")); catalogManager.getSampleManager().get(study.getFqn(), new Query(SampleDBAdaptor.QueryParams.ID.key(), file.getSamples().stream().map(Sample::getId).collect(Collectors.toList())), new QueryOptions(), sessionIdUser).getResult(); assertTrue(expectedSampleNames.containsAll(file.getSamples().stream().map(Sample::getId).collect(Collectors.toSet()))); }
@Test public void testGetBasicMetadata() throws CatalogException, IOException { File file = catalogManager.getFileManager().create(study.getFqn(), new File().setPath(folder.getPath() + "test.txt"), false, StringUtils.randomString(1000), null, sessionIdUser).first(); assertEquals(1000, file.getSize()); String creationDate = file.getCreationDate(); String modificationDate = file.getModificationDate(); URI fileUri = catalogManager.getFileManager().getUri(file); try { Thread.sleep(1000); //Sleep 1 second to see changes on the "modificationDate" } catch (InterruptedException ignored) {} OutputStream outputStream = new FileOutputStream(Paths.get(fileUri).toFile(), true); byte[] bytes2 = StringUtils.randomString(100).getBytes(); outputStream.write(bytes2); outputStream.close(); file = FileMetadataReader.get(catalogManager). setMetadataInformation(file, null, null, sessionIdUser, false); assertEquals(1000 + bytes2.length, file.getSize()); assertTrue(TimeUtils.toDate(modificationDate).getTime() < TimeUtils.toDate(file.getModificationDate()).getTime()); assertEquals(creationDate, file.getCreationDate()); }
@Test public void testReplaceExisting() throws IOException, CatalogException { // Create and register file1.txt and s/file2.txt File file = catalogManager.getFileManager().upload(study.getFqn(), CatalogManagerTest.createDebugFile().toURI(), new File().setPath(folder.getPath() + "file1.txt"), false, false, true, true, sessionIdUser).first(); catalogManager.getFileManager().upload(study.getFqn(), CatalogManagerTest.createDebugFile().toURI(), new File().setPath(folder.getPath() + "s/file2.txt"), false, true, true, true, sessionIdUser).first(); // Create same file structure, and replace CatalogManagerTest.createDebugFile(directory.resolve("file1.txt").toString()); Files.createDirectory(directory.resolve("s/")); CatalogManagerTest.createDebugFile(directory.resolve("s/file2.txt").toString()); FileScanner fileScanner = new FileScanner(catalogManager); fileScanner.scan(folder, directory.toUri(), FileScanner.FileScannerPolicy.REPLACE, true, true, sessionIdUser); File replacedFile = catalogManager.getFileManager().get(study.getFqn(), file.getPath(), null, sessionIdUser).first(); assertEquals(File.FileStatus.READY, replacedFile.getStatus().getName()); assertEquals(file.getUid(), replacedFile.getUid()); assertNotEquals(replacedFile.getChecksum(), file.getChecksum()); assertEquals(replacedFile.getChecksum(), catalogManager.getCatalogIOManagerFactory().getDefault().calculateChecksum(replacedFile.getUri())); }
@Test public void FileToSolrTest() { Study study = new Study().setFqn("user@project:study").setAttributes(new HashMap<>()); File file = new File("name", File.Type.FILE, File.Format.BAM, File.Bioformat.MICROARRAY_EXPRESSION_ONECHANNEL_AGILENT, "test/base", "convertorTest", new File.FileStatus("READY"), 1111L, 2); file.setUid(111).setSamples(Arrays.asList(new Sample().setId("1"), new Sample().setId("2")))
@Test public void testGetMetadataFromBam() throws CatalogException, FileNotFoundException { File file = catalogManager.getFileManager().upload(study.getFqn(), bamFileUri, new File().setPath(folder.getPath() + BAM_FILE_NAME), false, false, sessionIdUser).first(); assertTrue(file.getSize() > 0); file = FileMetadataReader.get(catalogManager). setMetadataInformation(file, null, null, sessionIdUser, false); assertEquals(File.FileStatus.READY, file.getStatus().getName()); // assertEquals(File.Format.GZIP, file.getFormat()); assertEquals(File.Bioformat.ALIGNMENT, file.getBioformat()); assertNotNull(file.getAttributes().get("alignmentHeader")); assertEquals(1, file.getSamples().size()); assertEquals("HG00096", catalogManager.getSampleManager().get(study.getFqn(), file.getSamples().get(0).getId(), null, sessionIdUser).first().getId()); }
@Test public void testDoNotOverwriteSampleIds() throws CatalogException, FileNotFoundException { File file = catalogManager.getFileManager().upload(study.getFqn(), vcfFileUri, new File().setPath(folder.getPath() + VCF_FILE_NAME), false, false, sessionIdUser).first(); assertEquals(File.FileStatus.READY, file.getStatus().getName()); assertEquals(File.Format.VCF, file.getFormat()); assertEquals(File.Bioformat.VARIANT, file.getBioformat()); assertNotNull(file.getAttributes().get(VARIANT_FILE_METADATA)); assertEquals(4, file.getSamples().size()); //Add a sampleId String sampleId = catalogManager.getSampleManager().create(study.getFqn(), new Sample().setId("Bad_Sample"), null, sessionIdUser) .first().getId(); catalogManager.getFileManager().update(study.getFqn(), file.getPath(), new ObjectMap(FileDBAdaptor.QueryParams.SAMPLES.key(), Collections.singletonList(sampleId)), new QueryOptions(), sessionIdUser); file = catalogManager.getFileManager().get(study.getFqn(), file.getPath(), null, sessionIdUser).first(); assertEquals(5, file.getSamples().size()); assertEquals(sampleId, file.getSamples().get(4).getId()); }
@Test public void getAllJobs() throws CatalogException { Job job = new Job() .setId("job1") .setToolId("toolId") .setOutDir(new File().setPath(data_d1_d2)); long job1 = catalogManager.getJobManager().create(studyFqn, job, null, ownerSessionId).first().getUid(); job.setId("job2"); long job2 = catalogManager.getJobManager().create(studyFqn, job, null, ownerSessionId).first().getUid(); job.setId("job3"); long job3 = catalogManager.getJobManager().create(studyFqn, job, null, ownerSessionId).first().getUid(); job.setId("job4"); long job4 = catalogManager.getJobManager().create(studyFqn, job, null, ownerSessionId).first().getUid(); checkGetAllJobs(Arrays.asList(job1, job2, job3, job4), ownerSessionId); //Owner can see everything checkGetAllJobs(Collections.emptyList(), externalSessionId); //Can't see inside data_d1_d2_d3 }
@Test public void testDeleteTrashed() throws IOException, CatalogException { QueryResult<File> queryResult = catalogManager.getFileManager().upload(study.getFqn(), CatalogManagerTest.createDebugFile().toURI(), new File().setPath(folder.getPath() + "file1.txt"), false, false, sessionIdUser); File file = queryResult.first(); catalogManager.getFileManager().delete(study.getFqn(), new Query(FileDBAdaptor.QueryParams.UID.key(), file.getUid()), new QueryOptions(), sessionIdUser); QueryResult<File> fileQueryResult = catalogManager.getFileManager().get(study.getFqn(), new Query() .append(FileDBAdaptor.QueryParams.UID.key(), file.getUid()) .append(FileDBAdaptor.QueryParams.STATUS_NAME.key(), "!=EMPTY"), new QueryOptions(), sessionIdUser); file = fileQueryResult.first(); assertEquals(File.FileStatus.TRASHED, file.getStatus().getName()); }