String status = file.getIndex() == null || file.getIndex().getStatus() == null ? FileIndex.IndexStatus.NONE : file.getIndex().getStatus().getName(); if (!status.equals(FileIndex.IndexStatus.READY)) { final FileIndex index; index = file.getIndex() == null ? new FileIndex() : file.getIndex(); if (index.getStatus() == null) { index.setStatus(new FileIndex.IndexStatus()); file.getIndex().getStatus().getName(), FileIndex.IndexStatus.READY); index.getStatus().setName(FileIndex.IndexStatus.READY); catalogManager.getFileManager().setFileIndex(studyConfiguration.getStudyName(), file.getPath(), index, sessionId); final FileIndex index; index = file.getIndex() == null ? new FileIndex() : file.getIndex(); String prevStatus = index.getStatus().getName(); String newStatus; if (hasTransformedFile(index)) {
: file.getIndex().getStatus().getName(); switch (status) { case FileIndex.IndexStatus.NONE:
|| file.getIndex().getStatus().getName() == null || file.getIndex().getStatus().getName().equals(FileIndex.IndexStatus.NONE)); if (vcf.getIndex() != null && vcf.getIndex().getStatus() != null && vcf.getIndex().getStatus().getName() != null) { status = vcf.getIndex().getStatus().getName(); index.setStatus(new FileIndex.IndexStatus(FileIndex.IndexStatus.TRANSFORMED, "Found transformed file"));
file.setIndex(new FileIndex("", "", new FileIndex.IndexStatus(Status.READY, ""), -1, Collections.emptyMap())); file.setSamples(samples);
switch (index.getStatus().getName()) { case FileIndex.IndexStatus.NONE: case FileIndex.IndexStatus.TRANSFORMED: logger.warn(indexStatusMessage); case FileIndex.IndexStatus.READY: //Do not show warn message when index status is READY. indexStatusName = index.getStatus().getName(); break; case FileIndex.IndexStatus.TRANSFORMING: throw new IllegalStateException("Unknown Index Status " + index.getStatus().getName());
&& !FileIndex.IndexStatus.NONE.equals(fileAux.getIndex().getStatus().getName()) && !FileIndex.IndexStatus.TRANSFORMED.equals(fileAux.getIndex().getStatus().getName())) { throw new CatalogException("Cannot delete file: " + fileAux.getName() + ". The index status is " + fileAux.getIndex().getStatus().getName()); while (iterator.hasNext()) { File next = iterator.next(); String status = next.getIndex().getStatus().getName(); switch (status) { case FileIndex.IndexStatus.READY: next.getIndex().getStatus().setName(FileIndex.IndexStatus.NONE); break; case FileIndex.IndexStatus.NONE:
&& !FileIndex.IndexStatus.NONE.equals(file.getIndex().getStatus().getName())) { errorFiles.add(file.getPath() + "(" + file.getUid() + ")");
variantManager.index(studyId, inputFile.getId(), outdir, queryOptions, sessionId); inputFile = catalogManager.getFileManager().get(studyId, inputFile.getId(), null, sessionId).first(); assertEquals(FileIndex.IndexStatus.TRANSFORMED, inputFile.getIndex().getStatus().getName()); assertNotNull(inputFile.getStats().get(FileMetadataReader.VARIANT_FILE_STATS));
if (file.getStatus().getName().equals(File.FileStatus.READY) && file.getFormat().equals(File.Format.VCF)) { String indexStatus; if (file.getIndex() != null && file.getIndex().getStatus() != null && file.getIndex().getStatus().getName() != null) { indexStatus = file.getIndex().getStatus().getName(); } else { indexStatus = FileIndex.IndexStatus.NONE;
assertEquals(FileIndex.IndexStatus.READY, catalogManager.getFileManager().get(studyId, nonIndexedFile.getName(), null, sessionId).first().getIndex().getStatus().getName()); studyConfigurationFactory.updateCatalogFromStudyConfiguration(sc, sessionId); assertEquals(FileIndex.IndexStatus.INDEXING, catalogManager.getFileManager().get(studyId, nonIndexedFile.getName(), null, sessionId).first().getIndex().getStatus().getName());
public QueryResult<FileIndex> updateFileIndexStatus(File file, String newStatus, String message, Integer release, String sessionId) throws CatalogException { String userId = catalogManager.getUserManager().getUserId(sessionId); Long studyId = file.getStudyUid(); authorizationManager.checkFilePermission(studyId, file.getUid(), userId, FileAclEntry.FilePermissions.WRITE); FileIndex index = file.getIndex(); if (index != null) { if (!FileIndex.IndexStatus.isValid(newStatus)) { throw new CatalogException("The status " + newStatus + " is not a valid status."); } else { index.setStatus(new FileIndex.IndexStatus(newStatus, message)); } } else { index = new FileIndex(userId, TimeUtils.getTime(), new FileIndex.IndexStatus(newStatus), -1, new ObjectMap()); } if (release != null) { if (newStatus.equals(FileIndex.IndexStatus.READY)) { index.setRelease(release); } } ObjectMap params = new ObjectMap(FileDBAdaptor.QueryParams.INDEX.key(), index); fileDBAdaptor.update(file.getUid(), params, QueryOptions.empty()); auditManager.recordUpdate(AuditRecord.Resource.file, file.getUid(), userId, params, null, null); return new QueryResult<>("Update file index", 0, 1, 1, "", "", Arrays.asList(index)); }
public List<File> removeFiles(StudyInfo studyInfo, QueryOptions options, String sessionId) throws CatalogException, StorageEngineException, IOException { // We get the credentials of the Datastore to insert the variants DataStore dataStore = studyInfo.getDataStores().get(File.Bioformat.VARIANT); // Update study configuration BEFORE executing the operation and fetching files from Catalog updateCatalogFromStudyConfiguration(sessionId, studyInfo.getStudyFQN(), dataStore); List<String> fileNames = new ArrayList<>(studyInfo.getFileInfos().size()); List<String> filePaths = new ArrayList<>(studyInfo.getFileInfos().size()); for (FileInfo fileInfo : studyInfo.getFileInfos()) { File file = catalogManager.getFileManager().get(studyInfo.getStudyFQN(), fileInfo.getPath(), null, sessionId).first(); if (file.getIndex().getStatus().getName().equals(FileIndex.IndexStatus.READY)) { fileNames.add(fileInfo.getName()); filePaths.add(fileInfo.getPath()); } else { throw new CatalogException("Unable to remove variants from file " + file.getName() + ". " + "IndexStatus = " + file.getIndex().getStatus().getName()); } } if (fileNames.isEmpty()) { throw new CatalogException("Nothing to do!"); } VariantStorageEngine variantStorageEngine = getVariantStorageEngine(dataStore); variantStorageEngine.getOptions().putAll(options); variantStorageEngine.removeFiles(studyInfo.getStudyFQN(), fileNames); // Update study configuration to synchronize updateCatalogFromStudyConfiguration(sessionId, studyInfo.getStudyFQN(), dataStore); return catalogManager.getFileManager().get(studyInfo.getStudyFQN(), new Query(FileDBAdaptor.QueryParams.PATH.key(), filePaths), new QueryOptions(), sessionId) .getResult(); }
private void removeFile(List<File> files, QueryOptions options, String outputId) throws Exception { List<String> fileIds = files.stream().map(File::getId).collect(Collectors.toList()); Study study = catalogManager.getFileManager().getStudy(files.get(0), sessionId); String studyId = study.getFqn(); List<File> removedFiles = variantManager.removeFile(fileIds, studyId, sessionId, new QueryOptions()); assertEquals(files.size(), removedFiles.size()); Cohort all = catalogManager.getCohortManager().get(studyId, new Query(CohortDBAdaptor.QueryParams.ID.key(), StudyEntry.DEFAULT_COHORT), null, sessionId).first(); Set<Long> allSampleIds = all.getSamples().stream().map(Sample::getUid).collect(Collectors.toSet()); assertThat(all.getStatus().getName(), anyOf(is(Cohort.CohortStatus.INVALID), is(Cohort.CohortStatus.NONE))); Set<Long> loadedSamples = catalogManager.getFileManager().get(studyId, new Query(FileDBAdaptor.QueryParams.INDEX_STATUS_NAME.key (), FileIndex.IndexStatus.READY), null, sessionId) .getResult() .stream() .flatMap(f -> f.getSamples().stream()) .map(Sample::getUid) .collect(Collectors.toSet()); assertEquals(loadedSamples, allSampleIds); for (File file : removedFiles) { assertEquals(FileIndex.IndexStatus.TRANSFORMED, file.getIndex().getStatus().getName()); } }
private void checkStudyConfiguration(Study study, StudyConfiguration studyConfiguration) throws CatalogException { assertEquals("user@p1:s1", studyConfiguration.getStudyName()); assertEquals(study.getUid(), studyConfiguration.getStudyId()); assertTrue(studyConfiguration.getInvalidStats().isEmpty()); for (Map.Entry<String, Integer> entry : studyConfiguration.getFileIds().entrySet()) { File file = catalogManager.getFileManager().get(studyConfiguration.getStudyName(), studyConfiguration.getFileIds().inverse().get(entry.getValue()), null, sessionId).first(); assertEquals(file.getName(), entry.getKey()); int id = (int) file.getUid(); assertEquals(file.getSamples().stream().map(Sample::getUid).map(Long::intValue).collect(Collectors.toSet()), studyConfiguration.getSamplesInFiles().get((id))); if (file.getIndex() == null || file.getIndex().getStatus() == null || file.getIndex().getStatus().getName() == null || !file.getIndex().getStatus().getName().equals(FileIndex.IndexStatus.READY)) { assertFalse(studyConfiguration.getIndexedFiles().contains(id)); // assertFalse("Should not contain header for file " + file.getId(), studyConfiguration.getHeaders().containsKey(id)); } // else { // assertTrue(studyConfiguration.getIndexedFiles().contains(id)); // assertTrue("Missing header for file " + file.getId(), studyConfiguration.getHeaders().containsKey(id)); // assertTrue("Missing header for file " + file.getId(), !studyConfiguration.getHeaders().get(id).isEmpty()); // } } }
@Test public void testIndexWithStatsWrongAggregationType() throws Exception { QueryOptions queryOptions = new QueryOptions(VariantStorageEngine.Options.ANNOTATE.key(), false); queryOptions.put(VariantStorageEngine.Options.CALCULATE_STATS.key(), true); queryOptions.put(VariantStorageEngine.Options.AGGREGATED_TYPE.key(), "wrong_type"); queryOptions.putIfNotNull(StorageOperation.CATALOG_PATH, outputId); try { variantManager.index(studyId, getFile(0).getId(), newTmpOutdir(), queryOptions, sessionId); fail("Expected StoragePipelineException exception"); } catch (StoragePipelineException e) { assertEquals(0, getDefaultCohort(studyId).getSamples().size()); assertEquals(Cohort.CohortStatus.NONE, getDefaultCohort(studyId).getStatus().getName()); assertEquals(FileIndex.IndexStatus.TRANSFORMED, catalogManager.getFileManager().get(studyId, getFile(0).getId(), null, sessionId).first().getIndex().getStatus().getName()); assertNotNull(catalogManager.getFileManager().get(studyId, getFile(0).getId(), null, sessionId).first().getStats().get(FileMetadataReader.VARIANT_FILE_STATS)); } queryOptions.put(VariantStorageEngine.Options.AGGREGATED_TYPE.key(), "none"); // File already transformed queryOptions.put(VariantFileIndexerStorageOperation.LOAD, true); variantManager.index(studyId, getFile(0).getId(), newTmpOutdir(), queryOptions, sessionId); assertEquals(500, getDefaultCohort(studyId).getSamples().size()); assertEquals(Cohort.CohortStatus.READY, getDefaultCohort(studyId).getStatus().getName()); assertNotNull(catalogManager.getFileManager().get(studyId, getFile(0).getId(), null, sessionId).first().getStats().get(FileMetadataReader.VARIANT_FILE_STATS)); }
@Override protected List<String> validate(String defaultStudyStr, List<String> values, Integer release, VariantQueryParam param, String sessionId) throws CatalogException { if (release == null) { AbstractManager.MyResources<File> uids = catalogManager.getFileManager().getUids(values, defaultStudyStr, sessionId); return uids.getResourceList().stream().map(File::getName).collect(Collectors.toList()); } else { return validate(defaultStudyStr, values, release, param, catalogManager.getFileManager(), File::getName, file -> ((int) file.getIndex().getRelease()), file -> { if (file.getIndex() == null || file.getIndex().getStatus() == null || file.getIndex().getStatus().getName() == null || !file.getIndex().getStatus().getName().equals(Status.READY)) { throw new VariantQueryException("File '" + file.getName() + "' is not indexed"); } }, sessionId); } } }
protected void checkEtlResults(String studyId, List<StoragePipelineResult> etlResults, String expectedStatus) throws CatalogException { for (StoragePipelineResult etlResult : etlResults) { File input = catalogManager.getFileManager().get(studyId, new Query(FileDBAdaptor.QueryParams.URI.key(), etlResult.getInput()), null, sessionId).first(); String indexedFileId; if (input.getRelatedFiles().isEmpty()) { indexedFileId = input.getId(); } else { long indexedFileUid = input.getRelatedFiles().get(0).getFileId(); indexedFileId = catalogManager.getFileManager().get(studyId, new Query(FileDBAdaptor.QueryParams.UID.key(), indexedFileUid), new QueryOptions(), sessionId).first().getId(); } assertEquals(expectedStatus, catalogManager.getFileManager().get(studyId, indexedFileId, null, sessionId).first().getIndex().getStatus().getName()); System.out.println("etlResult = " + etlResult); } }
public static File create(String resourceName, boolean indexed) throws IOException, CatalogException { File file; URI uri = getResourceUri(resourceName); file = fileMetadataReader.create(studyId, uri, "data/vcfs/", "", true, null, sessionId).first(); catalogFileUtils.upload(uri, file, null, sessionId, false, false, true, false, Long.MAX_VALUE); if (indexed) { FileIndex fileIndex = new FileIndex("user", "today", new FileIndex.IndexStatus(FileIndex.IndexStatus.READY), 1234, Collections.emptyMap()); catalogManager.getFileManager().setFileIndex(studyId, file.getPath(), fileIndex, sessionId); indexedFiles.add((int) file.getUid()); List<String> samples = catalogManager.getCohortManager().getSamples(studyId, cohortId, null, sessionId).getResult().stream().map(Sample::getId).collect(Collectors.toList()); samples.addAll(file.getSamples().stream().map(Sample::getId).collect(Collectors.toList())); catalogManager.getCohortManager().update(studyId, cohortId, new ObjectMap(CohortDBAdaptor.QueryParams.SAMPLES.key(), samples), true, null, sessionId); } return catalogManager.getFileManager().get(studyId, file.getId(), null, sessionId).first(); }