if (file.getStatus().getName().equals(File.FileStatus.READY) && file.getFormat().equals(File.Format.VCF)) { String indexStatus; if (file.getIndex() != null && file.getIndex().getStatus() != null && file.getIndex().getStatus().getName() != null) { indexStatus = file.getIndex().getStatus().getName(); } else { indexStatus = FileIndex.IndexStatus.NONE;
: file.getIndex().getStatus().getName(); switch (status) { case FileIndex.IndexStatus.NONE:
while (iterator.hasNext()) { File next = iterator.next(); String status = next.getIndex().getStatus().getName(); switch (status) { case FileIndex.IndexStatus.READY:
switch (index.getStatus().getName()) { case FileIndex.IndexStatus.NONE: case FileIndex.IndexStatus.TRANSFORMED: logger.warn(indexStatusMessage); case FileIndex.IndexStatus.READY: //Do not show warn message when index status is READY. indexStatusName = index.getStatus().getName(); break; case FileIndex.IndexStatus.TRANSFORMING: throw new IllegalStateException("Unknown Index Status " + index.getStatus().getName());
&& !FileIndex.IndexStatus.NONE.equals(fileAux.getIndex().getStatus().getName()) && !FileIndex.IndexStatus.TRANSFORMED.equals(fileAux.getIndex().getStatus().getName())) { throw new CatalogException("Cannot delete file: " + fileAux.getName() + ". The index status is " + fileAux.getIndex().getStatus().getName()); while (iterator.hasNext()) { File next = iterator.next(); String status = next.getIndex().getStatus().getName(); switch (status) { case FileIndex.IndexStatus.READY:
&& !FileIndex.IndexStatus.NONE.equals(file.getIndex().getStatus().getName())) { errorFiles.add(file.getPath() + "(" + file.getUid() + ")");
variantManager.index(studyId, inputFile.getId(), outdir, queryOptions, sessionId); inputFile = catalogManager.getFileManager().get(studyId, inputFile.getId(), null, sessionId).first(); assertEquals(FileIndex.IndexStatus.TRANSFORMED, inputFile.getIndex().getStatus().getName()); assertNotNull(inputFile.getStats().get(FileMetadataReader.VARIANT_FILE_STATS));
assertEquals(FileIndex.IndexStatus.READY, catalogManager.getFileManager().get(studyId, nonIndexedFile.getName(), null, sessionId).first().getIndex().getStatus().getName()); studyConfigurationFactory.updateCatalogFromStudyConfiguration(sc, sessionId); assertEquals(FileIndex.IndexStatus.INDEXING, catalogManager.getFileManager().get(studyId, nonIndexedFile.getName(), null, sessionId).first().getIndex().getStatus().getName());
public List<File> removeFiles(StudyInfo studyInfo, QueryOptions options, String sessionId) throws CatalogException, StorageEngineException, IOException { // We get the credentials of the Datastore to insert the variants DataStore dataStore = studyInfo.getDataStores().get(File.Bioformat.VARIANT); // Update study configuration BEFORE executing the operation and fetching files from Catalog updateCatalogFromStudyConfiguration(sessionId, studyInfo.getStudyFQN(), dataStore); List<String> fileNames = new ArrayList<>(studyInfo.getFileInfos().size()); List<String> filePaths = new ArrayList<>(studyInfo.getFileInfos().size()); for (FileInfo fileInfo : studyInfo.getFileInfos()) { File file = catalogManager.getFileManager().get(studyInfo.getStudyFQN(), fileInfo.getPath(), null, sessionId).first(); if (file.getIndex().getStatus().getName().equals(FileIndex.IndexStatus.READY)) { fileNames.add(fileInfo.getName()); filePaths.add(fileInfo.getPath()); } else { throw new CatalogException("Unable to remove variants from file " + file.getName() + ". " + "IndexStatus = " + file.getIndex().getStatus().getName()); } } if (fileNames.isEmpty()) { throw new CatalogException("Nothing to do!"); } VariantStorageEngine variantStorageEngine = getVariantStorageEngine(dataStore); variantStorageEngine.getOptions().putAll(options); variantStorageEngine.removeFiles(studyInfo.getStudyFQN(), fileNames); // Update study configuration to synchronize updateCatalogFromStudyConfiguration(sessionId, studyInfo.getStudyFQN(), dataStore); return catalogManager.getFileManager().get(studyInfo.getStudyFQN(), new Query(FileDBAdaptor.QueryParams.PATH.key(), filePaths), new QueryOptions(), sessionId) .getResult(); }
private void removeFile(List<File> files, QueryOptions options, String outputId) throws Exception { List<String> fileIds = files.stream().map(File::getId).collect(Collectors.toList()); Study study = catalogManager.getFileManager().getStudy(files.get(0), sessionId); String studyId = study.getFqn(); List<File> removedFiles = variantManager.removeFile(fileIds, studyId, sessionId, new QueryOptions()); assertEquals(files.size(), removedFiles.size()); Cohort all = catalogManager.getCohortManager().get(studyId, new Query(CohortDBAdaptor.QueryParams.ID.key(), StudyEntry.DEFAULT_COHORT), null, sessionId).first(); Set<Long> allSampleIds = all.getSamples().stream().map(Sample::getUid).collect(Collectors.toSet()); assertThat(all.getStatus().getName(), anyOf(is(Cohort.CohortStatus.INVALID), is(Cohort.CohortStatus.NONE))); Set<Long> loadedSamples = catalogManager.getFileManager().get(studyId, new Query(FileDBAdaptor.QueryParams.INDEX_STATUS_NAME.key (), FileIndex.IndexStatus.READY), null, sessionId) .getResult() .stream() .flatMap(f -> f.getSamples().stream()) .map(Sample::getUid) .collect(Collectors.toSet()); assertEquals(loadedSamples, allSampleIds); for (File file : removedFiles) { assertEquals(FileIndex.IndexStatus.TRANSFORMED, file.getIndex().getStatus().getName()); } }
private void checkStudyConfiguration(Study study, StudyConfiguration studyConfiguration) throws CatalogException { assertEquals("user@p1:s1", studyConfiguration.getStudyName()); assertEquals(study.getUid(), studyConfiguration.getStudyId()); assertTrue(studyConfiguration.getInvalidStats().isEmpty()); for (Map.Entry<String, Integer> entry : studyConfiguration.getFileIds().entrySet()) { File file = catalogManager.getFileManager().get(studyConfiguration.getStudyName(), studyConfiguration.getFileIds().inverse().get(entry.getValue()), null, sessionId).first(); assertEquals(file.getName(), entry.getKey()); int id = (int) file.getUid(); assertEquals(file.getSamples().stream().map(Sample::getUid).map(Long::intValue).collect(Collectors.toSet()), studyConfiguration.getSamplesInFiles().get((id))); if (file.getIndex() == null || file.getIndex().getStatus() == null || file.getIndex().getStatus().getName() == null || !file.getIndex().getStatus().getName().equals(FileIndex.IndexStatus.READY)) { assertFalse(studyConfiguration.getIndexedFiles().contains(id)); // assertFalse("Should not contain header for file " + file.getId(), studyConfiguration.getHeaders().containsKey(id)); } // else { // assertTrue(studyConfiguration.getIndexedFiles().contains(id)); // assertTrue("Missing header for file " + file.getId(), studyConfiguration.getHeaders().containsKey(id)); // assertTrue("Missing header for file " + file.getId(), !studyConfiguration.getHeaders().get(id).isEmpty()); // } } }
@Test public void testIndexWithStatsWrongAggregationType() throws Exception { QueryOptions queryOptions = new QueryOptions(VariantStorageEngine.Options.ANNOTATE.key(), false); queryOptions.put(VariantStorageEngine.Options.CALCULATE_STATS.key(), true); queryOptions.put(VariantStorageEngine.Options.AGGREGATED_TYPE.key(), "wrong_type"); queryOptions.putIfNotNull(StorageOperation.CATALOG_PATH, outputId); try { variantManager.index(studyId, getFile(0).getId(), newTmpOutdir(), queryOptions, sessionId); fail("Expected StoragePipelineException exception"); } catch (StoragePipelineException e) { assertEquals(0, getDefaultCohort(studyId).getSamples().size()); assertEquals(Cohort.CohortStatus.NONE, getDefaultCohort(studyId).getStatus().getName()); assertEquals(FileIndex.IndexStatus.TRANSFORMED, catalogManager.getFileManager().get(studyId, getFile(0).getId(), null, sessionId).first().getIndex().getStatus().getName()); assertNotNull(catalogManager.getFileManager().get(studyId, getFile(0).getId(), null, sessionId).first().getStats().get(FileMetadataReader.VARIANT_FILE_STATS)); } queryOptions.put(VariantStorageEngine.Options.AGGREGATED_TYPE.key(), "none"); // File already transformed queryOptions.put(VariantFileIndexerStorageOperation.LOAD, true); variantManager.index(studyId, getFile(0).getId(), newTmpOutdir(), queryOptions, sessionId); assertEquals(500, getDefaultCohort(studyId).getSamples().size()); assertEquals(Cohort.CohortStatus.READY, getDefaultCohort(studyId).getStatus().getName()); assertNotNull(catalogManager.getFileManager().get(studyId, getFile(0).getId(), null, sessionId).first().getStats().get(FileMetadataReader.VARIANT_FILE_STATS)); }
@Override protected List<String> validate(String defaultStudyStr, List<String> values, Integer release, VariantQueryParam param, String sessionId) throws CatalogException { if (release == null) { AbstractManager.MyResources<File> uids = catalogManager.getFileManager().getUids(values, defaultStudyStr, sessionId); return uids.getResourceList().stream().map(File::getName).collect(Collectors.toList()); } else { return validate(defaultStudyStr, values, release, param, catalogManager.getFileManager(), File::getName, file -> ((int) file.getIndex().getRelease()), file -> { if (file.getIndex() == null || file.getIndex().getStatus() == null || file.getIndex().getStatus().getName() == null || !file.getIndex().getStatus().getName().equals(Status.READY)) { throw new VariantQueryException("File '" + file.getName() + "' is not indexed"); } }, sessionId); } } }
protected void checkEtlResults(String studyId, List<StoragePipelineResult> etlResults, String expectedStatus) throws CatalogException { for (StoragePipelineResult etlResult : etlResults) { File input = catalogManager.getFileManager().get(studyId, new Query(FileDBAdaptor.QueryParams.URI.key(), etlResult.getInput()), null, sessionId).first(); String indexedFileId; if (input.getRelatedFiles().isEmpty()) { indexedFileId = input.getId(); } else { long indexedFileUid = input.getRelatedFiles().get(0).getFileId(); indexedFileId = catalogManager.getFileManager().get(studyId, new Query(FileDBAdaptor.QueryParams.UID.key(), indexedFileUid), new QueryOptions(), sessionId).first().getId(); } assertEquals(expectedStatus, catalogManager.getFileManager().get(studyId, indexedFileId, null, sessionId).first().getIndex().getStatus().getName()); System.out.println("etlResult = " + etlResult); } }
@Test public void testBySteps1() throws Exception { File transformFile = transformFile(inputFile1, new QueryOptions()); Assert.assertEquals(FileIndex.IndexStatus.TRANSFORMED, catalogManager.getFileManager().get(studyId, inputFile1.getPath(), null, sessionId).first().getIndex().getStatus().getName()); Assert.assertNull(catalogManager.getFileManager().get(studyId, inputFile2.getPath(), null, sessionId).first().getIndex().getStatus()); loadFile(transformFile, new QueryOptions(), outputId); Assert.assertEquals(FileIndex.IndexStatus.READY, catalogManager.getFileManager().get(studyId, inputFile1.getPath(), null, sessionId).first().getIndex().getStatus().getName()); Assert.assertNull(catalogManager.getFileManager().get(studyId, inputFile2.getPath(), null, sessionId).first().getIndex().getStatus()); }
@Test public void testIndex2() throws Exception { indexFile(inputFile2, new QueryOptions(), outputId); Assert.assertNull(catalogManager.getFileManager().get(studyId, inputFile1.getPath(), null, sessionId).first().getIndex().getStatus()); Assert.assertEquals(FileIndex.IndexStatus.READY, catalogManager.getFileManager().get(studyId, inputFile2.getPath(), null, sessionId).first().getIndex().getStatus().getName()); }
@Test public void testUpdateIndexStatus() throws CatalogException { QueryResult<File> fileResult = fileManager.create(studyFqn, File.Type.FILE, File.Format.VCF, File.Bioformat.VARIANT, "data/test.vcf", "", "description", new File.FileStatus(File.FileStatus.STAGE), 0, -1, Collections.emptyList(), -1, Collections.emptyMap(), Collections.emptyMap(), true, null, new QueryOptions(), sessionIdUser); fileManager.updateFileIndexStatus(fileResult.first(), FileIndex.IndexStatus.TRANSFORMED, null, sessionIdUser); QueryResult<File> read = fileManager.get(studyFqn, fileResult.first().getPath(), new QueryOptions(), sessionIdUser); assertEquals(FileIndex.IndexStatus.TRANSFORMED, read.first().getIndex().getStatus().getName()); }
@Test public void testBySteps2() throws Exception { File transformFile = transformFile(inputFile2, new QueryOptions()); Assert.assertNull(catalogManager.getFileManager().get(studyId, inputFile1.getPath(), null, sessionId).first().getIndex().getStatus()); Assert.assertEquals(FileIndex.IndexStatus.TRANSFORMED, catalogManager.getFileManager().get(studyId, inputFile2.getPath(), null, sessionId).first().getIndex().getStatus().getName()); loadFile(transformFile, new QueryOptions(), outputId); Assert.assertNull(catalogManager.getFileManager().get(studyId, inputFile1.getPath(), null, sessionId).first().getIndex().getStatus()); Assert.assertEquals(FileIndex.IndexStatus.READY, catalogManager.getFileManager().get(studyId, inputFile2.getPath(), null, sessionId).first().getIndex().getStatus().getName()); } }
@Test public void testIndex1() throws Exception { indexFile(inputFile1, new QueryOptions(), outputId); Assert.assertEquals(FileIndex.IndexStatus.READY, catalogManager.getFileManager().get(studyId, inputFile1.getPath(), null, sessionId).first().getIndex().getStatus().getName()); Assert.assertNull(catalogManager.getFileManager().get(studyId, inputFile2.getPath(), null, sessionId).first().getIndex().getStatus()); }