@Override public URI load(URI input) throws IOException, StorageEngineException { logger.info("Loading file " + input); List<Integer> fileIds = Collections.singletonList(getFileId()); if (getOptions().getInt(LOAD_SLEEP) > 0) { try { Thread.sleep(getOptions().getInt(LOAD_SLEEP)); } catch (InterruptedException e) { Thread.currentThread().interrupt(); throw new StorageEngineException("Interrupted", e); } } if (getOptions().getBoolean(VARIANTS_LOAD_FAIL) || getOptions().getString(VARIANTS_LOAD_FAIL).equals(Paths.get(input).getFileName().toString())) { getStudyConfigurationManager().atomicSetStatus(getStudyId(), BatchFileOperation.Status.ERROR, "load", fileIds); throw new StorageEngineException("Error loading file " + input); } else { getStudyConfigurationManager().atomicSetStatus(getStudyId(), BatchFileOperation.Status.DONE, "load", fileIds); } return input; }
protected DummyVariantStoragePipeline mockVariantStorageETL(DummyVariantStorageEngine vsm) throws StorageEngineException { DummyVariantStoragePipeline storageETL = spy(vsm.newStoragePipeline(true)); // doReturn(storageETL).when(vsm).newStoragePipeline(anyBoolean()); Mockito.doAnswer(invocation -> { DummyVariantStoragePipeline etl = (DummyVariantStoragePipeline) invocation.callRealMethod(); storageETL.init(etl.getOptions()); return storageETL; }).when(vsm).newStoragePipeline(anyBoolean()); return storageETL; }
@Override public URI postLoad(URI input, URI output) throws StorageEngineException { logger.info("Post load file " + input); VariantFileMetadata fileMetadata = readVariantFileMetadata(input); fileMetadata.setId(String.valueOf(getFileId())); dbAdaptor.getStudyConfigurationManager().updateVariantFileMetadata(getStudyId(), fileMetadata); return super.postLoad(input, output); }
public void init(ObjectMap options) { getOptions().clear(); getOptions().putAll(options); }
@Override protected void securePreLoad(StudyConfiguration studyConfiguration, VariantFileMetadata source) throws StorageEngineException { super.securePreLoad(studyConfiguration, source); List<Integer> fileIds = Collections.singletonList(getFileId()); BatchFileOperation op = new BatchFileOperation("load", fileIds, 1, BatchFileOperation.Type.LOAD); op.addStatus(BatchFileOperation.Status.RUNNING); studyConfiguration.getBatches().add(op); }
@Override public DummyVariantStoragePipeline newStoragePipeline(boolean connected) throws StorageEngineException { return new DummyVariantStoragePipeline(getConfiguration(), STORAGE_ENGINE_ID, getDBAdaptor(), getVariantReaderUtils()); }
@Test public void testIndexWithTransformError() throws Exception { QueryOptions queryOptions = new QueryOptions(VariantStorageEngine.Options.ANNOTATE.key(), false) .append(VariantStorageEngine.Options.CALCULATE_STATS.key(), false); DummyVariantStoragePipeline storageETL = mockVariantStorageETL(); List<File> files = Arrays.asList(getFile(0), getFile(1)); StorageEngineException transformException = StorageEngineException.unableToExecute("transform", 0, ""); Mockito.doThrow(transformException).when(storageETL) .transform(ArgumentMatchers.argThat(argument -> argument.toString().contains(files.get(1).getName())), Mockito.any(), Mockito.any()); try { indexFiles(files, queryOptions, outputId); } catch (StoragePipelineException exception) { assertEquals(files.size(), exception.getResults().size()); assertTrue(exception.getResults().get(0).isTransformExecuted()); assertNull(exception.getResults().get(0).getTransformError()); assertTrue(exception.getResults().get(1).isTransformExecuted()); assertSame(transformException, exception.getResults().get(1).getTransformError()); for (int i = files.size(); i > 0; i--) { assertFalse(exception.getResults().get(1).isLoadExecuted()); assertNull(exception.getResults().get(1).getLoadError()); } } mockVariantStorageETL(); // File 0 already transformed. // Expecting to transform and load only file 1 indexFiles(files, singletonList(files.get(1)), queryOptions, outputId); }