/** * Validate downloaded S3 files per storage unit information. * * @param baseDirectory the local parent directory path, relative to which the files are expected to be located * @param s3KeyPrefix the S3 key prefix that was prepended to the S3 file paths, when they were uploaded to S3 * @param storageUnit the storage unit that contains a list of storage files to be validated * * @throws IllegalStateException if files are not valid */ public void validateDownloadedS3Files(String baseDirectory, String s3KeyPrefix, StorageUnit storageUnit) throws IllegalStateException { validateDownloadedS3Files(baseDirectory, s3KeyPrefix, storageUnit.getStorageFiles()); }
/** * Retrieves and validates a list of storage files registered with the specified storage unit. This method makes sure that the list of storage files is not * empty and that all storage files match the expected s3 key prefix value. * * @param storageUnitEntity the storage unit entity the storage file paths to be validated * @param s3KeyPrefix the S3 key prefix that storage file paths are expected to start with * @param storageName the storage name * @param businessObjectDataKey the business object data key * * @return the list of storage files */ public List<StorageFile> getAndValidateStorageFiles(StorageUnitEntity storageUnitEntity, String s3KeyPrefix, String storageName, BusinessObjectDataKey businessObjectDataKey) { // Check if the list of storage files is not empty. if (CollectionUtils.isEmpty(storageUnitEntity.getStorageFiles())) { throw new IllegalArgumentException(String .format("Business object data has no storage files registered in \"%s\" storage. Business object data: {%s}", storageName, businessObjectDataHelper.businessObjectDataKeyToString(businessObjectDataKey))); } // Retrieve storage files. List<StorageFile> storageFiles = createStorageFilesFromEntities(storageUnitEntity.getStorageFiles()); // Validate storage file paths registered with this business object data in the specified storage. validateStorageFilePaths(getFilePathsFromStorageFiles(storageFiles), s3KeyPrefix, storageUnitEntity.getBusinessObjectData(), storageName); // Return the list of storage files. return storageFiles; }
.validateRegisteredS3Files(businessObjectDataRestoreDto.getStorageFiles(), actualS3Files, businessObjectDataRestoreDto.getStorageName(), businessObjectDataRestoreDto.getBusinessObjectDataKey()); s3FileTransferRequestParamsDto.setFiles(storageFileHelper.getFiles(storageFileHelper.createStorageFilesFromS3ObjectSummaries(glacierS3Files)));
Set<String> actualS3FilePaths = new HashSet<>(getFilePathsFromS3ObjectSummaries(s3ObjectSummaries)); Set<String> expectedFilePaths = new HashSet<>(getFilePathsFromStorageFiles(expectedStorageFiles));
List<String> actualS3Files = storageFileHelper.getFilePathsFromS3ObjectSummaries(s3Service.listDirectory(s3FileTransferRequestParamsDto, true)); storageFileHelper.validateStorageUnitS3Files(storageUnit, actualS3Files, s3KeyPrefixInformation.getS3KeyPrefix()); storageFileHelper.validateDownloadedS3Files(s3FileTransferRequestParamsDto.getLocalPath(), s3KeyPrefixInformation.getS3KeyPrefix(), storageUnit);
/** * Retrieves and validates a list of storage files registered with the specified storage unit. This method returns an empty list if storage unit has no * storage files. * * @param storageUnitEntity the storage unit entity the storage file paths to be validated * @param s3KeyPrefix the S3 key prefix that storage file paths are expected to start with * @param storageName the storage name * @param businessObjectDataKey the business object data key * * @return the list of storage files */ public List<StorageFile> getAndValidateStorageFilesIfPresent(StorageUnitEntity storageUnitEntity, String s3KeyPrefix, String storageName, BusinessObjectDataKey businessObjectDataKey) { return CollectionUtils.isEmpty(storageUnitEntity.getStorageFiles()) ? new ArrayList<>() : getAndValidateStorageFiles(storageUnitEntity, s3KeyPrefix, storageName, businessObjectDataKey); }
@Test public void testValidateRegisteredS3FilesUnexpectedZeroByteS3FileFound() throws IOException { // Create two lists of expected and actual storage files, with an actual file not being added to the list of expected files. List<StorageFile> testExpectedFiles = new ArrayList<>(); List<S3ObjectSummary> testActualFiles = Collections.singletonList(createS3ObjectSummary(TARGET_S3_KEY, FILE_SIZE_0_BYTE)); // Create a business object data key. BusinessObjectDataKey businessObjectDataKey = new BusinessObjectDataKey(BDEF_NAMESPACE, BDEF_NAME, FORMAT_USAGE_CODE, FORMAT_FILE_TYPE_CODE, FORMAT_VERSION, PARTITION_VALUE, SUBPARTITION_VALUES, DATA_VERSION); // Validate the files. The validation is expected not to fail when detecting an unregistered zero byte S3 file. storageFileHelper.validateRegisteredS3Files(testExpectedFiles, testActualFiles, STORAGE_NAME, businessObjectDataKey); }
/** * Creates and populates a business object data storage unit create response. * * @param storageUnitEntity the storage unit entity * * @return the business object data storage unit create response */ protected BusinessObjectDataStorageUnitCreateResponse createBusinessObjectDataStorageUnitCreateResponse(StorageUnitEntity storageUnitEntity) { // Get business object data key from the business object data entity. BusinessObjectDataKey businessObjectDataKey = businessObjectDataHelper.createBusinessObjectDataKeyFromEntity(storageUnitEntity.getBusinessObjectData()); // Create a business object data storage unit create response. BusinessObjectDataStorageUnitCreateResponse response = new BusinessObjectDataStorageUnitCreateResponse(); // Add business object data storage unit key. response.setBusinessObjectDataStorageUnitKey( storageUnitHelper.createBusinessObjectDataStorageUnitKey(businessObjectDataKey, storageUnitEntity.getStorage().getName())); // Add storage directory. if (storageUnitEntity.getDirectoryPath() != null) { response.setStorageDirectory(new StorageDirectory(storageUnitEntity.getDirectoryPath())); } // Add storage files. if (CollectionUtils.isNotEmpty(storageUnitEntity.getStorageFiles())) { response.setStorageFiles(storageFileHelper.createStorageFilesFromEntities(storageUnitEntity.getStorageFiles())); } // Return the response. return response; }
when(businessObjectDataHelper.createBusinessObjectDataKeyFromEntity(businessObjectDataEntity)).thenReturn(businessObjectDataKey); when(storageUnitHelper.createBusinessObjectDataStorageUnitKey(businessObjectDataKey, STORAGE_NAME)).thenReturn(businessObjectDataStorageUnitKey); when(storageFileHelper.createStorageFilesFromEntities(storageFileEntities)).thenReturn(storageFiles); verify(storageFileHelper).validateCreateRequestStorageFiles(storageFiles); verify(storageUnitHelper).getBusinessObjectDataKey(businessObjectDataStorageUnitKey); verify(businessObjectDataDaoHelper).getBusinessObjectDataEntity(businessObjectDataKey); verify(businessObjectDataHelper).createBusinessObjectDataKeyFromEntity(businessObjectDataEntity); verify(storageUnitHelper).createBusinessObjectDataStorageUnitKey(businessObjectDataKey, STORAGE_NAME); verify(storageFileHelper).createStorageFilesFromEntities(storageFileEntities); verify(storageUnitDao).saveAndRefresh(storageUnitEntity); verifyNoMoreInteractionsHelper();
@Test public void testValidateStorageUnitS3Files() throws IOException { StorageUnit storageUnit = createStorageUnit(TEST_S3_KEY_PREFIX, LOCAL_FILES, FILE_SIZE_1_KB); List<String> actualS3Files = new ArrayList<>(); for (String file : LOCAL_FILES) { actualS3Files.add(String.format("%s/%s", TEST_S3_KEY_PREFIX, file)); } storageFileHelper.validateStorageUnitS3Files(storageUnit, actualS3Files, TEST_S3_KEY_PREFIX); }
storageFileHelper.validateCreateRequestStorageFiles(businessObjectDataStorageFilesCreateRequest.getStorageFiles());
when(storageHelper.getStorageAttributeValueByName(S3_ATTRIBUTE_NAME_BUCKET_NAME, storageEntity, true)).thenReturn(S3_BUCKET_NAME); when(s3KeyPrefixHelper.buildS3KeyPrefix(storageEntity, businessObjectFormatEntity, businessObjectDataKey)).thenReturn(TEST_S3_KEY_PREFIX); when(storageFileHelper.getAndValidateStorageFilesIfPresent(storageUnitEntity, TEST_S3_KEY_PREFIX, STORAGE_NAME, businessObjectDataKey)) .thenReturn(storageFiles); doAnswer(new Answer<Void>() verify(storageHelper).getStorageAttributeValueByName(S3_ATTRIBUTE_NAME_BUCKET_NAME, storageEntity, true); verify(s3KeyPrefixHelper).buildS3KeyPrefix(storageEntity, businessObjectFormatEntity, businessObjectDataKey); verify(storageFileHelper).getAndValidateStorageFilesIfPresent(storageUnitEntity, TEST_S3_KEY_PREFIX, STORAGE_NAME, businessObjectDataKey); verify(storageFileDaoHelper).validateStorageFilesCount(STORAGE_NAME, businessObjectDataKey, TEST_S3_KEY_PREFIX, storageFileEntities.size()); verify(storageUnitDaoHelper).updateStorageUnitStatus(storageUnitEntity, StorageUnitStatusEntity.DISABLING, StorageUnitStatusEntity.DISABLING);
/** * Returns a list of S3 object keys associated with the given format, data key, and storage. The keys are found by matching the prefix. The result may be * empty if there are not matching keys found. * * @param businessObjectFormatEntity {@link BusinessObjectFormatEntity} * @param businessObjectDataKey {@link BusinessObjectDataKey} * @param storageEntity {@link StorageEntity} * * @return list of S3 object keys */ private List<String> getS3ObjectKeys(BusinessObjectFormatEntity businessObjectFormatEntity, BusinessObjectDataKey businessObjectDataKey, StorageEntity storageEntity) { String s3KeyPrefix = s3KeyPrefixHelper.buildS3KeyPrefix(storageEntity, businessObjectFormatEntity, businessObjectDataKey); S3FileTransferRequestParamsDto s3FileTransferRequestParamsDto = storageHelper.getS3BucketAccessParams(storageEntity); s3FileTransferRequestParamsDto.setS3KeyPrefix(s3KeyPrefix + '/'); return storageFileHelper.getFilePathsFromS3ObjectSummaries(s3Dao.listDirectory(s3FileTransferRequestParamsDto)); }
/** * Discovers storage files in the specified S3 storage that match the S3 key prefix. * * @param storageEntity the storage entity * @param s3KeyPrefix the S3 key prefix * * @return the list of discovered storage files */ private List<StorageFile> discoverStorageFiles(StorageEntity storageEntity, String s3KeyPrefix) { // Only S3 storage platform is currently supported for storage file discovery. Assert.isTrue(storageEntity.getStoragePlatform().getName().equals(StoragePlatformEntity.S3), String.format("Cannot discover storage files at \"%s\" storage platform.", storageEntity.getStoragePlatform().getName())); // Get S3 bucket access parameters. S3FileTransferRequestParamsDto params = storageHelper.getS3BucketAccessParams(storageEntity); // Retrieve a list of all keys/objects from the S3 bucket matching the specified S3 key prefix. // Since S3 key prefix represents the directory, we add a trailing '/' character to it, unless it is already present. params.setS3KeyPrefix(StringUtils.appendIfMissing(s3KeyPrefix, "/")); // When listing S3 files, we ignore 0 byte objects that represent S3 directories. List<S3ObjectSummary> s3ObjectSummaries = s3Service.listDirectory(params, true); // Fail registration if no storage files were discovered. if (CollectionUtils.isEmpty(s3ObjectSummaries)) { throw new ObjectNotFoundException(String.format("Found no files at \"s3://%s/%s\" location.", params.getS3BucketName(), params.getS3KeyPrefix())); } return storageFileHelper.createStorageFilesFromS3ObjectSummaries(s3ObjectSummaries); }
/** * Retrieves and validates a list of storage files registered with the specified storage unit. This method returns an empty list if storage unit has no * storage files. * * @param storageUnitEntity the storage unit entity the storage file paths to be validated * @param s3KeyPrefix the S3 key prefix that storage file paths are expected to start with * @param storageName the storage name * @param businessObjectDataKey the business object data key * * @return the list of storage files */ public List<StorageFile> getAndValidateStorageFilesIfPresent(StorageUnitEntity storageUnitEntity, String s3KeyPrefix, String storageName, BusinessObjectDataKey businessObjectDataKey) { return CollectionUtils.isEmpty(storageUnitEntity.getStorageFiles()) ? new ArrayList<>() : getAndValidateStorageFiles(storageUnitEntity, s3KeyPrefix, storageName, businessObjectDataKey); }
@Test public void testValidateRegisteredS3Files() throws IOException { // Create two lists of expected and actual storage files. // Please note we use different row count values to confirm that row count match is not validated. List<StorageFile> testExpectedFiles = new ArrayList<>(); List<S3ObjectSummary> testActualFiles = new ArrayList<>(); for (String file : LOCAL_FILES) { String filePath = String.format("%s/%s", TEST_S3_KEY_PREFIX, file); testExpectedFiles.add(new StorageFile(filePath, FILE_SIZE, ROW_COUNT)); testActualFiles.add(createS3ObjectSummary(filePath, FILE_SIZE)); } // Validate the files. storageFileHelper.validateRegisteredS3Files(testExpectedFiles, testActualFiles, STORAGE_NAME, new BusinessObjectDataKey(BDEF_NAMESPACE, BDEF_NAME, FORMAT_USAGE_CODE, FORMAT_FILE_TYPE_CODE, FORMAT_VERSION, PARTITION_VALUE, SUBPARTITION_VALUES, DATA_VERSION)); }
/** * Creates and populates a business object data storage unit create response. * * @param storageUnitEntity the storage unit entity * * @return the business object data storage unit create response */ protected BusinessObjectDataStorageUnitCreateResponse createBusinessObjectDataStorageUnitCreateResponse(StorageUnitEntity storageUnitEntity) { // Get business object data key from the business object data entity. BusinessObjectDataKey businessObjectDataKey = businessObjectDataHelper.createBusinessObjectDataKeyFromEntity(storageUnitEntity.getBusinessObjectData()); // Create a business object data storage unit create response. BusinessObjectDataStorageUnitCreateResponse response = new BusinessObjectDataStorageUnitCreateResponse(); // Add business object data storage unit key. response.setBusinessObjectDataStorageUnitKey( storageUnitHelper.createBusinessObjectDataStorageUnitKey(businessObjectDataKey, storageUnitEntity.getStorage().getName())); // Add storage directory. if (storageUnitEntity.getDirectoryPath() != null) { response.setStorageDirectory(new StorageDirectory(storageUnitEntity.getDirectoryPath())); } // Add storage files. if (CollectionUtils.isNotEmpty(storageUnitEntity.getStorageFiles())) { response.setStorageFiles(storageFileHelper.createStorageFilesFromEntities(storageUnitEntity.getStorageFiles())); } // Return the response. return response; }
Set<String> actualS3FilePaths = new HashSet<>(getFilePathsFromS3ObjectSummaries(s3ObjectSummaries)); Set<String> expectedFilePaths = new HashSet<>(getFilePathsFromStorageFiles(expectedStorageFiles));
@Test public void testValidateStorageUnitS3FilesNotRegisteredS3FileFound() throws IOException { StorageUnit storageUnit = createStorageUnit(TEST_S3_KEY_PREFIX, new ArrayList<>(), FILE_SIZE_1_KB); List<String> actualS3Files = Collections.singletonList(String.format("%s/%s", TEST_S3_KEY_PREFIX, LOCAL_FILES.get(0))); // Try to validate S3 files when we have not registered S3 file. try { storageFileHelper.validateStorageUnitS3Files(storageUnit, actualS3Files, TEST_S3_KEY_PREFIX); fail("Should throw a RuntimeException when S3 contains unregistered S3 file."); } catch (RuntimeException e) { String expectedErrMsg = String.format("Found S3 file \"%s\" in \"%s\" storage not registered with this business object data.", actualS3Files.get(0), storageUnit.getStorage().getName()); assertEquals(expectedErrMsg, e.getMessage()); } }
storageFileHelper.validateCreateRequestStorageFiles(businessObjectDataStorageFilesCreateRequest.getStorageFiles());