/** * Logs all files found in the specified S3 location. * * @param params the S3 file transfer request parameters */ private void logS3KeyPrefixContents(S3FileTransferRequestParamsDto params) { List<S3ObjectSummary> s3ObjectSummaries = s3Service.listDirectory(params); LOGGER.info( String.format("Found %d keys with prefix \"%s\" in bucket \"%s\":", s3ObjectSummaries.size(), params.getS3KeyPrefix(), params.getS3BucketName())); for (S3ObjectSummary s3ObjectSummary : s3ObjectSummaries) { LOGGER.info(String.format(" s3://%s/%s", params.getS3BucketName(), s3ObjectSummary.getKey())); } } }
Map<String, StorageFile> actualS3Keys = storageFileHelper.getStorageFilesMapFromS3ObjectSummaries(s3Service.listDirectory(params, true));
Map<String, StorageFile> actualS3Keys = storageFileHelper.getStorageFilesMapFromS3ObjectSummaries(s3Service.listDirectory(params, true));
/** * Discovers storage files in the specified S3 storage that match the S3 key prefix. * * @param storageEntity the storage entity * @param s3KeyPrefix the S3 key prefix * * @return the list of discovered storage files */ private List<StorageFile> discoverStorageFiles(StorageEntity storageEntity, String s3KeyPrefix) { // Only S3 storage platform is currently supported for storage file discovery. Assert.isTrue(storageEntity.getStoragePlatform().getName().equals(StoragePlatformEntity.S3), String.format("Cannot discover storage files at \"%s\" storage platform.", storageEntity.getStoragePlatform().getName())); // Get S3 bucket access parameters. S3FileTransferRequestParamsDto params = storageHelper.getS3BucketAccessParams(storageEntity); // Retrieve a list of all keys/objects from the S3 bucket matching the specified S3 key prefix. // Since S3 key prefix represents the directory, we add a trailing '/' character to it, unless it is already present. params.setS3KeyPrefix(StringUtils.appendIfMissing(s3KeyPrefix, "/")); // When listing S3 files, we ignore 0 byte objects that represent S3 directories. List<S3ObjectSummary> s3ObjectSummaries = s3Service.listDirectory(params, true); // Fail registration if no storage files were discovered. if (CollectionUtils.isEmpty(s3ObjectSummaries)) { throw new ObjectNotFoundException(String.format("Found no files at \"s3://%s/%s\" location.", params.getS3BucketName(), params.getS3KeyPrefix())); } return storageFileHelper.createStorageFilesFromS3ObjectSummaries(s3ObjectSummaries); }
/** * Discovers storage files in the specified S3 storage that match the S3 key prefix. * * @param storageEntity the storage entity * @param s3KeyPrefix the S3 key prefix * * @return the list of discovered storage files */ private List<StorageFile> discoverStorageFiles(StorageEntity storageEntity, String s3KeyPrefix) { // Only S3 storage platform is currently supported for storage file discovery. Assert.isTrue(storageEntity.getStoragePlatform().getName().equals(StoragePlatformEntity.S3), String.format("Cannot discover storage files at \"%s\" storage platform.", storageEntity.getStoragePlatform().getName())); // Get S3 bucket access parameters. S3FileTransferRequestParamsDto params = storageHelper.getS3BucketAccessParams(storageEntity); // Retrieve a list of all keys/objects from the S3 bucket matching the specified S3 key prefix. // Since S3 key prefix represents the directory, we add a trailing '/' character to it, unless it is already present. params.setS3KeyPrefix(StringUtils.appendIfMissing(s3KeyPrefix, "/")); // When listing S3 files, we ignore 0 byte objects that represent S3 directories. List<S3ObjectSummary> s3ObjectSummaries = s3Service.listDirectory(params, true); // Fail registration if no storage files were discovered. if (CollectionUtils.isEmpty(s3ObjectSummaries)) { throw new ObjectNotFoundException(String.format("Found no files at \"s3://%s/%s\" location.", params.getS3BucketName(), params.getS3KeyPrefix())); } return storageFileHelper.createStorageFilesFromS3ObjectSummaries(s3ObjectSummaries); }
/** * Cleans up the destination S3 key prefixes used by the uploader unit tests. */ protected void cleanupS3() { // Delete the test business object data versions. S3FileTransferRequestParamsDto s3FileTransferRequestParamsDto = getTestS3FileTransferRequestParamsDto(); for (String s3KeyPrefix : new String[] {S3_TEST_PARENT_PATH_V0, S3_TEST_PARENT_PATH_V1, S3_TEST_PATH_V0, S3_TEST_PATH_V1, S3_SIMPLE_TEST_PATH}) { s3FileTransferRequestParamsDto.setS3KeyPrefix(s3KeyPrefix); if (!s3Service.listDirectory(s3FileTransferRequestParamsDto).isEmpty()) { s3Service.deleteDirectory(s3FileTransferRequestParamsDto); } } }
/** * TODO: We need the herd web service mocking done and this test case rewritten, so it would fail right at the end of performUpload() method (on the * business object data registration step) and triggered the rollbackUpload() to occur. */ @Test(expected = RuntimeException.class) public void testPerformUploadRegistrationError() throws Exception { // Upload and register business object data parents. uploadAndRegisterTestDataParents(uploaderWebClient); runUpload(UploaderController.MIN_THREADS); // Clean up the local directory. FileUtils.deleteDirectory(LOCAL_TEMP_PATH_INPUT.toFile()); // Clean up the destination S3 folder. S3FileTransferRequestParamsDto s3FileTransferRequestParamsDto = getTestS3FileTransferRequestParamsDto(S3_TEST_PATH_V0); if (!s3Service.listDirectory(s3FileTransferRequestParamsDto).isEmpty()) { s3Service.deleteDirectory(s3FileTransferRequestParamsDto); } runUpload(UploaderController.MIN_THREADS); }
when(s3Service.listDirectory(updatedS3FileTransferRequestParamsDto, true)).thenReturn(s3Files); when(storageFileHelper.createStorageFilesFromS3ObjectSummaries(glacierS3Files)).thenReturn(glacierStorageFiles); when(storageFileHelper.getFiles(glacierStorageFiles)).thenReturn(files); verify(s3Service).listDirectory(any(S3FileTransferRequestParamsDto.class), eq(true)); verify(storageFileHelper).validateRegisteredS3Files(storageFiles, s3Files, STORAGE_NAME, businessObjectDataKey); verify(storageFileHelper).createStorageFilesFromS3ObjectSummaries(glacierS3Files);
List<S3ObjectSummary> actualS3Files = s3Service.listDirectory(s3FileTransferRequestParamsDto, true);
List<S3ObjectSummary> actualS3Files = s3Service.listDirectory(s3FileTransferRequestParamsDto, true);
List<S3ObjectSummary> actualS3Files = s3Service.listDirectory(s3FileTransferRequestParamsDto, true);
List<S3ObjectSummary> actualS3Files = s3Service.listDirectory(s3FileTransferRequestParamsDto, true);
when(s3Service.listDirectory(updatedS3FileTransferRequestParamsDto, true)).thenReturn(actualS3Files); when(storageFileHelper.createStorageFilesFromS3ObjectSummaries(actualS3Files)).thenReturn(storageFilesCreatedFromActualS3Files); when(storageFileHelper.getFiles(storageFilesCreatedFromActualS3Files)).thenReturn(filesToBeRestored); verify(s3Service).listDirectory(any(S3FileTransferRequestParamsDto.class), eq(true)); verify(storageFileHelper).validateRegisteredS3Files(storageFiles, actualS3Files, STORAGE_NAME, businessObjectDataKey); verify(storageFileHelper).createStorageFilesFromS3ObjectSummaries(actualS3Files);
when(s3Service.listDirectory(updatedS3FileTransferRequestParamsDto, true)).thenReturn(actualS3Files); verify(s3Service).listDirectory(any(S3FileTransferRequestParamsDto.class), eq(true)); verify(storageFileHelper).validateRegisteredS3Files(storageFiles, actualS3Files, STORAGE_NAME, businessObjectDataKey); verify(jsonHelper).objectToJson(businessObjectDataKey);
assertEquals(manifestFiles.size() + directoryPaths.size(), s3Service.listDirectory(s3FileTransferRequestParamsDto).size());
assertEquals(localFilePaths.size() + directoryPaths.size(), s3Service.listDirectory(s3FileTransferRequestParamsDto).size());
List<S3ObjectSummary> actualS3Files = s3Service.listDirectory(s3FileTransferRequestParamsDto, true);
List<S3ObjectSummary> actualS3FilesWithoutZeroByteDirectoryMarkers = s3Service.listDirectory(s3FileTransferRequestParamsDto, true); List<S3ObjectSummary> actualS3Files = s3Service.listDirectory(s3FileTransferRequestParamsDto, false);
List<S3ObjectSummary> actualS3FilesWithoutZeroByteDirectoryMarkers = s3Service.listDirectory(s3FileTransferRequestParamsDto, true); List<S3ObjectSummary> actualS3Files = s3Service.listDirectory(s3FileTransferRequestParamsDto, false);
List<S3ObjectSummary> actualS3Files = s3Service.listDirectory(s3FileTransferRequestParamsDto, true);