@Override public S3FileTransferResultsDto copyFile(S3FileCopyRequestParamsDto params) throws InterruptedException { return s3Dao.copyFile(params); }
@Override public void createDirectory(S3FileTransferRequestParamsDto params) { s3Dao.createDirectory(params); }
@Override public void deleteDirectory(S3FileTransferRequestParamsDto params) { s3Dao.deleteDirectory(params); }
@Override public List<S3ObjectSummary> listDirectory(S3FileTransferRequestParamsDto params, boolean ignoreZeroByteDirectoryMarkers) { return s3Dao.listDirectory(params, ignoreZeroByteDirectoryMarkers); }
/** * Test that we are able to perform the deleteDirectory S3Dao operation on S3 using our DAO tier. */ @Test public void testDeleteDirectory() throws IOException, InterruptedException { // Upload local directory to s3Dao. testUploadDirectory(); // Validate that S3 directory is not empty. S3FileTransferRequestParamsDto s3FileTransferRequestParamsDto = s3DaoTestHelper.getTestS3FileTransferRequestParamsDto(); s3FileTransferRequestParamsDto.setS3KeyPrefix(TEST_S3_KEY_PREFIX + "/"); Assert.assertTrue(s3Dao.listDirectory(s3FileTransferRequestParamsDto).size() > 0); // Delete directory from S3 using s3Dao. s3Dao.deleteDirectory(s3FileTransferRequestParamsDto); // Validate that S3 directory got deleted. Assert.assertEquals(0, s3Dao.listDirectory(s3FileTransferRequestParamsDto).size()); }
@Override public void restoreObjects(final S3FileTransferRequestParamsDto params, int expirationInDays) { s3Dao.restoreObjects(params, expirationInDays); }
/** * Test that we are able to perform the uploadDirectory S3Dao operation on a folder with 0 bytes of data using our DAO tier. */ @Test public void testUploadDirectoryZeroBytes() throws IOException, InterruptedException { // Create a zero size local file. File targetFile = createLocalFile(localTempPath.toString(), LOCAL_FILE, FILE_SIZE_0_BYTE); Assert.assertTrue(targetFile.isFile()); // Upload empty folder to s3Dao. S3FileTransferRequestParamsDto s3FileTransferRequestParamsDto = s3DaoTestHelper.getTestS3FileTransferRequestParamsDto(); s3FileTransferRequestParamsDto.setS3KeyPrefix(TEST_S3_KEY_PREFIX); s3FileTransferRequestParamsDto.setLocalPath(localTempPath.toString()); s3FileTransferRequestParamsDto.setRecursive(true); S3FileTransferResultsDto results = s3Dao.uploadDirectory(s3FileTransferRequestParamsDto); // Validate results. Assert.assertTrue(results.getTotalFilesTransferred() == 1L); // Validate the zero bytes upload. Assert.assertEquals(1, s3Dao.listDirectory(s3FileTransferRequestParamsDto).size()); }
verify(storageUnitDaoHelper).getStorageUnitEntity(StorageEntity.MANAGED_LOADING_DOCK_STORAGE, sourceBusinessObjectDataEntity); verify(awsHelper, times(2)).getAwsParamsDto(); verify(s3Dao).validateS3File(any(S3FileTransferRequestParamsDto.class), eq(FILE_SIZE)); verify(storageHelper).getStorageBucketName(targetStorageEntity); verify(storageHelper).getStorageKmsKeyId(targetStorageEntity); verify(s3Dao).s3FileExists(any(S3FileTransferRequestParamsDto.class)); verify(businessObjectDataDaoHelper).updateBusinessObjectDataStatus(sourceBusinessObjectDataEntity, BusinessObjectDataStatusEntity.RE_ENCRYPTING); verify(businessObjectDataDaoHelper).updateBusinessObjectDataStatus(targetBusinessObjectDataEntity, BusinessObjectDataStatusEntity.RE_ENCRYPTING);
/** * Test that we are able to perform the deleteFileList S3Dao operation on S3 using our DAO tier. */ @Test public void testDeleteFileList() throws IOException, InterruptedException { // Upload local directory to s3Dao. testUploadDirectory(); // Validate that S3 directory is not empty. S3FileTransferRequestParamsDto s3FileTransferRequestParamsDto = s3DaoTestHelper.getTestS3FileTransferRequestParamsDto(); s3FileTransferRequestParamsDto.setS3KeyPrefix(TEST_S3_KEY_PREFIX + "/"); List<S3ObjectSummary> s3ObjectSummaries = s3Dao.listDirectory(s3FileTransferRequestParamsDto); Assert.assertTrue(s3ObjectSummaries.size() > 0); // Delete a list of files from S3 using s3Dao. List<File> requestFileList = new ArrayList<>(); for (S3ObjectSummary s3ObjectSummary : s3ObjectSummaries) { requestFileList.add(new File(s3ObjectSummary.getKey())); } s3FileTransferRequestParamsDto.setFiles(requestFileList); s3FileTransferRequestParamsDto.setS3KeyPrefix(null); s3Dao.deleteFileList(s3FileTransferRequestParamsDto); // Validate that S3 directory got deleted. s3FileTransferRequestParamsDto.setS3KeyPrefix(TEST_S3_KEY_PREFIX + "/"); Assert.assertEquals(0, s3Dao.listDirectory(s3FileTransferRequestParamsDto).size()); }
@Override public void tagObjects(final S3FileTransferRequestParamsDto s3FileTransferRequestParamsDto, final S3FileTransferRequestParamsDto s3ObjectTaggerParamsDto, final List<S3ObjectSummary> s3ObjectSummaries, final Tag tag) { s3Dao.tagObjects(s3FileTransferRequestParamsDto, s3ObjectTaggerParamsDto, s3ObjectSummaries, tag); }
@Override public void assertS3ObjectKeyDoesNotExist(String bucketName, String key) { S3FileTransferRequestParamsDto s3FileTransferRequestParamsDto = new S3FileTransferRequestParamsDto(); s3FileTransferRequestParamsDto.setS3BucketName(bucketName); s3FileTransferRequestParamsDto.setS3KeyPrefix(key); AwsParamsDto awsParamsDto = awsHelper.getAwsParamsDto(); String httpProxyHost = awsParamsDto.getHttpProxyHost(); s3FileTransferRequestParamsDto.setHttpProxyHost(httpProxyHost); Integer httpProxyPort = awsParamsDto.getHttpProxyPort(); s3FileTransferRequestParamsDto.setHttpProxyPort(httpProxyPort); Assert.isTrue(!s3Dao.s3FileExists(s3FileTransferRequestParamsDto), String.format("A S3 object already exists in bucket \"%s\" and key \"%s\".", bucketName, key)); }
@Override public void deleteFileList(S3FileTransferRequestParamsDto params) { s3Dao.deleteFileList(params); }
@Override public S3FileTransferResultsDto downloadFile(S3FileTransferRequestParamsDto params) throws InterruptedException { return s3Dao.downloadFile(params); }
@Override public S3FileTransferResultsDto downloadDirectory(S3FileTransferRequestParamsDto params) throws InterruptedException { return s3Dao.downloadDirectory(params); }
@Override public List<S3ObjectSummary> listDirectory(S3FileTransferRequestParamsDto params) { // By default, we do not ignore 0 byte objects that represent S3 directories. return s3Dao.listDirectory(params, false); }
/** * Cleans up the local temp directory and S3 test path that we are using. */ @After public void cleanEnv() throws IOException { // Clean up the local directory. FileUtils.deleteDirectory(localTempPath.toFile()); // Delete test files from S3 storage. Since test S3 key prefix represents a directory, we add a trailing '/' character to it. for (S3FileTransferRequestParamsDto params : Arrays.asList(s3DaoTestHelper.getTestS3FileTransferRequestParamsDto(), S3FileTransferRequestParamsDto.builder().withS3BucketName(storageDaoTestHelper.getS3LoadingDockBucketName()) .withS3KeyPrefix(TEST_S3_KEY_PREFIX + "/").build(), S3FileTransferRequestParamsDto.builder().withS3BucketName(storageDaoTestHelper.getS3ExternalBucketName()).withS3KeyPrefix(TEST_S3_KEY_PREFIX + "/") .build())) { if (!s3Dao.listDirectory(params).isEmpty()) { s3Dao.deleteDirectory(params); } } s3Operations.rollback(); }
@Override public void restoreObjects(final S3FileTransferRequestParamsDto params, int expirationInDays) { s3Dao.restoreObjects(params, expirationInDays); }
/** * Test that we are able to perform the uploadDirectory S3Dao operation on a folder with more than 1000 files using our DAO tier. This test is needed to * make AmazonS3.listObjects() to start truncating ObjectListing that it returns. */ @Test public void testUploadDirectoryLargeNumberOfFiles() throws IOException, InterruptedException { final int NUM_OF_FILES = 1001; // Create local files. for (int i = 0; i < NUM_OF_FILES; i++) { File targetFile = createLocalFile(localTempPath.toString(), String.format("%04d_%s", i, LOCAL_FILE), FILE_SIZE_0_BYTE); Assert.assertTrue(targetFile.isFile()); } // Upload empty folder to s3Dao. S3FileTransferRequestParamsDto s3FileTransferRequestParamsDto = s3DaoTestHelper.getTestS3FileTransferRequestParamsDto(); s3FileTransferRequestParamsDto.setS3KeyPrefix(TEST_S3_KEY_PREFIX); s3FileTransferRequestParamsDto.setLocalPath(localTempPath.toString()); s3FileTransferRequestParamsDto.setRecursive(true); S3FileTransferResultsDto results = s3Dao.uploadDirectory(s3FileTransferRequestParamsDto); // Validate results. Assert.assertTrue(results.getTotalFilesTransferred() == NUM_OF_FILES); // Validate the empty folder upload. Assert.assertEquals(NUM_OF_FILES, s3Dao.listDirectory(s3FileTransferRequestParamsDto).size()); }
verify(storageUnitDaoHelper).getStorageUnitEntity(StorageEntity.MANAGED_LOADING_DOCK_STORAGE, sourceBusinessObjectDataEntity); verify(awsHelper, times(2)).getAwsParamsDto(); verify(s3Dao).validateS3File(any(S3FileTransferRequestParamsDto.class), eq(FILE_SIZE)); verify(storageHelper).getStorageBucketName(targetStorageEntity); verify(storageHelper).getStorageKmsKeyId(targetStorageEntity); verify(s3Dao).s3FileExists(any(S3FileTransferRequestParamsDto.class)); verify(businessObjectDataDaoHelper).updateBusinessObjectDataStatus(sourceBusinessObjectDataEntity, BusinessObjectDataStatusEntity.RE_ENCRYPTING); verify(jsonHelper).objectToJson(sourceBusinessObjectDataKey);
@Override public void tagObjects(final S3FileTransferRequestParamsDto s3FileTransferRequestParamsDto, final S3FileTransferRequestParamsDto s3ObjectTaggerParamsDto, final List<S3ObjectSummary> s3ObjectSummaries, final Tag tag) { s3Dao.tagObjects(s3FileTransferRequestParamsDto, s3ObjectTaggerParamsDto, s3ObjectSummaries, tag); }