/** * Puts an S3 object with the given parameters directly into S3. * * @param s3BucketName the S3 bucket name * @param s3ObjectKey the S3 object key * @param content the content of the S3 object */ private void putS3Object(String s3BucketName, String s3ObjectKey, String content) { PutObjectRequest putObjectRequest = new PutObjectRequest(s3BucketName, s3ObjectKey, new ByteArrayInputStream(content.getBytes()), new ObjectMetadata()); s3Operations.putObject(putObjectRequest, null); } }
/** * Puts a Java properties file content into S3. The Properties contains a single key-value defined by the given parameter object. * * @param s3BucketName the S3 bucket name * @param s3ObjectKey the S3 object key * @param parameter the parameter */ private void putParameterIntoS3(String s3BucketName, String s3ObjectKey, Parameter parameter) { String content = parameter.getName() + "=" + parameter.getValue(); byte[] bytes = content.getBytes(); ByteArrayInputStream inputStream = new ByteArrayInputStream(bytes); ObjectMetadata metadata = new ObjectMetadata(); metadata.setContentLength(content.length()); PutObjectRequest putObjectRequest = new PutObjectRequest(s3BucketName, s3ObjectKey, inputStream, metadata); s3Operations.putObject(putObjectRequest, null); }
@Test public void testValidateS3File() throws IOException, InterruptedException { // Put a 1 KB file in S3. PutObjectRequest putObjectRequest = new PutObjectRequest(storageDaoTestHelper.getS3ManagedBucketName(), TARGET_S3_KEY, new ByteArrayInputStream(new byte[(int) FILE_SIZE_1_KB]), null); s3Operations.putObject(putObjectRequest, null); // Validate the S3 file. S3FileTransferRequestParamsDto s3FileTransferRequestParamsDto = s3DaoTestHelper.getTestS3FileTransferRequestParamsDto(); s3FileTransferRequestParamsDto.setS3KeyPrefix(TARGET_S3_KEY); s3Dao.validateS3File(s3FileTransferRequestParamsDto, FILE_SIZE_1_KB); }
@Test(expected = IllegalArgumentException.class) public void testValidateS3FileRuntimeExceptionFileSizeDoesNotMatch() throws IOException, InterruptedException { // Put a 1 KB file in S3. PutObjectRequest putObjectRequest = new PutObjectRequest(storageDaoTestHelper.getS3ManagedBucketName(), TARGET_S3_KEY, new ByteArrayInputStream(new byte[(int) FILE_SIZE_1_KB]), null); s3Operations.putObject(putObjectRequest, null); // Try to validate an S3 file by specifying an incorrect file size. S3FileTransferRequestParamsDto s3FileTransferRequestParamsDto = s3DaoTestHelper.getTestS3FileTransferRequestParamsDto(); s3FileTransferRequestParamsDto.setS3KeyPrefix(TARGET_S3_KEY); s3Dao.validateS3File(s3FileTransferRequestParamsDto, FILE_SIZE_1_KB + 999); }
/** * The method is successful when both bucket and key exists. */ @Test public void testGetProperties() { String expectedKey = "foo"; String expectedValue = "bar"; ByteArrayInputStream inputStream = new ByteArrayInputStream((expectedKey + "=" + expectedValue).getBytes()); PutObjectRequest putObjectRequest = new PutObjectRequest(S3_BUCKET_NAME, TARGET_S3_KEY, inputStream, new ObjectMetadata()); s3Operations.putObject(putObjectRequest, null); Properties properties = s3Dao.getProperties(S3_BUCKET_NAME, TARGET_S3_KEY, s3DaoTestHelper.getTestS3FileTransferRequestParamsDto()); Assert.assertEquals("properties key '" + expectedKey + "'", expectedValue, properties.get(expectedKey)); }
@Test public void testGetAmazonS3AssertV4SigningAlwaysPresent() { S3Operations originalS3Operations = (S3Operations) ReflectionTestUtils.getField(s3Dao, "s3Operations"); S3Operations mockS3Operations = mock(S3Operations.class); ReflectionTestUtils.setField(s3Dao, "s3Operations", mockS3Operations); try { String s3BucketName = "s3BucketName"; String s3KeyPrefix = "s3KeyPrefix"; S3FileTransferRequestParamsDto s3FileTransferRequestParamsDto = new S3FileTransferRequestParamsDto(); s3FileTransferRequestParamsDto.setS3BucketName(s3BucketName); s3FileTransferRequestParamsDto.setS3KeyPrefix(s3KeyPrefix); when(mockS3Operations.putObject(any(), any())).then(invocation -> { AmazonS3Client amazonS3Client = invocation.getArgument(1); ClientConfiguration clientConfiguration = (ClientConfiguration) ReflectionTestUtils.getField(amazonS3Client, "clientConfiguration"); assertEquals(S3Dao.SIGNER_OVERRIDE_V4, clientConfiguration.getSignerOverride()); return new PutObjectResult(); }); } finally { ReflectionTestUtils.setField(s3Dao, "s3Operations", originalS3Operations); } }
@Test public void testValidateGlacierS3FilesRestored() { // Put a 1 byte already restored Glacier storage class file in S3. ObjectMetadata metadata = new ObjectMetadata(); metadata.setHeader(Headers.STORAGE_CLASS, StorageClass.Glacier); metadata.setOngoingRestore(false); s3Operations .putObject(new PutObjectRequest(storageDaoTestHelper.getS3ManagedBucketName(), TARGET_S3_KEY, new ByteArrayInputStream(new byte[1]), metadata), null); // Validate the file. S3FileTransferRequestParamsDto params = new S3FileTransferRequestParamsDto(); params.setS3BucketName(storageDaoTestHelper.getS3ManagedBucketName()); params.setFiles(Arrays.asList(new File(TARGET_S3_KEY))); s3Dao.validateGlacierS3FilesRestored(params); }
@Test public void testTagObjectsTargetTagKeyAlreadyExists() { // Create two S3 object tags having the same tag key. List<Tag> tags = Arrays.asList(new Tag(S3_OBJECT_TAG_KEY, S3_OBJECT_TAG_VALUE), new Tag(S3_OBJECT_TAG_KEY, S3_OBJECT_TAG_VALUE_2)); // Put a file in S3 that is already tagged with the first S3 object tag. PutObjectRequest putObjectRequest = new PutObjectRequest(S3_BUCKET_NAME, TARGET_S3_KEY, new ByteArrayInputStream(new byte[1]), new ObjectMetadata()); putObjectRequest.setTagging(new ObjectTagging(Collections.singletonList(tags.get(0)))); s3Operations.putObject(putObjectRequest, null); // Validate that the S3 object is tagged with the first tag. GetObjectTaggingResult getObjectTaggingResult = s3Operations.getObjectTagging(new GetObjectTaggingRequest(S3_BUCKET_NAME, TARGET_S3_KEY), null); assertEquals(Collections.singletonList(tags.get(0)), getObjectTaggingResult.getTagSet()); // Create an S3 file transfer request parameters DTO to access S3 objects. S3FileTransferRequestParamsDto params = new S3FileTransferRequestParamsDto(); params.setS3BucketName(S3_BUCKET_NAME); // Create an S3 object summary. S3ObjectSummary s3ObjectSummary = new S3ObjectSummary(); s3ObjectSummary.setKey(TARGET_S3_KEY); // Tag the S3 file with the second S3 object tag. s3Dao.tagObjects(params, new S3FileTransferRequestParamsDto(), Collections.singletonList(s3ObjectSummary), tags.get(1)); // Validate that the S3 object is tagged with the second tag now. getObjectTaggingResult = s3Operations.getObjectTagging(new GetObjectTaggingRequest(S3_BUCKET_NAME, TARGET_S3_KEY), null); assertEquals(Collections.singletonList(tags.get(1)), getObjectTaggingResult.getTagSet()); }
@Test public void testValidateGlacierS3FilesRestoredGlacierObjectRestoreNotInitiated() { // Put a 1 byte Glacier storage class file in S3 that has no restore initiated (OngoingRestore flag is null). ObjectMetadata metadata = new ObjectMetadata(); metadata.setHeader(Headers.STORAGE_CLASS, StorageClass.Glacier); s3Operations .putObject(new PutObjectRequest(storageDaoTestHelper.getS3ManagedBucketName(), TARGET_S3_KEY, new ByteArrayInputStream(new byte[1]), metadata), null); // Try to validate if the Glacier S3 file is already restored. try { S3FileTransferRequestParamsDto params = new S3FileTransferRequestParamsDto(); params.setS3BucketName(storageDaoTestHelper.getS3ManagedBucketName()); params.setFiles(Arrays.asList(new File(TARGET_S3_KEY))); s3Dao.validateGlacierS3FilesRestored(params); fail("Should throw an IllegalArgumentException when Glacier S3 file is not restored."); } catch (IllegalArgumentException e) { assertEquals(String .format("Archived Glacier S3 file \"%s\" is not restored. StorageClass {GLACIER}, OngoingRestore flag {null}, S3 bucket name {%s}", TARGET_S3_KEY, storageDaoTestHelper.getS3ManagedBucketName()), e.getMessage()); } }
/** * The method is successful when both bucket and key exists. */ @Test public void testS3FileExists() { String expectedKey = "foo"; String expectedValue = "bar"; ByteArrayInputStream inputStream = new ByteArrayInputStream((expectedKey + "=" + expectedValue).getBytes()); PutObjectRequest putObjectRequest = new PutObjectRequest(S3_BUCKET_NAME, TARGET_S3_KEY, inputStream, new ObjectMetadata()); s3Operations.putObject(putObjectRequest, null); S3FileTransferRequestParamsDto params = s3DaoTestHelper.getTestS3FileTransferRequestParamsDto(); params.setS3BucketName(S3_BUCKET_NAME); params.setS3KeyPrefix(TARGET_S3_KEY); Assert.assertTrue(s3Dao.s3FileExists(params)); }
@Test public void testTagObjectsOtherTagKeyAlreadyExists() { // Create two S3 object tags having different tag keys. List<Tag> tags = Arrays.asList(new Tag(S3_OBJECT_TAG_KEY, S3_OBJECT_TAG_VALUE), new Tag(S3_OBJECT_TAG_KEY_2, S3_OBJECT_TAG_VALUE_2)); // Put a file in S3 that is already tagged with the first S3 object tag. PutObjectRequest putObjectRequest = new PutObjectRequest(S3_BUCKET_NAME, TARGET_S3_KEY, new ByteArrayInputStream(new byte[1]), new ObjectMetadata()); putObjectRequest.setTagging(new ObjectTagging(Collections.singletonList(tags.get(0)))); s3Operations.putObject(putObjectRequest, null); // Validate that the S3 object is tagged with the first tag only. GetObjectTaggingResult getObjectTaggingResult = s3Operations.getObjectTagging(new GetObjectTaggingRequest(S3_BUCKET_NAME, TARGET_S3_KEY), null); assertEquals(Collections.singletonList(tags.get(0)), getObjectTaggingResult.getTagSet()); // Create an S3 file transfer request parameters DTO to access S3 objects. S3FileTransferRequestParamsDto params = new S3FileTransferRequestParamsDto(); params.setS3BucketName(S3_BUCKET_NAME); // Create an S3 object summary. S3ObjectSummary s3ObjectSummary = new S3ObjectSummary(); s3ObjectSummary.setKey(TARGET_S3_KEY); // Tag the S3 file with the second S3 object tag. s3Dao.tagObjects(params, new S3FileTransferRequestParamsDto(), Collections.singletonList(s3ObjectSummary), tags.get(1)); // Validate that the S3 object is now tagged with both tags. getObjectTaggingResult = s3Operations.getObjectTagging(new GetObjectTaggingRequest(S3_BUCKET_NAME, TARGET_S3_KEY), null); assertEquals(tags.size(), getObjectTaggingResult.getTagSet().size()); assertTrue(getObjectTaggingResult.getTagSet().containsAll(tags)); }
@Test public void testDeleteBusinessObjectDataS3FileExists() throws Exception { // Prepare database entries required for testing. StorageEntity storageEntity = createTestStorageEntity(STORAGE_NAME, s3BucketName); BusinessObjectDataKey testBusinessObjectKey = new BusinessObjectDataKey(NAMESPACE, BDEF_NAME, FORMAT_USAGE_CODE, FORMAT_FILE_TYPE_CODE, FORMAT_VERSION, PARTITION_VALUE, SUBPARTITION_VALUES, DATA_VERSION); createTestDatabaseEntities(testBusinessObjectKey, BusinessObjectDataStatusEntity.UPLOADING, storageEntity, TARGET_S3_KEY, 15); // Put a file in S3. PutObjectRequest putObjectRequest = new PutObjectRequest(s3BucketName, TARGET_S3_KEY, new ByteArrayInputStream(new byte[1]), new ObjectMetadata()); s3Operations.putObject(putObjectRequest, null); // Delete the business object data. List<BusinessObjectDataKey> resultBusinessObjectDataKeys = fileUploadCleanupService.deleteBusinessObjectData(STORAGE_NAME, 10); // Validate the results. assertNotNull(resultBusinessObjectDataKeys); assertTrue(resultBusinessObjectDataKeys.isEmpty()); validateBusinessObjectDataStatus(testBusinessObjectKey, BusinessObjectDataStatusEntity.UPLOADING); }
@Test public void testRestoreObjects() { // Put a 1 byte Glacier storage class file in S3. ObjectMetadata metadata = new ObjectMetadata(); metadata.setHeader(Headers.STORAGE_CLASS, StorageClass.Glacier); metadata.setOngoingRestore(false); s3Operations .putObject(new PutObjectRequest(storageDaoTestHelper.getS3ManagedBucketName(), TARGET_S3_KEY, new ByteArrayInputStream(new byte[1]), metadata), null); // Initiate a restore request for the test S3 file. S3FileTransferRequestParamsDto params = new S3FileTransferRequestParamsDto(); params.setS3BucketName(storageDaoTestHelper.getS3ManagedBucketName()); params.setFiles(Arrays.asList(new File(TARGET_S3_KEY))); s3Dao.restoreObjects(params, S3_RESTORE_OBJECT_EXPIRATION_IN_DAYS); // Validate that there is an ongoing restore request for this object. ObjectMetadata objectMetadata = s3Operations.getObjectMetadata(storageDaoTestHelper.getS3ManagedBucketName(), TARGET_S3_KEY, null); assertTrue(objectMetadata.getOngoingRestore()); }
@Test public void testRestoreObjectsGlacierObjectAlreadyBeingRestored() { // Put a 1 byte Glacier storage class file in S3 flagged as already being restored. ObjectMetadata metadata = new ObjectMetadata(); metadata.setHeader(Headers.STORAGE_CLASS, StorageClass.Glacier); metadata.setOngoingRestore(true); s3Operations .putObject(new PutObjectRequest(storageDaoTestHelper.getS3ManagedBucketName(), TARGET_S3_KEY, new ByteArrayInputStream(new byte[1]), metadata), null); // Initiate a restore request for the test S3 file. S3FileTransferRequestParamsDto params = new S3FileTransferRequestParamsDto(); params.setS3BucketName(storageDaoTestHelper.getS3ManagedBucketName()); params.setFiles(Arrays.asList(new File(TARGET_S3_KEY))); s3Dao.restoreObjects(params, S3_RESTORE_OBJECT_EXPIRATION_IN_DAYS); // Validate that there is still an ongoing restore request for this object. ObjectMetadata objectMetadata = s3Operations.getObjectMetadata(storageDaoTestHelper.getS3ManagedBucketName(), TARGET_S3_KEY, null); assertTrue(objectMetadata.getOngoingRestore()); }
@Test public void testTagObjects() { // Create an S3 file transfer request parameters DTO to access S3 objects. S3FileTransferRequestParamsDto params = new S3FileTransferRequestParamsDto(); params.setS3BucketName(S3_BUCKET_NAME); // Create an S3 object summary. S3ObjectSummary s3ObjectSummary = new S3ObjectSummary(); s3ObjectSummary.setKey(TARGET_S3_KEY); // Create an S3 object tag. Tag tag = new Tag(S3_OBJECT_TAG_KEY, S3_OBJECT_TAG_VALUE); // Put a file in S3. s3Operations.putObject(new PutObjectRequest(S3_BUCKET_NAME, TARGET_S3_KEY, new ByteArrayInputStream(new byte[1]), new ObjectMetadata()), null); // Tag the file with an S3 object tag. s3Dao.tagObjects(params, new S3FileTransferRequestParamsDto(), Collections.singletonList(s3ObjectSummary), tag); // Validate that the object got tagged. GetObjectTaggingResult getObjectTaggingResult = s3Operations.getObjectTagging(new GetObjectTaggingRequest(S3_BUCKET_NAME, TARGET_S3_KEY), null); assertEquals(Collections.singletonList(tag), getObjectTaggingResult.getTagSet()); }
/** * Test S3 file copy without a KMS ID specified. */ @Test public void testCopyFileNoKmsId() throws InterruptedException { // Put a 1 byte file in S3. s3Operations .putObject(new PutObjectRequest(storageDaoTestHelper.getS3LoadingDockBucketName(), TARGET_S3_KEY, new ByteArrayInputStream(new byte[1]), null), null); // Perform an S3 file copy operation when KMS ID value is not specified. for (String kmsId : Arrays.asList(BLANK_TEXT, null)) { S3FileCopyRequestParamsDto transferDto = new S3FileCopyRequestParamsDto(); transferDto.setSourceBucketName(storageDaoTestHelper.getS3LoadingDockBucketName()); transferDto.setTargetBucketName(storageDaoTestHelper.getS3ExternalBucketName()); transferDto.setSourceObjectKey(TARGET_S3_KEY); transferDto.setTargetObjectKey(TARGET_S3_KEY); transferDto.setKmsKeyId(kmsId); S3FileTransferResultsDto resultsDto = s3Dao.copyFile(transferDto); assertEquals(Long.valueOf(1L), resultsDto.getTotalFilesTransferred()); } }
/** * Creates an object in S3 with the prefix constructed from the given parameters. The object's full path will be {prefix}/{UUID} * * @param businessObjectFormatEntity business object format * @param request request with partition values and storage * @param businessObjectDataVersion business object data version to put */ public void createS3Object(BusinessObjectFormatEntity businessObjectFormatEntity, BusinessObjectDataInvalidateUnregisteredRequest request, int businessObjectDataVersion) { StorageEntity storageEntity = storageDao.getStorageByName(request.getStorageName()); String s3BucketName = storageHelper.getS3BucketAccessParams(storageEntity).getS3BucketName(); BusinessObjectDataKey businessObjectDataKey = getBusinessObjectDataKey(request); businessObjectDataKey.setBusinessObjectDataVersion(businessObjectDataVersion); String s3KeyPrefix = s3KeyPrefixHelper .buildS3KeyPrefix(AbstractServiceTest.S3_KEY_PREFIX_VELOCITY_TEMPLATE, businessObjectFormatEntity, businessObjectDataKey, storageEntity.getName()); String s3ObjectKey = s3KeyPrefix + "/test"; PutObjectRequest putObjectRequest = new PutObjectRequest(s3BucketName, s3ObjectKey, new ByteArrayInputStream(new byte[1]), new ObjectMetadata()); s3Operations.putObject(putObjectRequest, null); }
/** * Test S3 file copy without any errors. */ @Test public void testCopyFile() throws InterruptedException { // Put a 1 byte file in S3. s3Operations .putObject(new PutObjectRequest(storageDaoTestHelper.getS3LoadingDockBucketName(), TARGET_S3_KEY, new ByteArrayInputStream(new byte[1]), null), null); S3FileCopyRequestParamsDto transferDto = new S3FileCopyRequestParamsDto(); transferDto.setSourceBucketName(storageDaoTestHelper.getS3LoadingDockBucketName()); transferDto.setTargetBucketName(storageDaoTestHelper.getS3ExternalBucketName()); transferDto.setSourceObjectKey(TARGET_S3_KEY); transferDto.setTargetObjectKey(TARGET_S3_KEY); transferDto.setKmsKeyId(MockS3OperationsImpl.MOCK_KMS_ID); S3FileTransferResultsDto resultsDto = s3Dao.copyFile(transferDto); assertEquals(Long.valueOf(1L), resultsDto.getTotalFilesTransferred()); }
@Test public void testDeleteSourceFileFromS3() { // Create and persists entities required for testing. // Create source and target business object data entities. BusinessObjectDataEntity sourceBusinessObjectDataEntity = businessObjectDataDaoTestHelper .createBusinessObjectDataEntity(NAMESPACE, BDEF_NAME, FORMAT_USAGE_CODE, FORMAT_FILE_TYPE_CODE, INITIAL_FORMAT_VERSION, PARTITION_VALUE, INITIAL_DATA_VERSION, LATEST_VERSION_FLAG_SET, BusinessObjectDataStatusEntity.RE_ENCRYPTING); BusinessObjectDataEntity targetBusinessObjectDataEntity = businessObjectDataDaoTestHelper .createBusinessObjectDataEntity(NAMESPACE_2, BDEF_NAME_2, FORMAT_USAGE_CODE_2, FORMAT_FILE_TYPE_CODE_2, INITIAL_FORMAT_VERSION, PARTITION_VALUE, INITIAL_DATA_VERSION, LATEST_VERSION_FLAG_SET, BusinessObjectDataStatusEntity.RE_ENCRYPTING); // Put a 1 KB file in S3. s3Operations.putObject( new PutObjectRequest(storageDaoTestHelper.getS3LoadingDockBucketName(), TARGET_S3_KEY, new ByteArrayInputStream(new byte[(int) FILE_SIZE_1_KB]), null), null); // Initialize parameters required to delete the S3 source file CompleteUploadSingleParamsDto completeUploadSingleParamsDto = new CompleteUploadSingleParamsDto(businessObjectDataHelper.getBusinessObjectDataKey(sourceBusinessObjectDataEntity), storageDaoTestHelper.getS3LoadingDockBucketName(), TARGET_S3_KEY, BusinessObjectDataStatusEntity.UPLOADING, BusinessObjectDataStatusEntity.RE_ENCRYPTING, businessObjectDataHelper.getBusinessObjectDataKey(targetBusinessObjectDataEntity), storageDaoTestHelper.getS3ExternalBucketName(), TARGET_S3_KEY, BusinessObjectDataStatusEntity.UPLOADING, BusinessObjectDataStatusEntity.RE_ENCRYPTING, MockS3OperationsImpl.MOCK_KMS_ID, emrHelper.getAwsParamsDto()); // Delete the source file from S3 uploadDownloadHelperService.deleteSourceFileFromS3(completeUploadSingleParamsDto); }
/** * Test S3 file copy with an invalid KMS Id. This should throw an AmazonServiceException. */ @Test public void testCopyFileInvalidKmsId() throws InterruptedException { // Put a 1 byte file in S3. s3Operations .putObject(new PutObjectRequest(storageDaoTestHelper.getS3LoadingDockBucketName(), TARGET_S3_KEY, new ByteArrayInputStream(new byte[1]), null), null); try { S3FileCopyRequestParamsDto transferDto = new S3FileCopyRequestParamsDto(); transferDto.setSourceBucketName(storageDaoTestHelper.getS3LoadingDockBucketName()); transferDto.setTargetBucketName(storageDaoTestHelper.getS3ExternalBucketName()); transferDto.setSourceObjectKey(TARGET_S3_KEY); transferDto.setTargetObjectKey(TARGET_S3_KEY); transferDto.setKmsKeyId(MockS3OperationsImpl.MOCK_KMS_ID_FAILED_TRANSFER); s3Dao.copyFile(transferDto); fail("An AmazonServiceException was expected but not thrown."); } catch (AmazonServiceException ex) { Assert.assertTrue("Invalid AmazonServiceException message returned.", ex.getMessage().contains("Key '" + MockS3OperationsImpl.MOCK_KMS_ID_FAILED_TRANSFER + "' does not exist")); } }