@Override public List<S3ObjectSummary> listDirectory(S3FileTransferRequestParamsDto params) { // By default, we do not ignore 0 byte objects that represent S3 directories. return s3Dao.listDirectory(params, false); }
@Override public List<S3ObjectSummary> listDirectory(S3FileTransferRequestParamsDto params, boolean ignoreZeroByteDirectoryMarkers) { return s3Dao.listDirectory(params, ignoreZeroByteDirectoryMarkers); }
@Override public List<S3ObjectSummary> listDirectory(S3FileTransferRequestParamsDto params, boolean ignoreZeroByteDirectoryMarkers) { return s3Dao.listDirectory(params, ignoreZeroByteDirectoryMarkers); }
@Override public List<S3ObjectSummary> listDirectory(S3FileTransferRequestParamsDto params) { // By default, we do not ignore 0 byte objects that represent S3 directories. return s3Dao.listDirectory(params, false); }
/** * Test that we get an exception when trying to perform listDirectory operation without properly initialized S3FileTransferRequestParamsDto parameters. */ @Test(expected = NullPointerException.class) public void testListDirectoryNullPointerException() { s3Dao.listDirectory(null); }
/** * Returns a list of S3 object keys associated with the given format, data key, and storage. The keys are found by matching the prefix. The result may be * empty if there are not matching keys found. * * @param businessObjectFormatEntity {@link BusinessObjectFormatEntity} * @param businessObjectDataKey {@link BusinessObjectDataKey} * @param storageEntity {@link StorageEntity} * * @return list of S3 object keys */ private List<String> getS3ObjectKeys(BusinessObjectFormatEntity businessObjectFormatEntity, BusinessObjectDataKey businessObjectDataKey, StorageEntity storageEntity) { String s3KeyPrefix = s3KeyPrefixHelper.buildS3KeyPrefix(storageEntity, businessObjectFormatEntity, businessObjectDataKey); S3FileTransferRequestParamsDto s3FileTransferRequestParamsDto = storageHelper.getS3BucketAccessParams(storageEntity); s3FileTransferRequestParamsDto.setS3KeyPrefix(s3KeyPrefix + '/'); return storageFileHelper.getFilePathsFromS3ObjectSummaries(s3Dao.listDirectory(s3FileTransferRequestParamsDto)); }
/** * Returns a list of S3 object keys associated with the given format, data key, and storage. The keys are found by matching the prefix. The result may be * empty if there are not matching keys found. * * @param businessObjectFormatEntity {@link BusinessObjectFormatEntity} * @param businessObjectDataKey {@link BusinessObjectDataKey} * @param storageEntity {@link StorageEntity} * * @return list of S3 object keys */ private List<String> getS3ObjectKeys(BusinessObjectFormatEntity businessObjectFormatEntity, BusinessObjectDataKey businessObjectDataKey, StorageEntity storageEntity) { String s3KeyPrefix = s3KeyPrefixHelper.buildS3KeyPrefix(storageEntity, businessObjectFormatEntity, businessObjectDataKey); S3FileTransferRequestParamsDto s3FileTransferRequestParamsDto = storageHelper.getS3BucketAccessParams(storageEntity); s3FileTransferRequestParamsDto.setS3KeyPrefix(s3KeyPrefix + '/'); return storageFileHelper.getFilePathsFromS3ObjectSummaries(s3Dao.listDirectory(s3FileTransferRequestParamsDto)); }
@Test public void testListDirectory() { // Create an S3 file transfer request parameters DTO. S3FileTransferRequestParamsDto s3FileTransferRequestParamsDto = new S3FileTransferRequestParamsDto(); // Create a list of S3 object summaries. List<S3ObjectSummary> s3ObjectSummaries = Collections.singletonList(new S3ObjectSummary()); // Mock the external calls. when(s3Dao.listDirectory(s3FileTransferRequestParamsDto, false)).thenReturn(s3ObjectSummaries); // Call the method under test. List<S3ObjectSummary> result = s3Service.listDirectory(s3FileTransferRequestParamsDto); // Verify the external calls. By default, we do not ignore 0 byte objects that represent S3 directories. verify(s3Dao).listDirectory(s3FileTransferRequestParamsDto, false); verifyNoMoreInteractions(s3Dao); // Validate the returned object. assertEquals(s3ObjectSummaries, result); }
@Test public void testListDirectoryIgnoreZeroByteDirectoryMarkers() { // Create an S3 file transfer request parameters DTO. S3FileTransferRequestParamsDto s3FileTransferRequestParamsDto = new S3FileTransferRequestParamsDto(); // Create a list of S3 object summaries. List<S3ObjectSummary> s3ObjectSummaries = Collections.singletonList(new S3ObjectSummary()); // Mock the external calls. when(s3Dao.listDirectory(s3FileTransferRequestParamsDto, true)).thenReturn(s3ObjectSummaries); // Call the method under test. List<S3ObjectSummary> result = s3Service.listDirectory(s3FileTransferRequestParamsDto, true); // Verify the external calls. verify(s3Dao).listDirectory(s3FileTransferRequestParamsDto, true); verifyNoMoreInteractions(s3Dao); // Validate the returned object. assertEquals(s3ObjectSummaries, result); }
/** * Validates uploaded S3 files. * * @param s3FileTransferRequestParamsDto the S3 file transfer request parameters DTO * @param expectedS3Keys the list of expected S3 keys */ public void validateS3FileUpload(S3FileTransferRequestParamsDto s3FileTransferRequestParamsDto, List<String> expectedS3Keys) { // Validate the upload. List<S3ObjectSummary> s3ObjectSummaries = s3Dao.listDirectory(s3FileTransferRequestParamsDto); assertTrue(s3ObjectSummaries.size() == expectedS3Keys.size()); // Build a list of the actual S3 keys. List<String> actualS3Keys = new ArrayList<>(); for (S3ObjectSummary s3ObjectSummary : s3ObjectSummaries) { actualS3Keys.add(s3ObjectSummary.getKey()); } // Check that all local test files got uploaded. assertTrue(expectedS3Keys.containsAll(actualS3Keys)); assertTrue(actualS3Keys.containsAll(expectedS3Keys)); } }
/** * Test that we are able to perform the deleteDirectory S3Dao operation on S3 using our DAO tier. */ @Test public void testDeleteDirectory() throws IOException, InterruptedException { // Upload local directory to s3Dao. testUploadDirectory(); // Validate that S3 directory is not empty. S3FileTransferRequestParamsDto s3FileTransferRequestParamsDto = s3DaoTestHelper.getTestS3FileTransferRequestParamsDto(); s3FileTransferRequestParamsDto.setS3KeyPrefix(TEST_S3_KEY_PREFIX + "/"); Assert.assertTrue(s3Dao.listDirectory(s3FileTransferRequestParamsDto).size() > 0); // Delete directory from S3 using s3Dao. s3Dao.deleteDirectory(s3FileTransferRequestParamsDto); // Validate that S3 directory got deleted. Assert.assertEquals(0, s3Dao.listDirectory(s3FileTransferRequestParamsDto).size()); }
@Test public void testListDirectoryRootKeyPrefix() { for (String s3KeyPrefix : Arrays.asList(null, BLANK_TEXT, "/")) { try { s3Dao.listDirectory(S3FileTransferRequestParamsDto.builder().withS3KeyPrefix(s3KeyPrefix).build()); fail("Should throw an IllegalArgumentException when S3 key prefix specifies a root directory."); } catch (IllegalArgumentException e) { assertEquals("Listing of S3 objects from root directory is not allowed.", e.getMessage()); } } }
/** * Test that we are able to perform the deleteFileList S3Dao operation on S3 using our DAO tier. */ @Test public void testDeleteFileList() throws IOException, InterruptedException { // Upload local directory to s3Dao. testUploadDirectory(); // Validate that S3 directory is not empty. S3FileTransferRequestParamsDto s3FileTransferRequestParamsDto = s3DaoTestHelper.getTestS3FileTransferRequestParamsDto(); s3FileTransferRequestParamsDto.setS3KeyPrefix(TEST_S3_KEY_PREFIX + "/"); List<S3ObjectSummary> s3ObjectSummaries = s3Dao.listDirectory(s3FileTransferRequestParamsDto); Assert.assertTrue(s3ObjectSummaries.size() > 0); // Delete a list of files from S3 using s3Dao. List<File> requestFileList = new ArrayList<>(); for (S3ObjectSummary s3ObjectSummary : s3ObjectSummaries) { requestFileList.add(new File(s3ObjectSummary.getKey())); } s3FileTransferRequestParamsDto.setFiles(requestFileList); s3FileTransferRequestParamsDto.setS3KeyPrefix(null); s3Dao.deleteFileList(s3FileTransferRequestParamsDto); // Validate that S3 directory got deleted. s3FileTransferRequestParamsDto.setS3KeyPrefix(TEST_S3_KEY_PREFIX + "/"); Assert.assertEquals(0, s3Dao.listDirectory(s3FileTransferRequestParamsDto).size()); }
@Test public void testListDirectoryNoSuchBucket() { S3FileTransferRequestParamsDto s3FileTransferRequestParamsDto = new S3FileTransferRequestParamsDto(); s3FileTransferRequestParamsDto.setS3BucketName(MockS3OperationsImpl.MOCK_S3_BUCKET_NAME_NO_SUCH_BUCKET_EXCEPTION); s3FileTransferRequestParamsDto.setS3KeyPrefix(TEST_S3_KEY_PREFIX); try { s3Dao.listDirectory(s3FileTransferRequestParamsDto); fail("expected an IllegalArgumentException to be thrown, but no exceptions thrown"); } catch (Exception e) { assertEquals("thrown exception type", IllegalArgumentException.class, e.getClass()); assertEquals("thrown exception message", "The specified bucket '" + MockS3OperationsImpl.MOCK_S3_BUCKET_NAME_NO_SUCH_BUCKET_EXCEPTION + "' does not exist.", e.getMessage()); } }
/** * Test that we are able to perform the uploadDirectory S3Dao operation on a folder with 0 bytes of data using our DAO tier. */ @Test public void testUploadDirectoryZeroBytes() throws IOException, InterruptedException { // Create a zero size local file. File targetFile = createLocalFile(localTempPath.toString(), LOCAL_FILE, FILE_SIZE_0_BYTE); Assert.assertTrue(targetFile.isFile()); // Upload empty folder to s3Dao. S3FileTransferRequestParamsDto s3FileTransferRequestParamsDto = s3DaoTestHelper.getTestS3FileTransferRequestParamsDto(); s3FileTransferRequestParamsDto.setS3KeyPrefix(TEST_S3_KEY_PREFIX); s3FileTransferRequestParamsDto.setLocalPath(localTempPath.toString()); s3FileTransferRequestParamsDto.setRecursive(true); S3FileTransferResultsDto results = s3Dao.uploadDirectory(s3FileTransferRequestParamsDto); // Validate results. Assert.assertTrue(results.getTotalFilesTransferred() == 1L); // Validate the zero bytes upload. Assert.assertEquals(1, s3Dao.listDirectory(s3FileTransferRequestParamsDto).size()); }
/** * Test that we are able to perform the uploadDirectory S3Dao operation on a folder with more than 1000 files using our DAO tier. This test is needed to * make AmazonS3.listObjects() to start truncating ObjectListing that it returns. */ @Test public void testUploadDirectoryLargeNumberOfFiles() throws IOException, InterruptedException { final int NUM_OF_FILES = 1001; // Create local files. for (int i = 0; i < NUM_OF_FILES; i++) { File targetFile = createLocalFile(localTempPath.toString(), String.format("%04d_%s", i, LOCAL_FILE), FILE_SIZE_0_BYTE); Assert.assertTrue(targetFile.isFile()); } // Upload empty folder to s3Dao. S3FileTransferRequestParamsDto s3FileTransferRequestParamsDto = s3DaoTestHelper.getTestS3FileTransferRequestParamsDto(); s3FileTransferRequestParamsDto.setS3KeyPrefix(TEST_S3_KEY_PREFIX); s3FileTransferRequestParamsDto.setLocalPath(localTempPath.toString()); s3FileTransferRequestParamsDto.setRecursive(true); S3FileTransferResultsDto results = s3Dao.uploadDirectory(s3FileTransferRequestParamsDto); // Validate results. Assert.assertTrue(results.getTotalFilesTransferred() == NUM_OF_FILES); // Validate the empty folder upload. Assert.assertEquals(NUM_OF_FILES, s3Dao.listDirectory(s3FileTransferRequestParamsDto).size()); }
/** * Cleans up the local temp directory and S3 test path that we are using. */ @After public void cleanEnv() throws IOException { // Clean up the local directory. FileUtils.deleteDirectory(localTempPath.toFile()); // Delete test files from S3 storage. Since test S3 key prefix represents a directory, we add a trailing '/' character to it. for (S3FileTransferRequestParamsDto params : Arrays.asList(s3DaoTestHelper.getTestS3FileTransferRequestParamsDto(), S3FileTransferRequestParamsDto.builder().withS3BucketName(storageDaoTestHelper.getS3LoadingDockBucketName()) .withS3KeyPrefix(TEST_S3_KEY_PREFIX + "/").build(), S3FileTransferRequestParamsDto.builder().withS3BucketName(storageDaoTestHelper.getS3ExternalBucketName()).withS3KeyPrefix(TEST_S3_KEY_PREFIX + "/") .build())) { if (!s3Dao.listDirectory(params).isEmpty()) { s3Dao.deleteDirectory(params); } } s3Operations.rollback(); }
/** * Cleans up the S3 test path that we are using. */ @After public void cleanEnv() throws IOException { // Clean up the source and target S3 folders. Since test S3 key prefix represents a directory, we add a trailing '/' character to it. for (S3FileTransferRequestParamsDto params : Arrays.asList( S3FileTransferRequestParamsDto.builder().withS3BucketName(storageDaoTestHelper.getS3LoadingDockBucketName()) .withS3KeyPrefix(TEST_S3_KEY_PREFIX + "/").build(), S3FileTransferRequestParamsDto.builder().withS3BucketName(storageDaoTestHelper.getS3ExternalBucketName()).withS3KeyPrefix(TEST_S3_KEY_PREFIX + "/") .build())) { if (!s3Dao.listDirectory(params).isEmpty()) { s3Dao.deleteDirectory(params); } } s3Operations.rollback(); }
@Test public void testDeleteBusinessObjectDataS3ManagedBucket() throws Exception { // Create test database entities. createTestDatabaseEntities(StorageEntity.MANAGED_STORAGE, StoragePlatformEntity.S3, testS3KeyPrefix, LOCAL_FILES); // Create and upload to the test S3 storage a set of files. businessObjectDataServiceTestHelper.prepareTestS3Files(testS3KeyPrefix, localTempPath, LOCAL_FILES); // Validate that this business object data exists. BusinessObjectDataKey businessObjectDataKey = new BusinessObjectDataKey(NAMESPACE, BDEF_NAME, FORMAT_USAGE_CODE, FORMAT_FILE_TYPE_CODE, INITIAL_FORMAT_VERSION, PARTITION_VALUE, new ArrayList<String>(), INITIAL_DATA_VERSION); BusinessObjectDataEntity businessObjectDataEntity = businessObjectDataDao.getBusinessObjectDataByAltKey(businessObjectDataKey); assertNotNull(businessObjectDataEntity); // Delete the business object data with delete files flag set to true. BusinessObjectData deletedBusinessObjectData = businessObjectDataService.deleteBusinessObjectData( new BusinessObjectDataKey(NAMESPACE, BDEF_NAME, FORMAT_USAGE_CODE, FORMAT_FILE_TYPE_CODE, INITIAL_FORMAT_VERSION, PARTITION_VALUE, NO_SUBPARTITION_VALUES, INITIAL_DATA_VERSION), true); // Validate the returned object. businessObjectDataServiceTestHelper .validateBusinessObjectData(businessObjectDataEntity.getId(), NAMESPACE, BDEF_NAME, FORMAT_USAGE_CODE, FORMAT_FILE_TYPE_CODE, INITIAL_FORMAT_VERSION, PARTITION_VALUE, NO_SUBPARTITION_VALUES, INITIAL_DATA_VERSION, true, BDATA_STATUS, deletedBusinessObjectData); // Validate that data files got deleted from S3. S3FileTransferRequestParamsDto params = s3DaoTestHelper.getTestS3FileTransferRequestParamsDto(); params.setS3KeyPrefix(testS3KeyPrefix); assertTrue(s3Dao.listDirectory(params).isEmpty()); // Ensure that this business object data is no longer there. assertNull(businessObjectDataDao.getBusinessObjectDataByAltKey(businessObjectDataKey)); }
@Test public void testDeleteBusinessObjectDataS3NonManagedBucketDeletingDirectory() throws Exception { // Create test database entities. createTestDatabaseEntities(STORAGE_NAME, StoragePlatformEntity.S3, TEST_S3_KEY_PREFIX, new ArrayList<String>()); // Create and upload to the test S3 storage a set of files. businessObjectDataServiceTestHelper.prepareTestS3Files(TEST_S3_KEY_PREFIX, localTempPath, LOCAL_FILES); // Validate that this business object data exists. BusinessObjectDataKey businessObjectDataKey = new BusinessObjectDataKey(NAMESPACE, BDEF_NAME, FORMAT_USAGE_CODE, FORMAT_FILE_TYPE_CODE, INITIAL_FORMAT_VERSION, PARTITION_VALUE, NO_SUBPARTITION_VALUES, INITIAL_DATA_VERSION); BusinessObjectDataEntity businessObjectDataEntity = businessObjectDataDao.getBusinessObjectDataByAltKey(businessObjectDataKey); assertNotNull(businessObjectDataEntity); // Delete the business object data with delete files flag set to true. BusinessObjectData deletedBusinessObjectData = businessObjectDataService.deleteBusinessObjectData( new BusinessObjectDataKey(NAMESPACE, BDEF_NAME, FORMAT_USAGE_CODE, FORMAT_FILE_TYPE_CODE, INITIAL_FORMAT_VERSION, PARTITION_VALUE, NO_SUBPARTITION_VALUES, INITIAL_DATA_VERSION), true); // Validate the returned object. businessObjectDataServiceTestHelper .validateBusinessObjectData(businessObjectDataEntity.getId(), NAMESPACE, BDEF_NAME, FORMAT_USAGE_CODE, FORMAT_FILE_TYPE_CODE, INITIAL_FORMAT_VERSION, PARTITION_VALUE, NO_SUBPARTITION_VALUES, INITIAL_DATA_VERSION, true, BDATA_STATUS, deletedBusinessObjectData); // Validate that data files got deleted from S3. S3FileTransferRequestParamsDto params = s3DaoTestHelper.getTestS3FileTransferRequestParamsDto(); params.setS3KeyPrefix(TEST_S3_KEY_PREFIX); assertTrue(s3Dao.listDirectory(params).isEmpty()); // Ensure that this business object data is no longer there. assertNull(businessObjectDataDao.getBusinessObjectDataByAltKey(businessObjectDataKey)); }