@Bean public S3Dao s3Dao() { return new S3DaoImpl(); }
@Override public Properties getProperties(String bucketName, String key, S3FileTransferRequestParamsDto s3FileTransferRequestParamsDto) { AmazonS3Client s3Client = getAmazonS3(s3FileTransferRequestParamsDto); try { S3Object s3Object = getS3Object(s3Client, bucketName, key, true); return javaPropertiesHelper.getProperties(s3Object.getObjectContent()); } catch (IllegalArgumentException e) { throw new IllegalArgumentException("The properties file in S3 bucket '" + bucketName + "' and key '" + key + "' is invalid.", e); } finally { s3Client.shutdown(); } }
@Override public void validateS3File(S3FileTransferRequestParamsDto params, Long fileSizeInBytes) throws RuntimeException { ObjectMetadata objectMetadata = getObjectMetadata(params); if (objectMetadata == null) { throw new ObjectNotFoundException(String.format("File not found at s3://%s/%s location.", params.getS3BucketName(), params.getS3KeyPrefix())); } Assert.isTrue(fileSizeInBytes == null || Objects.equals(fileSizeInBytes, objectMetadata.getContentLength()), String .format("Specified file size (%d bytes) does not match to the actual file size (%d bytes) reported by S3 for s3://%s/%s file.", fileSizeInBytes, objectMetadata.getContentLength(), params.getS3BucketName(), params.getS3KeyPrefix())); }
Assert.isTrue(!isRootKeyPrefix(params.getS3KeyPrefix()), "Deleting from root directory is not allowed."); List<S3VersionSummary> s3VersionSummaries = listVersions(params); LOGGER.info("Found keys/key versions in S3 for deletion. s3KeyCount={} s3KeyPrefix=\"{}\" s3BucketName=\"{}\"", s3VersionSummaries.size(), params.getS3KeyPrefix(), params.getS3BucketName()); AmazonS3Client s3Client = getAmazonS3(params); deleteKeyVersions(s3Client, params.getS3BucketName(), keyVersions);
TransferManager transferManager = getTransferManager(params); TransferProgress transferProgress = transfer.getProgress(); logTransferProgress(transferProgress); logTransferProgress(transferProgress); results.setTotalFilesTransferred((long) listDirectory(params).size());
@Override public List<S3ObjectSummary> listDirectory(final S3FileTransferRequestParamsDto params, boolean ignoreZeroByteDirectoryMarkers) Assert.isTrue(!isRootKeyPrefix(params.getS3KeyPrefix()), "Listing of S3 objects from root directory is not allowed."); AmazonS3Client s3Client = getAmazonS3(params); List<S3ObjectSummary> s3ObjectSummaries = new ArrayList<>();
/** * Gets a transfer manager with the specified parameters including proxy host, proxy port, S3 access key, S3 secret key, and max threads. * * @param params the parameters. * * @return a newly created transfer manager. */ private TransferManager getTransferManager(final S3FileTransferRequestParamsDto params) { // We are returning a new transfer manager each time it is called. Although the Javadocs of TransferManager say to share a single instance // if possible, this could potentially be a problem if TransferManager.shutdown(true) is called and underlying resources are not present when needed // for subsequent transfers. if (params.getMaxThreads() == null) { // Create a transfer manager that will internally use an appropriate number of threads. return new TransferManager(getAmazonS3(params)); } else { // Create a transfer manager with our own executor configured with the specified total threads. LOGGER.info("Creating a transfer manager. fixedThreadPoolSize={}", params.getMaxThreads()); return new TransferManager(getAmazonS3(params), Executors.newFixedThreadPool(params.getMaxThreads())); } }
@Override public void createDirectory(final S3FileTransferRequestParamsDto params) { // Create metadata for the directory marker and set content-length to 0 bytes. ObjectMetadata metadata = new ObjectMetadata(); metadata.setContentLength(0); prepareMetadata(params, metadata); // Create empty content. InputStream emptyContent = new ByteArrayInputStream(new byte[0]); // Create a PutObjectRequest passing the folder name suffixed by '/'. String directoryName = StringUtils.appendIfMissing(params.getS3KeyPrefix(), "/"); PutObjectRequest putObjectRequest = new PutObjectRequest(params.getS3BucketName(), directoryName, emptyContent, metadata); // KMS key ID is being set through prepareMetadata() AmazonS3Client s3Client = getAmazonS3(params); try { s3Operations.putObject(putObjectRequest, s3Client); } catch (AmazonServiceException e) { throw new IllegalStateException(String .format("Failed to create 0 byte S3 object with \"%s\" key in bucket \"%s\". Reason: %s", directoryName, params.getS3BucketName(), e.getMessage()), e); } finally { // Shutdown the AmazonS3Client instance to release resources. s3Client.shutdown(); } }
AmazonS3Client s3Client = getAmazonS3(params); deleteKeyVersions(s3Client, params.getS3BucketName(), keys);
@Override public List<S3ObjectSummary> listDirectory(final S3FileTransferRequestParamsDto params) { // By default, we do not ignore 0 byte objects that represent S3 directories. return listDirectory(params, false); }
@Test public void testDeleteDirectoryNoS3VersionsExist() { // Create an S3 file transfer request parameters DTO to access S3 objects. S3FileTransferRequestParamsDto s3FileTransferRequestParamsDto = new S3FileTransferRequestParamsDto(); s3FileTransferRequestParamsDto.setS3BucketName(S3_BUCKET_NAME); s3FileTransferRequestParamsDto.setS3KeyPrefix(S3_KEY_PREFIX); // Create a retry policy. RetryPolicy retryPolicy = new RetryPolicy(PredefinedRetryPolicies.DEFAULT_RETRY_CONDITION, PredefinedRetryPolicies.DEFAULT_BACKOFF_STRATEGY, INTEGER_VALUE, true); // Create an empty version listing. VersionListing versionListing = new VersionListing(); // Mock the external calls. when(retryPolicyFactory.getRetryPolicy()).thenReturn(retryPolicy); when(s3Operations.listVersions(any(ListVersionsRequest.class), any(AmazonS3Client.class))).thenReturn(versionListing); // Call the method under test. s3DaoImpl.deleteDirectory(s3FileTransferRequestParamsDto); // Verify the external calls. verify(retryPolicyFactory).getRetryPolicy(); verify(s3Operations).listVersions(any(ListVersionsRequest.class), any(AmazonS3Client.class)); verifyNoMoreInteractionsHelper(); }
amazonS3Client = new AmazonS3Client(getAWSCredentialsProvider(params), clientConfiguration);
Assert.isTrue(!isRootKeyPrefix(params.getS3KeyPrefix()), "Deleting from root directory is not allowed."); List<S3VersionSummary> s3VersionSummaries = listVersions(params); LOGGER.info("Found keys/key versions in S3 for deletion. s3KeyCount={} s3KeyPrefix=\"{}\" s3BucketName=\"{}\"", s3VersionSummaries.size(), params.getS3KeyPrefix(), params.getS3BucketName()); AmazonS3Client s3Client = getAmazonS3(params); deleteKeyVersions(s3Client, params.getS3BucketName(), keyVersions);
TransferManager transferManager = getTransferManager(params); TransferProgress transferProgress = transfer.getProgress(); logTransferProgress(transferProgress); logTransferProgress(transferProgress); results.setTotalFilesTransferred((long) listDirectory(params).size());
@Override public List<S3ObjectSummary> listDirectory(final S3FileTransferRequestParamsDto params, boolean ignoreZeroByteDirectoryMarkers) Assert.isTrue(!isRootKeyPrefix(params.getS3KeyPrefix()), "Listing of S3 objects from root directory is not allowed."); AmazonS3Client s3Client = getAmazonS3(params); List<S3ObjectSummary> s3ObjectSummaries = new ArrayList<>();
/** * Gets a transfer manager with the specified parameters including proxy host, proxy port, S3 access key, S3 secret key, and max threads. * * @param params the parameters. * * @return a newly created transfer manager. */ private TransferManager getTransferManager(final S3FileTransferRequestParamsDto params) { // We are returning a new transfer manager each time it is called. Although the Javadocs of TransferManager say to share a single instance // if possible, this could potentially be a problem if TransferManager.shutdown(true) is called and underlying resources are not present when needed // for subsequent transfers. if (params.getMaxThreads() == null) { // Create a transfer manager that will internally use an appropriate number of threads. return new TransferManager(getAmazonS3(params)); } else { // Create a transfer manager with our own executor configured with the specified total threads. LOGGER.info("Creating a transfer manager. fixedThreadPoolSize={}", params.getMaxThreads()); return new TransferManager(getAmazonS3(params), Executors.newFixedThreadPool(params.getMaxThreads())); } }
@Override public void createDirectory(final S3FileTransferRequestParamsDto params) { // Create metadata for the directory marker and set content-length to 0 bytes. ObjectMetadata metadata = new ObjectMetadata(); metadata.setContentLength(0); prepareMetadata(params, metadata); // Create empty content. InputStream emptyContent = new ByteArrayInputStream(new byte[0]); // Create a PutObjectRequest passing the folder name suffixed by '/'. String directoryName = StringUtils.appendIfMissing(params.getS3KeyPrefix(), "/"); PutObjectRequest putObjectRequest = new PutObjectRequest(params.getS3BucketName(), directoryName, emptyContent, metadata); // KMS key ID is being set through prepareMetadata() AmazonS3Client s3Client = getAmazonS3(params); try { s3Operations.putObject(putObjectRequest, s3Client); } catch (AmazonServiceException e) { throw new IllegalStateException(String .format("Failed to create 0 byte S3 object with \"%s\" key in bucket \"%s\". Reason: %s", directoryName, params.getS3BucketName(), e.getMessage()), e); } finally { // Shutdown the AmazonS3Client instance to release resources. s3Client.shutdown(); } }
AmazonS3Client s3Client = getAmazonS3(params); deleteKeyVersions(s3Client, params.getS3BucketName(), keys);
@Override public List<S3ObjectSummary> listDirectory(final S3FileTransferRequestParamsDto params) { // By default, we do not ignore 0 byte objects that represent S3 directories. return listDirectory(params, false); }
s3DaoImpl.deleteDirectory(s3FileTransferRequestParamsDto);