AwsParamsDto awsParamsDto = awsHelper.getAwsParamsDto();
/** * Sends out an SQS message that contains the specified storage policy selection to the SQS queue name. * * @param sqsQueueName the SQS queue name * @param storagePolicySelection the storage policy selection */ protected void sendStoragePolicySelectionSqsMessage(String sqsQueueName, StoragePolicySelection storagePolicySelection) { // Send the storage policy selection to the relative AWS SQS queue. AwsParamsDto awsParamsDto = awsHelper.getAwsParamsDto(); String messageText = null; try { messageText = jsonHelper.objectToJson(storagePolicySelection); sqsDao.sendMessage(awsParamsDto, sqsQueueName, messageText, null); } catch (RuntimeException e) { LOGGER.error("Failed to publish message to the JMS queue. jmsQueueName=\"{}\" jmsMessagePayload={}", sqsQueueName, messageText); // Rethrow the original exception. throw e; } }
/** * Sends out an SQS message that contains the specified storage policy selection to the SQS queue name. * * @param sqsQueueName the SQS queue name * @param storagePolicySelection the storage policy selection */ protected void sendStoragePolicySelectionSqsMessage(String sqsQueueName, StoragePolicySelection storagePolicySelection) { // Send the storage policy selection to the relative AWS SQS queue. AwsParamsDto awsParamsDto = awsHelper.getAwsParamsDto(); String messageText = null; try { messageText = jsonHelper.objectToJson(storagePolicySelection); sqsDao.sendMessage(awsParamsDto, sqsQueueName, messageText, null); } catch (RuntimeException e) { LOGGER.error("Failed to publish message to the JMS queue. jmsQueueName=\"{}\" jmsMessagePayload={}", sqsQueueName, messageText); // Rethrow the original exception. throw e; } }
/** * Gets the decrypted value for the given configuration option. * * @param configurationValue - The configuration which contains the encrypted value. * * @return decrypted value */ public String getDecryptedConfigurationValue(ConfigurationValue configurationValue) { String encryptedValue = configurationHelper.getProperty(configurationValue); if (StringUtils.isBlank(encryptedValue)) { throw new IllegalStateException("Unable to decrypt configuration value \"" + configurationValue.getKey() + "\" since it is not configured."); } try { return kmsDao.decrypt(awsHelper.getAwsParamsDto(), encryptedValue); } catch (Exception e) { throw new IllegalStateException("Error decrypting configuration value \"" + configurationValue.getKey() + "\".", e); } } }
/** * Gets the decrypted value for the given configuration option. * * @param configurationValue - The configuration which contains the encrypted value. * * @return decrypted value */ public String getDecryptedConfigurationValue(ConfigurationValue configurationValue) { String encryptedValue = configurationHelper.getProperty(configurationValue); if (StringUtils.isBlank(encryptedValue)) { throw new IllegalStateException("Unable to decrypt configuration value \"" + configurationValue.getKey() + "\" since it is not configured."); } try { return kmsDao.decrypt(awsHelper.getAwsParamsDto(), encryptedValue); } catch (Exception e) { throw new IllegalStateException("Error decrypting configuration value \"" + configurationValue.getKey() + "\".", e); } } }
/** * Gets a temporary session token that is only good for downloading the specified object key from the given bucket for a limited amount of time. * * @param storageEntity The storage entity of the external storage. * @param sessionName The session name to use for the temporary credentials. * @param awsPolicyBuilder The AWS policy builder. * * @return {@link Credentials} temporary session token */ private Credentials getDownloaderCredentials(StorageEntity storageEntity, String sessionName, AwsPolicyBuilder awsPolicyBuilder) { return stsDao.getTemporarySecurityCredentials(awsHelper.getAwsParamsDto(), sessionName, getStorageDownloadRoleArn(storageEntity), getStorageDownloadSessionDuration(storageEntity), awsPolicyBuilder.build()); }
/** * Gets a temporary session token that is only good for downloading the specified object key from the given bucket for a limited amount of time. * * @param storageEntity The storage entity of the external storage. * @param sessionName The session name to use for the temporary credentials. * @param awsPolicyBuilder The AWS policy builder. * * @return {@link Credentials} temporary session token */ private Credentials getDownloaderCredentials(StorageEntity storageEntity, String sessionName, AwsPolicyBuilder awsPolicyBuilder) { return stsDao.getTemporarySecurityCredentials(awsHelper.getAwsParamsDto(), sessionName, getStorageDownloadRoleArn(storageEntity), getStorageDownloadSessionDuration(storageEntity), awsPolicyBuilder.build()); }
@Override public void assertS3ObjectKeyDoesNotExist(String bucketName, String key) { S3FileTransferRequestParamsDto s3FileTransferRequestParamsDto = new S3FileTransferRequestParamsDto(); s3FileTransferRequestParamsDto.setS3BucketName(bucketName); s3FileTransferRequestParamsDto.setS3KeyPrefix(key); AwsParamsDto awsParamsDto = awsHelper.getAwsParamsDto(); String httpProxyHost = awsParamsDto.getHttpProxyHost(); s3FileTransferRequestParamsDto.setHttpProxyHost(httpProxyHost); Integer httpProxyPort = awsParamsDto.getHttpProxyPort(); s3FileTransferRequestParamsDto.setHttpProxyPort(httpProxyPort); Assert.isTrue(!s3Dao.s3FileExists(s3FileTransferRequestParamsDto), String.format("A S3 object already exists in bucket \"%s\" and key \"%s\".", bucketName, key)); }
@Override public void assertS3ObjectKeyDoesNotExist(String bucketName, String key) { S3FileTransferRequestParamsDto s3FileTransferRequestParamsDto = new S3FileTransferRequestParamsDto(); s3FileTransferRequestParamsDto.setS3BucketName(bucketName); s3FileTransferRequestParamsDto.setS3KeyPrefix(key); AwsParamsDto awsParamsDto = awsHelper.getAwsParamsDto(); String httpProxyHost = awsParamsDto.getHttpProxyHost(); s3FileTransferRequestParamsDto.setHttpProxyHost(httpProxyHost); Integer httpProxyPort = awsParamsDto.getHttpProxyPort(); s3FileTransferRequestParamsDto.setHttpProxyPort(httpProxyPort); Assert.isTrue(!s3Dao.s3FileExists(s3FileTransferRequestParamsDto), String.format("A S3 object already exists in bucket \"%s\" and key \"%s\".", bucketName, key)); }
/** * Gets a temporary session token that is only good for downloading the specified object key from the given bucket for a limited amount of time. * * @param storageEntity The storage entity of the external storage. * @param sessionName The session name to use for the temporary credentials. * @param s3ObjectKey The S3 object key of the path to the data in the bucket. * * @return {@link Credentials} temporary session token */ private Credentials getDownloaderCredentialsNoKmsKey(StorageEntity storageEntity, String sessionName, String s3ObjectKey) { return stsDao.getTemporarySecurityCredentials(awsHelper.getAwsParamsDto(), sessionName, getStorageDownloadRoleArn(storageEntity), getStorageDownloadSessionDuration(storageEntity), createDownloaderPolicy(storageHelper.getStorageBucketName(storageEntity), s3ObjectKey)); }
/** * Gets a temporary session token that is only good for downloading the specified object key from the given bucket for a limited amount of time. * * @param storageEntity The storage entity of the external storage. * @param sessionName The session name to use for the temporary credentials. * @param s3ObjectKey The S3 object key of the path to the data in the bucket. * * @return {@link Credentials} temporary session token */ private Credentials getDownloaderCredentialsNoKmsKey(StorageEntity storageEntity, String sessionName, String s3ObjectKey) { return stsDao.getTemporarySecurityCredentials(awsHelper.getAwsParamsDto(), sessionName, getStorageDownloadRoleArn(storageEntity), getStorageDownloadSessionDuration(storageEntity), createDownloaderPolicy(storageHelper.getStorageBucketName(storageEntity), s3ObjectKey)); }
/** * Gets a JMS connection factory. * * @return the JMS connection factory. */ @Bean public ConnectionFactory jmsConnectionFactory() { AwsParamsDto awsParamsDto = awsHelper.getAwsParamsDto(); ClientConfiguration clientConfiguration = new ClientConfiguration(); // Only set the proxy hostname and/or port if they're configured. if (StringUtils.isNotBlank(awsParamsDto.getHttpProxyHost())) { clientConfiguration.setProxyHost(awsParamsDto.getHttpProxyHost()); } if (awsParamsDto.getHttpProxyPort() != null) { clientConfiguration.setProxyPort(awsParamsDto.getHttpProxyPort()); } return SQSConnectionFactory.builder().withClientConfiguration(clientConfiguration).build(); }
/** * Gets a JMS connection factory. * * @return the JMS connection factory. */ @Bean public ConnectionFactory jmsConnectionFactory() { AwsParamsDto awsParamsDto = awsHelper.getAwsParamsDto(); ClientConfiguration clientConfiguration = new ClientConfiguration(); // Only set the proxy hostname and/or port if they're configured. if (StringUtils.isNotBlank(awsParamsDto.getHttpProxyHost())) { clientConfiguration.setProxyHost(awsParamsDto.getHttpProxyHost()); } if (awsParamsDto.getHttpProxyPort() != null) { clientConfiguration.setProxyPort(awsParamsDto.getHttpProxyPort()); } return SQSConnectionFactory.builder().withClientConfiguration(clientConfiguration).build(); }
@Test public void testAddSecurityGroupsToEc2Instance() { // Add the security groups to an EC2 instance. List<String> testSecurityGroups = Arrays.asList(EC2_SECURITY_GROUP_1, EC2_SECURITY_GROUP_2); List<String> resultSecurityGroups = ec2Dao.addSecurityGroupsToEc2Instance(EC2_INSTANCE_ID, testSecurityGroups, awsHelper.getAwsParamsDto()); // Validate the results. assertNotNull(resultSecurityGroups); assertTrue(resultSecurityGroups.containsAll(testSecurityGroups)); }
/** * Gets a temporary session token that is only good for downloading the specified object key from the given bucket for a limited amount of time. * * @param storageEntity The storage entity of the external storage * @param sessionName the session name to use for the temporary credentials. * @param s3ObjectKey the S3 object key of the path to the data in the bucket. * * @return {@link Credentials} temporary session token */ private Credentials getExternalDownloaderCredentials(StorageEntity storageEntity, String sessionName, String s3ObjectKey) { return stsDao.getTemporarySecurityCredentials(awsHelper.getAwsParamsDto(), sessionName, getStorageDownloadRoleArn(storageEntity), getStorageDownloadSessionDuration(storageEntity), createDownloaderPolicy(storageHelper.getStorageBucketName(storageEntity), s3ObjectKey, storageHelper.getStorageKmsKeyId(storageEntity))); }
/** * Gets a temporary session token that is only good for downloading the specified object key from the given bucket for a limited amount of time. * * @param storageEntity The storage entity of the external storage * @param sessionName the session name to use for the temporary credentials. * @param s3ObjectKey the S3 object key of the path to the data in the bucket. * * @return {@link Credentials} temporary session token */ private Credentials getExternalDownloaderCredentials(StorageEntity storageEntity, String sessionName, String s3ObjectKey) { return stsDao.getTemporarySecurityCredentials(awsHelper.getAwsParamsDto(), sessionName, getStorageDownloadRoleArn(storageEntity), getStorageDownloadSessionDuration(storageEntity), createDownloaderPolicy(storageHelper.getStorageBucketName(storageEntity), s3ObjectKey, storageHelper.getStorageKmsKeyId(storageEntity))); }
/** * Deletes a source S3 object based on the given bucket name and file path. * * @param s3BucketName the S3 bucket name * @param storageFilePath the storage file path * @param businessObjectDataKey the business object key */ private void deleteSourceS3ObjectAfterError(String s3BucketName, String storageFilePath, BusinessObjectDataKey businessObjectDataKey) { // Delete the file from S3 if storage file information exists. if (!StringUtils.isEmpty(storageFilePath)) { try { // Delete the source file from S3. AwsParamsDto awsParams = awsHelper.getAwsParamsDto(); S3FileTransferRequestParamsDto s3FileTransferRequestParamsDto = S3FileTransferRequestParamsDto.builder().withS3BucketName(s3BucketName).withS3KeyPrefix(storageFilePath) .withHttpProxyHost(awsParams.getHttpProxyHost()).withHttpProxyPort(awsParams.getHttpProxyPort()).build(); s3Dao.deleteDirectory(s3FileTransferRequestParamsDto); } catch (Exception e) { LOGGER.error("Failed to delete source business object data file. s3Key=\"{}\" sourceS3BucketName=\"{}\" sourceBusinessObjectDataKey={}", storageFilePath, s3BucketName, jsonHelper.objectToJson(businessObjectDataKey), e); } } }
/** * Deletes a source S3 object based on the given bucket name and file path. * * @param s3BucketName the S3 bucket name * @param storageFilePath the storage file path * @param businessObjectDataKey the business object key */ private void deleteSourceS3ObjectAfterError(String s3BucketName, String storageFilePath, BusinessObjectDataKey businessObjectDataKey) { // Delete the file from S3 if storage file information exists. if (!StringUtils.isEmpty(storageFilePath)) { try { // Delete the source file from S3. AwsParamsDto awsParams = awsHelper.getAwsParamsDto(); S3FileTransferRequestParamsDto s3FileTransferRequestParamsDto = S3FileTransferRequestParamsDto.builder().withS3BucketName(s3BucketName).withS3KeyPrefix(storageFilePath) .withHttpProxyHost(awsParams.getHttpProxyHost()).withHttpProxyPort(awsParams.getHttpProxyPort()).build(); s3Dao.deleteDirectory(s3FileTransferRequestParamsDto); } catch (Exception e) { LOGGER.error("Failed to delete source business object data file. s3Key=\"{}\" sourceS3BucketName=\"{}\" sourceBusinessObjectDataKey={}", storageFilePath, s3BucketName, jsonHelper.objectToJson(businessObjectDataKey), e); } } }
@Test public void testGetAwsParamsDto() throws Exception { // Get AWS parameters DTO. AwsParamsDto resultAwsParamsDto = awsHelper.getAwsParamsDto(); // Validate the results. // Since local users could set environment variables with a real HTTP proxy to test the real application, we can't update the environment to test // specific values. Instead, we can only test that the returned DTO contains the values in the environment. assertEquals(configurationHelper.getProperty(ConfigurationValue.HTTP_PROXY_HOST), resultAwsParamsDto.getHttpProxyHost()); assertEquals(configurationHelper.getProperty(ConfigurationValue.HTTP_PROXY_PORT, Integer.class), resultAwsParamsDto.getHttpProxyPort()); assertNotNull(resultAwsParamsDto); }
@Test public void testGetEc2Client() throws Exception { // Get the EMR client with proxy configuration. AwsParamsDto awsParamsDto = awsHelper.getAwsParamsDto(); awsParamsDto.setHttpProxyHost(HTTP_PROXY_HOST); awsParamsDto.setHttpProxyPort(HTTP_PROXY_PORT); assertNotNull(ec2Dao.getEc2Client(awsParamsDto)); // Set the proxy host as blank to get the EMR client without proxy. awsParamsDto.setHttpProxyHost(BLANK_TEXT); awsParamsDto.setHttpProxyPort(HTTP_PROXY_PORT); assertNotNull(ec2Dao.getEc2Client(awsParamsDto)); // Set the proxy port as null to get the EMR client without proxy. awsParamsDto.setHttpProxyHost(HTTP_PROXY_HOST); awsParamsDto.setHttpProxyPort(null); assertNotNull(ec2Dao.getEc2Client(awsParamsDto)); // Set the proxy host as blank and proxy port as null to get the EMR client without proxy. awsParamsDto.setHttpProxyHost(BLANK_TEXT); awsParamsDto.setHttpProxyPort(null); assertNotNull(ec2Dao.getEc2Client(awsParamsDto)); }