/** * Constructs a new client to invoke service methods on S3 using the specified parameters. All * service calls made using this new client object are blocking, and will not return until the * service call completes. * * @param s3ClientParams Object providing S3 client parameters. * @see AmazonS3ClientBuilder For a fluent way to construct a client. */ @SdkInternalApi AmazonS3Client(AmazonS3ClientParams s3ClientParams) { super(s3ClientParams.getClientParams()); this.awsCredentialsProvider = s3ClientParams.getClientParams().getCredentialsProvider(); this.skipMd5CheckStrategy = SkipMd5CheckStrategy.INSTANCE; setS3ClientOptions(s3ClientParams.getS3ClientOptions()); init(); }
private void initalizeEndpointOverride(final ProcessContext context, final AmazonS3Client s3) { // if ENDPOINT_OVERRIDE is set, use PathStyleAccess if(StringUtils.trimToEmpty(context.getProperty(ENDPOINT_OVERRIDE).evaluateAttributeExpressions().getValue()).isEmpty() == false){ final S3ClientOptions s3Options = new S3ClientOptions(); s3Options.setPathStyleAccess(true); s3.setS3ClientOptions(s3Options); } }
amazonS3Client.setS3ClientOptions(clientOptions);
@Test public void testCreateRequestAccelerate() { s3.setS3ClientOptions(accelerateOption); final String bucketName = "bucket"; final String key = "key"; final File file = new File(key); final HttpMethodName method = HttpMethodName.PUT; final PutObjectRequest originalRequest = new PutObjectRequest(bucketName, key, file); final Request<?> request = s3.createRequest(bucketName, key, originalRequest, method); assertEquals(String.format("%s.s3-accelerate.amazonaws.com", bucketName), request.getEndpoint().getHost()); assertEquals(method, request.getHttpMethod()); }
@Test public void testCreateRequestAccelerateUnsupportedMethod() { s3.setS3ClientOptions(accelerateOption); final String bucketName = "bucket"; final HttpMethodName method = HttpMethodName.DELETE; final DeleteBucketRequest originalRequest = new DeleteBucketRequest(bucketName); final Request<?> request = s3.createRequest(bucketName, null, originalRequest, method); assertEquals(String.format("%s.s3.amazonaws.com", bucketName), request.getEndpoint().getHost()); assertEquals(method, request.getHttpMethod()); }
@Test public void testCreateSignerWithSpecialCharacterKeys() { s3.setS3ClientOptions(accelerateOption); final Regions region = Regions.US_WEST_2; s3.setRegion(Region.getRegion(region)); final String bucketName = "bucket"; final String key = "key%^!@#*()"; final HttpMethodName method = HttpMethodName.GET; final GetObjectRequest originalRequest = new GetObjectRequest(bucketName, key); final Request<?> request = s3.createRequest(bucketName, key, originalRequest, method); final Signer signer = s3.createSigner(request, bucketName, key); assertTrue(signer instanceof AWSS3V4Signer); signer.sign(request, creds); final String authorization = request.getHeaders().get("Authorization"); assertNotNull(authorization); final String regionName = authorization.split("/")[2]; assertEquals(region.getName(), regionName); assertTrue(request.getResourcePath().contains(key)); }
@Test public void testCreateSigner() { s3.setS3ClientOptions(accelerateOption); final Regions region = Regions.US_WEST_2; s3.setRegion(Region.getRegion(region)); final String bucketName = "bucket"; final String key = "key"; final HttpMethodName method = HttpMethodName.GET; final GetObjectRequest originalRequest = new GetObjectRequest(bucketName, key); final Request<?> request = s3.createRequest(bucketName, key, originalRequest, method); final Signer signer = s3.createSigner(request, bucketName, key); assertTrue(signer instanceof AWSS3V4Signer); signer.sign(request, creds); final String authorization = request.getHeaders().get("Authorization"); assertNotNull(authorization); final String regionName = authorization.split("/")[2]; assertEquals(region.getName(), regionName); }
private void initalizeEndpointOverride(final ProcessContext context, final AmazonS3Client s3) { // if ENDPOINT_OVERRIDE is set, use PathStyleAccess if(StringUtils.trimToEmpty(context.getProperty(ENDPOINT_OVERRIDE).evaluateAttributeExpressions().getValue()).isEmpty() == false){ final S3ClientOptions s3Options = new S3ClientOptions(); s3Options.setPathStyleAccess(true); s3.setS3ClientOptions(s3Options); } }
/** * Constructs a new client to invoke service methods on S3 using the specified parameters. All * service calls made using this new client object are blocking, and will not return until the * service call completes. * * @param s3ClientParams Object providing S3 client parameters. * @see AmazonS3ClientBuilder For a fluent way to construct a client. */ @SdkInternalApi AmazonS3Client(AmazonS3ClientParams s3ClientParams) { super(s3ClientParams.getClientParams()); this.awsCredentialsProvider = s3ClientParams.getClientParams().getCredentialsProvider(); this.skipMd5CheckStrategy = SkipMd5CheckStrategy.INSTANCE; setS3ClientOptions(s3ClientParams.getS3ClientOptions()); init(); }
static AmazonS3Client configureClient(AmazonS3Client s3Client) { if (System.getProperty(S3_REGION) != null) { String region = System.getProperty(S3_REGION); Log.debug("S3 region specified: ", region); s3Client.setRegion(RegionUtils.getRegion(region)); } // Region overrides end-point settings if (System.getProperty(S3_END_POINT) != null) { String endPoint = System.getProperty(S3_END_POINT); Log.debug("S3 endpoint specified: ", endPoint); s3Client.setEndpoint(endPoint); } if (System.getProperty(S3_ENABLE_PATH_STYLE) != null && Boolean.valueOf(System.getProperty(S3_ENABLE_PATH_STYLE))) { Log.debug("S3 path style access enabled"); S3ClientOptions sco = new S3ClientOptions(); sco.setPathStyleAccess(true); s3Client.setS3ClientOptions(sco); } return s3Client; }
private AmazonS3Client getAmazonS3Client(BackupRestoreContext ctx) throws URISyntaxException { final String accessKey = ctx.getAccountId(); final String secretKey = ctx.getSecretKey(); String endpoint = getEndpoint(ctx); LOGGER.info("endpoint: {}", endpoint); final BasicAWSCredentials basicAWSCredentials = new BasicAWSCredentials(accessKey, secretKey); final AmazonS3Client amazonS3Client = new AmazonS3Client(basicAWSCredentials); amazonS3Client.setEndpoint(endpoint); if (ctx.usesEmc()) { final S3ClientOptions options = new S3ClientOptions(); options.setPathStyleAccess(true); amazonS3Client.setS3ClientOptions(options); } return amazonS3Client; }
void setBinaryTransferAccelerationEnabled(boolean enabled) { if (enabled) { // verify acceleration is enabled on the bucket BucketAccelerateConfiguration accelerateConfig = s3service.getBucketAccelerateConfiguration(new GetBucketAccelerateConfigurationRequest(bucket)); if (accelerateConfig.isAccelerateEnabled()) { // If transfer acceleration is enabled for presigned URIs, we need a separate AmazonS3Client // instance with the acceleration mode enabled, because we don't want the requests from the // data store itself to S3 to use acceleration s3PresignService = Utils.openService(properties); s3PresignService.setS3ClientOptions(S3ClientOptions.builder().setAccelerateModeEnabled(true).build()); LOG.info("S3 Transfer Acceleration enabled for presigned URIs."); } else { LOG.warn("S3 Transfer Acceleration is not enabled on the bucket {}. Will create normal, non-accelerated presigned URIs.", bucket, S3Constants.PRESIGNED_URI_ENABLE_ACCELERATION); } } else { s3PresignService = s3service; } }
void setBinaryTransferAccelerationEnabled(boolean enabled) { if (enabled) { // verify acceleration is enabled on the bucket BucketAccelerateConfiguration accelerateConfig = s3service.getBucketAccelerateConfiguration(new GetBucketAccelerateConfigurationRequest(bucket)); if (accelerateConfig.isAccelerateEnabled()) { // If transfer acceleration is enabled for presigned URIs, we need a separate AmazonS3Client // instance with the acceleration mode enabled, because we don't want the requests from the // data store itself to S3 to use acceleration s3PresignService = Utils.openService(properties); s3PresignService.setS3ClientOptions(S3ClientOptions.builder().setAccelerateModeEnabled(true).build()); LOG.info("S3 Transfer Acceleration enabled for presigned URIs."); } else { LOG.warn("S3 Transfer Acceleration is not enabled on the bucket {}. Will create normal, non-accelerated presigned URIs.", bucket, S3Constants.PRESIGNED_URI_ENABLE_ACCELERATION); } } else { s3PresignService = s3service; } }
@Override public void prepare(Object configurationObject) { lineWriterUtil = LineReadWriteUtil.getInstance(s3WriterConfiguration); // Connect to S3 synchronized (this) { try { // if the user has chosen to not set the object mapper, then set a default object mapper for them. if (this.objectMapper == null) { this.objectMapper = StreamsJacksonMapper.getInstance(); } // Create the credentials Object if (this.amazonS3Client == null) { AWSCredentials credentials = new BasicAWSCredentials(s3WriterConfiguration.getKey(), s3WriterConfiguration.getSecretKey()); ClientConfiguration clientConfig = new ClientConfiguration(); clientConfig.setProtocol(Protocol.valueOf(s3WriterConfiguration.getProtocol().toString())); // We do not want path style access S3ClientOptions clientOptions = new S3ClientOptions(); clientOptions.setPathStyleAccess(false); this.amazonS3Client = new AmazonS3Client(credentials, clientConfig); if (StringUtils.isNotEmpty(s3WriterConfiguration.getRegion())) { this.amazonS3Client.setRegion(Region.getRegion(Regions.fromName(s3WriterConfiguration.getRegion()))); } this.amazonS3Client.setS3ClientOptions(clientOptions); } } catch (Exception ex) { LOGGER.error("Exception while preparing the S3 client: {}", ex); } Preconditions.checkArgument(this.amazonS3Client != null); } }
clientOptions.setPathStyleAccess(pathStyleAccess.get()); s3Client.setS3ClientOptions(clientOptions);
this.amazonS3Client.setRegion(Region.getRegion(Regions.fromName(s3ReaderConfiguration.getRegion()))); this.amazonS3Client.setS3ClientOptions(clientOptions);
S3ClientOptions s3ClientOptions = new S3ClientOptions(); s3ClientOptions.setPathStyleAccess(true); client.setS3ClientOptions(s3ClientOptions); client.setEndpoint(endpoint);
S3ClientOptions s3ClientOptions = new S3ClientOptions(); s3ClientOptions.setPathStyleAccess(true); client.setS3ClientOptions(s3ClientOptions); client.setEndpoint(endpoint);
@Override public Storage newStorage(Config config) { AmazonS3Client client = new AmazonS3Client( buildCredentialsProvider(config), buildClientConfiguration(config)); if (config.has("endpoint")) { client.setEndpoint(config.get("endpoint", String.class)); } if (config.has("path-style-access")) { client.setS3ClientOptions( S3ClientOptions.builder().setPathStyleAccess( config.get("path-style-access", Boolean.class, false) ).build()); } String bucket = config.get("bucket", String.class); return new S3Storage(config, client, bucket); }