@Test public void s3Test() throws IOException { AmazonS3 s3 = AmazonS3ClientBuilder .standard() .withEndpointConfiguration(localstack.getEndpointConfiguration(S3)) .withCredentials(localstack.getDefaultCredentialsProvider()) .build(); s3.createBucket("foo"); s3.putObject("foo", "bar", "baz"); final List<Bucket> buckets = s3.listBuckets(); assertEquals("The created bucket is present", 1, buckets.size()); final Bucket bucket = buckets.get(0); assertEquals("The created bucket has the right name", "foo", bucket.getName()); assertEquals("The created bucket has the right name", "foo", bucket.getName()); final ObjectListing objectListing = s3.listObjects("foo"); assertEquals("The created bucket has 1 item in it", 1, objectListing.getObjectSummaries().size()); final S3Object object = s3.getObject("foo", "bar"); final String content = IOUtils.toString(object.getObjectContent(), Charset.forName("UTF-8")); assertEquals("The object can be retrieved", "baz", content); } }
public S3Aws(S3Params s3Params, String region){ s3Client = AmazonS3ClientBuilder.standard() .withCredentials(new AWSStaticCredentialsProvider(new BasicAWSCredentials(s3Params.getAccessKey(), s3Params.getSecretKey()))) .withClientConfiguration(S3URLConnection.buildClientConfig()) .withPathStyleAccessEnabled(true) .withEndpointConfiguration(new AwsClientBuilder.EndpointConfiguration(s3Params.getEndpoint(), region)) .build(); }
private static Connector createClient(Map<String, Object> cfg) { String url = (String) cfg.get("url"); String accessKey = (String) cfg.get("accessKey"); String secretKey = (String) cfg.get("secretKey"); String bucketName = (String) cfg.get("bucketName"); Supplier<AmazonS3> client = () -> { log.info("createClient -> connecting to {}...", url); AWSCredentials credentials = new BasicAWSCredentials(accessKey, secretKey); AmazonS3 s3 = AmazonS3ClientBuilder.standard() .withClientConfiguration(new ClientConfiguration()) .withEndpointConfiguration(new AwsClientBuilder.EndpointConfiguration(url, "other-v2-signature")) .withCredentials(new AWSStaticCredentialsProvider(credentials)) .enablePathStyleAccess() .build(); try { createBucket(s3, bucketName); } catch (Exception e) { log.error("createClient -> error while creating buckets", e); throw e; } return s3; }; return new Connector(client, bucketName); }
@Override protected AmazonS3 newS3Client(PluginTask task) { RiakCsPluginTask t = (RiakCsPluginTask) task; return super .defaultS3ClientBuilder(task) .withEndpointConfiguration(new EndpointConfiguration(t.getEndpoint(), null)) .build(); }
private void initS3() { credentials = new BasicAWSCredentials(accessKey, secretKey); ClientConfiguration clientConfiguration = new ClientConfiguration(); clientConfiguration.setProtocol(Protocol.HTTP); s3 = AmazonS3ClientBuilder.standard() .withPathStyleAccessEnabled(true) .withCredentials(this) .withClientConfiguration(clientConfiguration) .withEndpointConfiguration(new EndpointConfiguration(endpoint, region)) .build(); }
public void init(boolean connect) { AWSCredentials credentials; if (!Strings.isNullOrEmpty(accessKey) || !Strings.isNullOrEmpty(secretKey)) { credentials = new BasicAWSCredentials(accessKey, secretKey); } else { credentials = new AnonymousAWSCredentials(); } AmazonS3ClientBuilder builder = AmazonS3ClientBuilder.standard() .withCredentials(new AWSStaticCredentialsProvider(credentials)) .withClientConfiguration(new ClientConfiguration() .withMaxConnections(Runtime.getRuntime().availableProcessors() * 10) .withMaxErrorRetry(50)); if (!Strings.isNullOrEmpty(region)) { builder = builder.withRegion(region); } if (!Strings.isNullOrEmpty(endpoint)) { builder = builder.withEndpointConfiguration(new AwsClientBuilder.EndpointConfiguration(endpoint, null)); } this.client = builder.build(); if (connect) { // testing connection this.client.listObjects(bucket); } }
protected void before() throws Throwable { s3Mock = S3Mock.create(Helper.findRandomOpenPortOnAllLocalInterfaces()); localAddress = s3Mock.start().localAddress(); amazonS3 = AmazonS3ClientBuilder.standard() .withEndpointConfiguration(new EndpointConfiguration(getEndpoint(), "eu-west-1")) .build(); buckets.forEach(amazonS3::createBucket); files.forEach(fakeFile -> amazonS3.putObject(fakeFile.bucket, fakeFile.name, fakeFile.content)); }
public static MockedS3Client createInstance(String region) { AwsClientBuilder.EndpointConfiguration endpointConfiguration = new AwsClientBuilder.EndpointConfiguration(S3_MOCK_URL, region); AmazonS3 client = AmazonS3ClientBuilder .standard() .withPathStyleAccessEnabled(true) .withEndpointConfiguration(endpointConfiguration) .build(); return new MockedS3Client(client); }
private AmazonS3 newInstance(String region, S3S3CopierOptions s3s3CopierOptions) { HadoopAWSCredentialProviderChain credentialsChain = getCredentialsProviderChain(); AmazonS3ClientBuilder builder = AmazonS3ClientBuilder.standard().withCredentials(credentialsChain); URI s3Endpoint = s3s3CopierOptions.getS3Endpoint(region); if (s3Endpoint != null) { EndpointConfiguration endpointConfiguration = new EndpointConfiguration(s3Endpoint.toString(), region); builder.withEndpointConfiguration(endpointConfiguration); } else { builder.withRegion(region); } return builder.build(); }
private AmazonS3 newInstance(String region, S3S3CopierOptions s3s3CopierOptions) { HadoopAWSCredentialProviderChain credentialsChain = getCredentialsProviderChain(); AmazonS3ClientBuilder builder = AmazonS3ClientBuilder.standard().withCredentials(credentialsChain); URI s3Endpoint = s3s3CopierOptions.getS3Endpoint(region); if (s3Endpoint != null) { EndpointConfiguration endpointConfiguration = new EndpointConfiguration(s3Endpoint.toString(), region); builder.withEndpointConfiguration(endpointConfiguration); } else { builder.withRegion(region); } return builder.build(); }
/** * Creates an {@link AmazonS3} client instance that is configured to call the started S3Mock * server using HTTPS for a given region. * * @param region Region to define regional endpoint. * * @return The {@link AmazonS3} instance. */ public AmazonS3 createS3Client(final String region) { final BasicAWSCredentials credentials = new BasicAWSCredentials("foo", "bar"); return AmazonS3ClientBuilder.standard() .withCredentials(new AWSStaticCredentialsProvider(credentials)) .withClientConfiguration( configureClientToIgnoreInvalidSslCertificates(new ClientConfiguration())) .withEndpointConfiguration(getEndpointCongiguration(region)) .enablePathStyleAccess() .build(); }
private AmazonS3 newGlobalInstance(S3S3CopierOptions s3s3CopierOptions) { HadoopAWSCredentialProviderChain credentialsChain = getCredentialsProviderChain(); AmazonS3ClientBuilder builder = AmazonS3ClientBuilder .standard() .withForceGlobalBucketAccessEnabled(Boolean.TRUE) .withCredentials(credentialsChain); URI s3Endpoint = s3s3CopierOptions.getS3Endpoint(); if (s3Endpoint != null) { EndpointConfiguration endpointConfiguration = new EndpointConfiguration(s3Endpoint.toString(), Region.US_Standard.getFirstRegionId()); builder.withEndpointConfiguration(endpointConfiguration); } return builder.build(); }
private AmazonS3 newGlobalInstance(S3S3CopierOptions s3s3CopierOptions) { HadoopAWSCredentialProviderChain credentialsChain = getCredentialsProviderChain(); AmazonS3ClientBuilder builder = AmazonS3ClientBuilder .standard() .withForceGlobalBucketAccessEnabled(Boolean.TRUE) .withCredentials(credentialsChain); URI s3Endpoint = s3s3CopierOptions.getS3Endpoint(); if (s3Endpoint != null) { EndpointConfiguration endpointConfiguration = new EndpointConfiguration(s3Endpoint.toString(), Region.US_Standard.getFirstRegionId()); builder.withEndpointConfiguration(endpointConfiguration); } return builder.build(); }
public AmazonS3 newInstance(Configuration conf) { int maxErrorRetry = conf.getInt(ConfigurationVariable.UPLOAD_RETRY_COUNT.getName(), ConfigurationVariable.UPLOAD_RETRY_COUNT.defaultIntValue()); long errorRetryDelay = conf.getLong(ConfigurationVariable.UPLOAD_RETRY_DELAY_MS.getName(), ConfigurationVariable.UPLOAD_RETRY_DELAY_MS.defaultLongValue()); LOG.info("Creating AWS S3 client with a retry policy of {} retries and {} ms of exponential backoff delay", maxErrorRetry, errorRetryDelay); RetryPolicy retryPolicy = new RetryPolicy(new CounterBasedRetryCondition(maxErrorRetry), new ExponentialBackoffStrategy(errorRetryDelay), maxErrorRetry, true); ClientConfiguration clientConfiguration = new ClientConfiguration(); clientConfiguration.setRetryPolicy(retryPolicy); clientConfiguration.setMaxErrorRetry(maxErrorRetry); AmazonS3ClientBuilder builder = AmazonS3ClientBuilder .standard() .withCredentials(new HadoopAWSCredentialProviderChain(conf)) .withClientConfiguration(clientConfiguration); EndpointConfiguration endpointConfiguration = getEndpointConfiguration(conf); if (endpointConfiguration != null) { builder.withEndpointConfiguration(endpointConfiguration); } else { builder.withRegion(getRegion(conf)); } return builder.build(); }
public AmazonS3 newInstance(Configuration conf) { int maxErrorRetry = conf.getInt(ConfigurationVariable.UPLOAD_RETRY_COUNT.getName(), ConfigurationVariable.UPLOAD_RETRY_COUNT.defaultIntValue()); long errorRetryDelay = conf.getLong(ConfigurationVariable.UPLOAD_RETRY_DELAY_MS.getName(), ConfigurationVariable.UPLOAD_RETRY_DELAY_MS.defaultLongValue()); LOG.info("Creating AWS S3 client with a retry policy of {} retries and {} ms of exponential backoff delay", maxErrorRetry, errorRetryDelay); RetryPolicy retryPolicy = new RetryPolicy(new CounterBasedRetryCondition(maxErrorRetry), new ExponentialBackoffStrategy(errorRetryDelay), maxErrorRetry, true); ClientConfiguration clientConfiguration = new ClientConfiguration(); clientConfiguration.setRetryPolicy(retryPolicy); clientConfiguration.setMaxErrorRetry(maxErrorRetry); AmazonS3ClientBuilder builder = AmazonS3ClientBuilder .standard() .withCredentials(new HadoopAWSCredentialProviderChain(conf)) .withClientConfiguration(clientConfiguration); EndpointConfiguration endpointConfiguration = getEndpointConfiguration(conf); if (endpointConfiguration != null) { builder.withEndpointConfiguration(endpointConfiguration); } else { builder.withRegion(getRegion(conf)); } return builder.build(); }
builder = builder.withEndpointConfiguration(new EndpointConfiguration( endpoint, clientRegion)); builder = builder.withPathStyleAccessEnabled(pathStyleAccess);
@Override protected AmazonS3 getS3Connection() { this.endpoint = new EndpointConfiguration(S3_TEST_URL, S3_TEST_REGION); AmazonS3 s3ClientTest = AmazonS3ClientBuilder .standard() .withPathStyleAccessEnabled(true) .withEndpointConfiguration(endpoint) .withCredentials(new AWSStaticCredentialsProvider(new AnonymousAWSCredentials())) .build(); s3ClientTest.createBucket(S3_TEST_BUCKET_NAME); return spy(s3ClientTest); } }
@Override public AmazonS3ClientBuilder createBuilder(S3Options s3Options) { AmazonS3ClientBuilder builder = AmazonS3ClientBuilder.standard().withCredentials(s3Options.getAwsCredentialsProvider()); if (s3Options.getClientConfiguration() != null) { builder = builder.withClientConfiguration(s3Options.getClientConfiguration()); } if (Strings.isNullOrEmpty(s3Options.getAwsServiceEndpoint())) { builder = builder.withRegion(s3Options.getAwsRegion()); } else { builder = builder.withEndpointConfiguration( new AwsClientBuilder.EndpointConfiguration( s3Options.getAwsServiceEndpoint(), s3Options.getAwsRegion())); } return builder; } }
private AmazonS3 getMockS3Connection() { final EndpointConfiguration endpoint = new EndpointConfiguration(S3_TEST_URL, S3_TEST_REGION); AmazonS3 s3ClientTest = AmazonS3ClientBuilder .standard() .withPathStyleAccessEnabled(true) .withEndpointConfiguration(endpoint) .withCredentials(new AWSStaticCredentialsProvider(new AnonymousAWSCredentials())) .build(); return s3ClientTest; } }
public static AmazonS3 buildS3(LocalServerConfig configuration) { String region = configuration.getProperty(pRegion); String endpoint = configuration.getProperty(pEndpoint); String credentialsFile = configuration.getProperty(pCredentialFile); String credentialsProfile = configuration.getProperty(pCredentialProfile); AmazonS3ClientBuilder builder = AmazonS3ClientBuilder.standard(); if ( endpoint == null ) builder.withRegion(region); else { // Needed for S3mock builder.withPathStyleAccessEnabled(true); builder.withEndpointConfiguration(new EndpointConfiguration(endpoint, region)); builder.withCredentials(new AWSStaticCredentialsProvider(new AnonymousAWSCredentials())); } if ( credentialsFile != null ) builder.withCredentials(new ProfileCredentialsProvider(credentialsFile, credentialsProfile)); return builder.build(); }