/** * Enables path-style access to S3 buckets if configured. By default, the * behavior is to use virtual hosted-style access with URIs of the form * http://bucketname.s3.amazonaws.com. Enabling path-style access and a * region-specific endpoint switches the behavior to use URIs of the form * http://s3-eu-west-1.amazonaws.com/bucketname. * * @param s3 S3 client * @param conf Hadoop configuration */ private static void enablePathStyleAccessIfRequired(AmazonS3 s3, Configuration conf) { final boolean pathStyleAccess = conf.getBoolean(S3Constants.PATH_STYLE_ACCESS, false); if (pathStyleAccess) { logger.debug("Enabling path style access!"); s3.setS3ClientOptions(new S3ClientOptions().withPathStyleAccess(true)); } }
public static AmazonS3 s3client(Map<String, String> config) { // Use default credentials provider that looks in Env + Java properties + profile + instance role AmazonS3 s3Client = new AmazonS3Client(); // If worker config sets explicit endpoint override (e.g. for testing) use that String s3Endpoint = config.get("s3.endpoint"); if (s3Endpoint != null && !Objects.equals(s3Endpoint, "")) { s3Client.setEndpoint(s3Endpoint); } Boolean s3PathStyle = Boolean.parseBoolean(config.get("s3.path_style")); if (s3PathStyle) { s3Client.setS3ClientOptions(new S3ClientOptions().withPathStyleAccess(true)); } return s3Client; }
@Override public void start(Map<String, String> props) throws ConnectException { readConfig(props); // Use default credentials provider that looks in Env + Java properties + profile + instance role AmazonS3 s3Client = new AmazonS3Client(); // If worker config sets explicit endpoint override (e.g. for testing) use that if (overrideS3Endpoint != "") { s3Client.setEndpoint(overrideS3Endpoint); } if (s3PathStyle) { s3Client.setS3ClientOptions(new S3ClientOptions().withPathStyleAccess(true)); } s3 = new S3Writer(bucket, prefix, s3Client); // Recover initial assignments Set<TopicPartition> assignment = context.assignment(); recoverAssignment(assignment); }