private AmazonS3 resolveS3Client() { return s3Client == null ? AmazonS3ClientBuilder.defaultClient() : s3Client; }
private static List<String> listByFullPathPrefix(final String bucket, final String s3prefix) { return AmazonS3ClientBuilder.defaultClient().listObjects(bucket, s3prefix).getObjectSummaries() .stream().map(S3ObjectSummary::getKey).collect(Collectors.toList()); }
public static void deleteWebsiteConfig(String bucket_name) { final AmazonS3 s3 = AmazonS3ClientBuilder.defaultClient(); try { s3.deleteBucketWebsiteConfiguration(bucket_name); } catch (AmazonServiceException e) { System.err.println(e.getErrorMessage()); System.out.println("Failed to delete website configuration!"); System.exit(1); } }
public static void setBucketPolicy(String bucket_name, String policy_text) { final AmazonS3 s3 = AmazonS3ClientBuilder.defaultClient(); try { s3.setBucketPolicy(bucket_name, policy_text); } catch (AmazonServiceException e) { System.err.println(e.getErrorMessage()); System.exit(1); } }
public static void main(String[] args) { final AmazonS3 s3 = AmazonS3ClientBuilder.defaultClient(); List<Bucket> buckets = s3.listBuckets(); System.out.println("Your Amazon S3 buckets are:"); for (Bucket b : buckets) { System.out.println("* " + b.getName()); } } }
public static Bucket getBucket(String bucket_name) { final AmazonS3 s3 = AmazonS3ClientBuilder.defaultClient(); Bucket named_bucket = null; List<Bucket> buckets = s3.listBuckets(); for (Bucket b : buckets) { if (b.getName().equals(bucket_name)) { named_bucket = b; } } return named_bucket; }
private static void deleteFile(ParameterTool params) { final String bucket = params.getRequired("bucket"); final String s3file = params.getRequired("s3file"); AmazonS3ClientBuilder.defaultClient().deleteObject(bucket, s3file); }
private static void numberOfLinesInFile(ParameterTool params) { final String bucket = params.getRequired("bucket"); final String s3file = params.getRequired("s3file"); AmazonS3 s3client = AmazonS3ClientBuilder.defaultClient(); System.out.print(S3QueryUtil.queryFile(s3client, bucket, s3file, countQuery)); s3client.shutdown(); }
public static void getWebsiteConfig(String bucket_name) { final AmazonS3 s3 = AmazonS3ClientBuilder.defaultClient(); try { BucketWebsiteConfiguration config = s3.getBucketWebsiteConfiguration(bucket_name); if (config == null) { System.out.println("No website configuration found!"); } else { System.out.format("Index document: %s\n", config.getIndexDocumentSuffix()); System.out.format("Error document: %s\n", config.getErrorDocument()); } } catch (AmazonServiceException e) { System.err.println(e.getErrorMessage()); System.out.println("Failed to get website configuration!"); System.exit(1); } }
public static Bucket createBucket(String bucket_name) { final AmazonS3 s3 = AmazonS3ClientBuilder.defaultClient(); Bucket b = null; if (s3.doesBucketExist(bucket_name)) { System.out.format("Bucket %s already exists.\n", bucket_name); b = getBucket(bucket_name); } else { try { b = s3.createBucket(bucket_name); } catch (AmazonS3Exception e) { System.err.println(e.getErrorMessage()); } } return b; }
private static void deleteByFullPathPrefix(ParameterTool params) { final String bucket = params.getRequired("bucket"); final String s3prefix = params.getRequired("s3prefix"); String[] keys = listByFullPathPrefix(bucket, s3prefix).toArray(new String[] {}); if (keys.length > 0) { DeleteObjectsRequest request = new DeleteObjectsRequest(bucket).withKeys(keys); AmazonS3ClientBuilder.defaultClient().deleteObjects(request); } }
public static void setObjectAcl(String bucket_name, String object_key, String email, String access) { System.out.format("Setting %s access for %s\n", access, email); System.out.println("for object: " + object_key); System.out.println(" in bucket: " + bucket_name); final AmazonS3 s3 = AmazonS3ClientBuilder.defaultClient(); try { // get the current ACL AccessControlList acl = s3.getObjectAcl(bucket_name, object_key); // set access for the grantee EmailAddressGrantee grantee = new EmailAddressGrantee(email); Permission permission = Permission.valueOf(access); acl.grantPermission(grantee, permission); s3.setObjectAcl(bucket_name, object_key, acl); } catch (AmazonServiceException e) { System.err.println(e.getErrorMessage()); System.exit(1); } }
private static void numberOfLinesInFilesWithFullAndNamePrefix(ParameterTool params) { final String bucket = params.getRequired("bucket"); final String s3prefix = params.getRequired("s3prefix"); final String s3filePrefix = params.get("s3filePrefix", ""); int parallelism = params.getInt("parallelism", 10); List<String> files = listByFullPathPrefix(bucket, s3prefix); ExecutorService executor = Executors.newFixedThreadPool(parallelism); AmazonS3 s3client = AmazonS3ClientBuilder.defaultClient(); List<CompletableFuture<Integer>> requests = submitLineCountingRequestsForFilesAsync(executor, s3client, bucket, files, s3filePrefix); int count = waitAndComputeTotalLineCountResult(requests); executor.shutdownNow(); s3client.shutdown(); System.out.print(count); }
public static void getObjectAcl(String bucket_name, String object_key) { System.out.println("Retrieving ACL for object: " + object_key); System.out.println(" in bucket: " + bucket_name); final AmazonS3 s3 = AmazonS3ClientBuilder.defaultClient(); try { AccessControlList acl = s3.getObjectAcl(bucket_name, object_key); List<Grant> grants = acl.getGrantsAsList(); for (Grant grant : grants) { System.out.format(" %s: %s\n", grant.getGrantee().getIdentifier(), grant.getPermission().toString()); } } catch (AmazonServiceException e) { System.err.println(e.getErrorMessage()); System.exit(1); } }
public static void setBucketAcl(String bucket_name, String email, String access) { System.out.format("Setting %s access for %s\n", access, email); System.out.println("on bucket: " + bucket_name); final AmazonS3 s3 = AmazonS3ClientBuilder.defaultClient(); try { // get the current ACL AccessControlList acl = s3.getBucketAcl(bucket_name); // set access for the grantee EmailAddressGrantee grantee = new EmailAddressGrantee(email); Permission permission = Permission.valueOf(access); acl.grantPermission(grantee, permission); s3.setBucketAcl(bucket_name, acl); } catch (AmazonServiceException e) { System.err.println(e.getErrorMessage()); System.exit(1); } }
public static void getBucketAcl(String bucket_name) { System.out.println("Retrieving ACL for bucket: " + bucket_name); final AmazonS3 s3 = AmazonS3ClientBuilder.defaultClient(); try { AccessControlList acl = s3.getBucketAcl(bucket_name); List<Grant> grants = acl.getGrantsAsList(); for (Grant grant : grants) { System.out.format(" %s: %s\n", grant.getGrantee().getIdentifier(), grant.getPermission().toString()); } } catch (AmazonServiceException e) { System.err.println(e.getErrorMessage()); System.exit(1); } }
/** * Uses AES/CBC algorithm, no key wrapping. */ public void encryptionOnly_CustomerManagedKey() throws NoSuchAlgorithmException { SecretKey secretKey = KeyGenerator.getInstance("AES").generateKey(); AmazonS3Encryption s3Encryption = AmazonS3EncryptionClientBuilder .standard() .withRegion(Regions.US_WEST_2) .withCryptoConfiguration(new CryptoConfiguration(CryptoMode.EncryptionOnly)) .withEncryptionMaterials(new StaticEncryptionMaterialsProvider(new EncryptionMaterials(secretKey))) .build(); AmazonS3 s3NonEncrypt = AmazonS3ClientBuilder.defaultClient(); s3Encryption.putObject(BUCKET_NAME, ENCRYPTED_KEY, "some contents"); s3NonEncrypt.putObject(BUCKET_NAME, NON_ENCRYPTED_KEY, "some other contents"); System.out.println(s3Encryption.getObjectAsString(BUCKET_NAME, ENCRYPTED_KEY)); System.out.println(s3Encryption.getObjectAsString(BUCKET_NAME, NON_ENCRYPTED_KEY)); }
/** * Uses AES/GCM with AESWrap key wrapping to encrypt the key. Uses v2 metadata schema. Note that authenticated * encryption requires the bouncy castle provider to be on the classpath. Also, for authenticated encryption the size * of the data can be no longer than 64 GB. */ public void authenticatedEncryption_CustomerManagedKey() throws NoSuchAlgorithmException { SecretKey secretKey = KeyGenerator.getInstance("AES").generateKey(); AmazonS3Encryption s3Encryption = AmazonS3EncryptionClientBuilder .standard() .withRegion(Regions.US_WEST_2) .withCryptoConfiguration(new CryptoConfiguration(CryptoMode.AuthenticatedEncryption)) .withEncryptionMaterials(new StaticEncryptionMaterialsProvider(new EncryptionMaterials(secretKey))) .build(); AmazonS3 s3NonEncrypt = AmazonS3ClientBuilder.defaultClient(); s3Encryption.putObject(BUCKET_NAME, ENCRYPTED_KEY, "some contents"); s3NonEncrypt.putObject(BUCKET_NAME, NON_ENCRYPTED_KEY, "some other contents"); System.out.println(s3Encryption.getObjectAsString(BUCKET_NAME, ENCRYPTED_KEY)); System.out.println(s3Encryption.getObjectAsString(BUCKET_NAME, NON_ENCRYPTED_KEY)); }
/** * This uses the V2 metadata schema with a key wrap algorithm of 'kms' and a CEK algorithm of AES/CBC/PKCS5Padding. */ public void encryptionOnly_KmsManagedKey() throws NoSuchAlgorithmException { AmazonS3Encryption s3Encryption = AmazonS3EncryptionClientBuilder .standard() .withRegion(Regions.US_WEST_2) .withCryptoConfiguration(new CryptoConfiguration(CryptoMode.EncryptionOnly).withAwsKmsRegion(Region.getRegion(Regions.US_WEST_2))) // Can either be Key ID or alias (prefixed with 'alias/') .withEncryptionMaterials(new KMSEncryptionMaterialsProvider("alias/s3-kms-key")) .build(); AmazonS3 s3NonEncrypt = AmazonS3ClientBuilder.defaultClient(); s3Encryption.putObject(BUCKET_NAME, ENCRYPTED_KEY, "some contents"); s3NonEncrypt.putObject(BUCKET_NAME, NON_ENCRYPTED_KEY, "some other contents"); System.out.println(s3Encryption.getObjectAsString(BUCKET_NAME, ENCRYPTED_KEY)); System.out.println(s3Encryption.getObjectAsString(BUCKET_NAME, NON_ENCRYPTED_KEY)); }
/** * This uses the V2 metadata schema with a key wrap algorithm of 'kms' and a CEK algorithm of AES/GCM/NoPadding. */ public void authenticatedEncryption_KmsManagedKey() throws NoSuchAlgorithmException { AmazonS3Encryption s3Encryption = AmazonS3EncryptionClientBuilder .standard() .withRegion(Regions.US_WEST_2) .withCryptoConfiguration(new CryptoConfiguration(CryptoMode.AuthenticatedEncryption).withAwsKmsRegion(Region.getRegion(Regions.US_WEST_2))) // Can either be Key ID or alias (prefixed with 'alias/') .withEncryptionMaterials(new KMSEncryptionMaterialsProvider("alias/s3-kms-key")) .build(); AmazonS3 s3NonEncrypt = AmazonS3ClientBuilder.defaultClient(); s3Encryption.putObject(BUCKET_NAME, ENCRYPTED_KEY, "some contents"); s3NonEncrypt.putObject(BUCKET_NAME, NON_ENCRYPTED_KEY, "some other contents"); System.out.println(s3Encryption.getObjectAsString(BUCKET_NAME, ENCRYPTED_KEY)); System.out.println(s3Encryption.getObjectAsString(BUCKET_NAME, NON_ENCRYPTED_KEY)); }