/** * Returns the named bucket. * * @param bucketName * the name of the bucket to find. * @return the bucket, or null if no the named bucket has found. * @throws SdkClientException */ public Bucket getBucket( String bucketName ) throws SdkClientException { return s3Client.doesBucketExistV2( bucketName ) ? new Bucket( bucketName ) : null; }
private Bucket createBucket(String bucket_name) { final AmazonS3 s3 = s3Client(); Bucket b = null; if (s3.doesBucketExistV2(bucket_name)) { System.out.format("Bucket %s already exists.\n", bucket_name); b = getBucket(bucket_name); } else { try { b = s3.createBucket(bucket_name); } catch (AmazonS3Exception e) { System.err.println(e.getErrorMessage()); } } return b; }
private Bucket createBucket(String bucket_name) { final AmazonS3 s3 = s3Client(); Bucket b = null; if (s3.doesBucketExistV2(bucket_name)) { System.out.format("Bucket %s already exists.\n", bucket_name); b = getBucket(bucket_name); } else { try { b = s3.createBucket(bucket_name); } catch (AmazonS3Exception e) { System.err.println(e.getErrorMessage()); } } return b; }
if (s3.doesBucketExistV2(bucket_name)) { System.out.format("\nCannot create the bucket. \n" + "A bucket named '%s' already exists.", bucket_name);
@Override public void createBucketIfNotExists( String stagingBucketName ) { if ( !s3Client.doesBucketExistV2( stagingBucketName ) ) { s3Client.createBucket( stagingBucketName ); } }
public boolean createBucketIfDoesNotExist(String bucket) { if (!CLIENT.doesBucketExistV2(bucket)) { CLIENT.createBucket(bucket); return true; } return false; }
private void makeBucketExist(String bucketName) { if (!s3.doesBucketExistV2(bucketName)) { LOG.info("Bucket {} does not already exists.Create it.\n", bucketName); try { CreateBucketRequest request = new CreateBucketRequest(bucketName); request.setCannedAcl(acl); s3.createBucket(request); } catch (AmazonS3Exception e) { LOG.error("Create bucket error.\n", e); } } }
@Override public PatchStore create(LocalServerConfig config) { // Early create. DetailsS3 s3 = accessS3(config); if ( ! s3.client.doesBucketExistV2(s3.bucketName) ) { FmtLog.info(Delta.DELTA_LOG, "Creating bucket %s", s3.bucketName); s3.client.createBucket(s3.bucketName); } // The usual PatchStoreZk for the index, but remembering DetailsS3 CuratorFramework client = curator(config); return new PatchStoreZkS3(client, this, s3); }
@Override public boolean isConnected() { try { return getAmazonType() != null && getAmazonType().getAmazonS3() != null && getAmazonType().getAmazonS3().doesBucketExistV2(getAmazonType().getS3Bucket()); } catch (Exception e) { return false; } }
/** * Registers Amazon S3 client * @throws Exception throws if credentials are invalid or bucket is not exist */ protected void initAmazonS3Client() throws Exception { this.amazonS3SessionCredentials = getAmazonSessionCredentials().getObject(); if(this.amazonS3SessionCredentials != null) { try { this.amazonClient = AmazonS3ClientBuilder.standard() .withCredentials(new AWSStaticCredentialsProvider(new BasicSessionCredentials(this.amazonS3SessionCredentials.getAccessKeyId(), this.amazonS3SessionCredentials.getSecretAccessKey(), this.amazonS3SessionCredentials.getSessionToken()))) .withRegion(Regions.fromName(this.amazonS3SessionCredentials.getRegion())).build(); if(! this.amazonClient.doesBucketExistV2(this.amazonS3SessionCredentials.getBucket())) { throw new Exception(String.format("Amazon S3 bucket with name '%s' doesn't exist.", this.amazonS3SessionCredentials.getBucket())); } } catch (Exception e) { throw new Exception("Amazon integration is invalid. Verify your credentials or region.", e); } } }
/** * Registers Amazon S3 client * @throws Exception throws if credentials are invalid or bucket is not exist */ protected void initAmazonS3Client() throws Exception { this.amazonS3SessionCredentials = getAmazonSessionCredentials().getObject(); if(this.amazonS3SessionCredentials != null) { try { this.amazonClient = AmazonS3ClientBuilder.standard() .withCredentials(new AWSStaticCredentialsProvider(new BasicSessionCredentials(this.amazonS3SessionCredentials.getAccessKeyId(), this.amazonS3SessionCredentials.getSecretAccessKey(), this.amazonS3SessionCredentials.getSessionToken()))) .withRegion(Regions.fromName(this.amazonS3SessionCredentials.getRegion())).build(); if(! this.amazonClient.doesBucketExistV2(this.amazonS3SessionCredentials.getBucket())) { throw new Exception(String.format("Amazon S3 bucket with name '%s' doesn't exist.", this.amazonS3SessionCredentials.getBucket())); } } catch (Exception e) { throw new Exception("Amazon integration is invalid. Verify your credentials or region.", e); } } }
private String testWriteToMockS3General(@NonNull final Configuration conf) throws IOException { final JavaRDD<AvroPayload> testData = AvroPayloadUtil.generateTestDataNew(this.jsc.get(), NUM_RECORD, StringTypes.EMPTY); final FileSinkDataConverter converter = new FileSinkDataConverter(conf, new ErrorExtractor()); final FileSinkConfiguration fileConf = new FileSinkConfiguration(conf); final AwsConfiguration awsConf = new AwsConfiguration(fileConf); final MockAwsFileSink awsMockSink = spy(new MockAwsFileSink(fileConf, converter)); awsMockSink.write(testData); final AmazonS3 MockClient = awsMockSink.getS3Client(); verify(awsMockSink, times(EXPECTED_INVOCATIONS)).write(Matchers.any(JavaRDD.class)); verify(MockClient, times(EXPECTED_PARTITION_NUM)).putObject(Matchers.any(PutObjectRequest.class)); assertTrue(MockClient.doesBucketExistV2(fileConf.getBucketName().get())); for (int i = 0 ; i < EXPECTED_PARTITION_NUM ; i++) { final Boolean objectExist = MockClient.doesObjectExist(fileConf.getBucketName().get(), awsConf.getS3FilePrefix()+ "_0000" + i); assertTrue(objectExist); } return awsConf.getS3FilePrefix(); }
@Ignore @Test public void testWriteToMockS3WithMultiOverWrite() throws IOException { final Configuration conf1 = initConfigWithAws(this.pathPrefix, OBJ_KEY_1, LOCAL1, VERSION, TIMESTAMP1, SOURCE_SUB_PATH1, S3_TEST_BUCKET_NAME); final Configuration conf2 = initConfigWithAws(this.pathPrefix, OBJ_KEY_1, LOCAL2, VERSION, TIMESTAMP2, SOURCE_SUB_PATH2, S3_TEST_BUCKET_NAME); final Configuration conf3 = initConfigWithAws(this.pathPrefix, OBJ_KEY_1, LOCAL3, OVERWRITE, TIMESTAMP3, SOURCE_SUB_PATH2, S3_TEST_BUCKET_NAME); final List<String> filePrefix = new ArrayList<String>(); filePrefix.add(testWriteToMockS3General(conf1)); filePrefix.add(testWriteToMockS3General(conf2)); filePrefix.add(testWriteToMockS3General(conf3)); final AmazonS3 MockClient = getMockS3Connection(); assertTrue(MockClient.doesBucketExistV2(S3_TEST_BUCKET_NAME)); for (int i = 0 ; i < filePrefix.size() ; i++) { for (int j = 0; j < EXPECTED_PARTITION_NUM; j++) { final Boolean objectExist = MockClient.doesObjectExist(S3_TEST_BUCKET_NAME, filePrefix.get(i) + "_0000" + j); if (i == 1) { assertFalse(objectExist); } else { assertTrue(objectExist); } } } }
@Ignore @Test public void testWriteToMockS3WithMultiVersion() throws IOException { final Configuration conf1 = initConfigWithAws(this.pathPrefix, OBJ_KEY_1, LOCAL1, VERSION, TIMESTAMP1, SOURCE_SUB_PATH1, S3_TEST_BUCKET_NAME); final Configuration conf2 = initConfigWithAws(this.pathPrefix, OBJ_KEY_1, LOCAL2, VERSION, TIMESTAMP2, SOURCE_SUB_PATH1, S3_TEST_BUCKET_NAME); final Configuration conf3 = initConfigWithAws(this.pathPrefix, OBJ_KEY_1, LOCAL3, VERSION, TIMESTAMP3, SOURCE_SUB_PATH2, S3_TEST_BUCKET_NAME); final List<String> filePrefix = new ArrayList<>(); filePrefix.add(testWriteToMockS3General(conf1)); filePrefix.add(testWriteToMockS3General(conf2)); filePrefix.add(testWriteToMockS3General(conf3)); final AmazonS3 MockClient = getMockS3Connection(); assertTrue(MockClient.doesBucketExistV2(S3_TEST_BUCKET_NAME)); for (String aFilePrefix : filePrefix) { for (int j = 0; j < EXPECTED_PARTITION_NUM; j++) { final Boolean objectExist = MockClient.doesObjectExist(S3_TEST_BUCKET_NAME, aFilePrefix + "_0000" + j); assertTrue(objectExist); } } } @Ignore