public AwsFileSink(@NonNull final FileSinkConfiguration conf, @NonNull final FileSinkDataConverter converter) { super(conf, converter); this.awsConf = new AwsConfiguration(conf); this.s3Client = getS3Connection(); }
/** * This method is used to initialize {@link AwsFileSink#s3Client} * with aws configurations in {@link AwsConfiguration} * * @return new AmazonS3 client */ protected AmazonS3 getS3Connection() { final AWSStaticCredentialsProvider awsCredentialProvider = new AWSStaticCredentialsProvider( new BasicAWSCredentials(this.awsConf.getAwsAccessKeyId(), this.awsConf.getAwsSecretAccessKey())); return AmazonS3ClientBuilder.standard().withRegion(this.awsConf.getRegion()) .withCredentials(awsCredentialProvider).build(); }
metadata.setContentLength(contentBytes.length); log.info("Uploading from {} to S3 bucket {}/{}", path.toString() , this.awsConf.getBucketName(), this.awsConf.getObjectKey()); try (final InputStream inputStream = fileSystem.open(path)) { final String objKey = String.format("%s_%0" + this.digitNum + "d", this.awsConf.getS3FilePrefix(), partNum); final PutObjectRequest request = new PutObjectRequest(this.awsConf.getBucketName(), objKey, inputStream, metadata); this.s3Client.putObject(request); } catch (AmazonServiceException e) { log.error("Failed while putObject to bucket %s with message %s" , this.awsConf.getBucketName(), e.getErrorMessage()); throw new JobRuntimeException(e); } catch (IOException e) {
@Test public void testConfigurationWithS3() { final Configuration c = initS3("S3"); final FileSinkConfiguration conf = new FileSinkConfiguration(c); final AwsConfiguration awsConf = new AwsConfiguration(conf); Assert.assertEquals(conf.getSinkType().name(), "S3" ); Assert.assertEquals(conf.getAwsLocal(), "/aws_test"); Assert.assertEquals(awsConf.getBucketName(), "aws-test" ); Assert.assertEquals(awsConf.getObjectKey(), "marmaray_test/test1" ); Assert.assertEquals(awsConf.getAwsAccessKeyId(), "username"); Assert.assertEquals(awsConf.getAwsSecretAccessKey(), "password"); Assert.assertEquals(awsConf.getRegion(), "us-east-1"); Assert.assertEquals(awsConf.getSourcePath(), this.fileSystem.get().getWorkingDirectory()+"/aws_test"); }
final Path destPath = new Path(this.awsConf.getSourcePath()); final FileSystem fs = destPath.getFileSystem(new HadoopConfiguration(this.conf.getConf()).getHadoopConf()); , this.awsConf.getSourcePath()); throw new JobRuntimeException(e); final Path destPath = new Path(this.awsConf.getFileSystemPrefix()); log.info("Start to load file system object for intermediate storage."); try { log.info("Start to overwrite files."); final ListObjectsRequest listObjectsRequest = new ListObjectsRequest().withBucketName(this.awsConf.getBucketName()) .withPrefix(this.awsConf.getPathKey()); final ArrayList<KeyVersion> keysToDelete = new ArrayList<>(); ObjectListing objects = this.s3Client.listObjects(listObjectsRequest); new DeleteObjectsRequest(this.awsConf.getBucketName()) .withKeys(keysToDelete) .withQuiet(false); this.awsConf.getBucketName(), this.awsConf.getPathKey()); final Path temporaryOutput = new Path(this.awsConf.getSourcePath()); final FileStatus[] status = fileSystem.listStatus(temporaryOutput); log.info("Start to upload to S3 bucket."); log.info("Finished deleting temporary output path: {}", this.awsConf.getSourcePath());
private String testWriteToMockS3General(@NonNull final Configuration conf) throws IOException { final JavaRDD<AvroPayload> testData = AvroPayloadUtil.generateTestDataNew(this.jsc.get(), NUM_RECORD, StringTypes.EMPTY); final FileSinkDataConverter converter = new FileSinkDataConverter(conf, new ErrorExtractor()); final FileSinkConfiguration fileConf = new FileSinkConfiguration(conf); final AwsConfiguration awsConf = new AwsConfiguration(fileConf); final MockAwsFileSink awsMockSink = spy(new MockAwsFileSink(fileConf, converter)); awsMockSink.write(testData); final AmazonS3 MockClient = awsMockSink.getS3Client(); verify(awsMockSink, times(EXPECTED_INVOCATIONS)).write(Matchers.any(JavaRDD.class)); verify(MockClient, times(EXPECTED_PARTITION_NUM)).putObject(Matchers.any(PutObjectRequest.class)); assertTrue(MockClient.doesBucketExistV2(fileConf.getBucketName().get())); for (int i = 0 ; i < EXPECTED_PARTITION_NUM ; i++) { final Boolean objectExist = MockClient.doesObjectExist(fileConf.getBucketName().get(), awsConf.getS3FilePrefix()+ "_0000" + i); assertTrue(objectExist); } return awsConf.getS3FilePrefix(); }
public AwsConfiguration(@NonNull final FileSinkConfiguration conf) { ConfigUtil.checkMandatoryProperties(conf.getConf(), this.getMandatoryProperties()); this.region = conf.getAwsRegion().get(); this.awsAccessKeyId = conf.getAwsAccessKeyId().get(); this.awsSecretAccessKey = conf.getAwsSecretAccesskey().get(); this.bucketName = conf.getBucketName().get(); this.objectKey = conf.getObjectKey().get(); this.sourcePath = conf.getFullPath(); this.fileSystemPrefix = conf.getPathPrefix(); String s3FilePrefix; if (conf.getSourcePartitionPath().isPresent()) { s3FilePrefix = String.format("%s/%s/", this.objectKey, conf.getSourcePartitionPath().get()); } else { s3FilePrefix = String.format("%s/", this.objectKey); } this.pathKey = s3FilePrefix; s3FilePrefix += conf.getFileNamePrefix(); this.s3FilePrefix = s3FilePrefix; }
@Test(expected = MissingPropertyException.class) public void testConfigurationWithS3MissRegion() { final Configuration c = initS3MissConfig("S3", FileSinkConfiguration.AWS_REGION); final FileSinkConfiguration conf = new FileSinkConfiguration(c); final AwsConfiguration awsConf = new AwsConfiguration(conf); }
@Test(expected = MissingPropertyException.class) public void testConfigurationWithS3MissCredentials() { final Configuration c = initS3MissConfig("S3", FileSinkConfiguration.AWS_ACCESS_KEY_ID); final FileSinkConfiguration conf = new FileSinkConfiguration(c); final AwsConfiguration awsConf = new AwsConfiguration(conf); }
@Test(expected = MissingPropertyException.class) public void testConfigurationWithS3MissBucketName() { final Configuration c = initS3MissConfig("S3", FileSinkConfiguration.BUCKET_NAME); final FileSinkConfiguration conf = new FileSinkConfiguration(c); final AwsConfiguration awsConf = new AwsConfiguration(conf); }
@Test(expected = MissingPropertyException.class) public void testConfigurationWithS3MissObjectKey() { final Configuration c = initS3MissConfig("S3", FileSinkConfiguration.OBJECT_KEY); final FileSinkConfiguration conf = new FileSinkConfiguration(c); final AwsConfiguration awsConf = new AwsConfiguration(conf); } }