public AwsFileSink(@NonNull final FileSinkConfiguration conf, @NonNull final FileSinkDataConverter converter) { super(conf, converter); this.awsConf = new AwsConfiguration(conf); this.s3Client = getS3Connection(); }
@Test(expected = MissingPropertyException.class) public void testConfigurationWithS3MissRegion() { final Configuration c = initS3MissConfig("S3", FileSinkConfiguration.AWS_REGION); final FileSinkConfiguration conf = new FileSinkConfiguration(c); final AwsConfiguration awsConf = new AwsConfiguration(conf); }
@Test(expected = MissingPropertyException.class) public void testConfigurationWithS3MissCredentials() { final Configuration c = initS3MissConfig("S3", FileSinkConfiguration.AWS_ACCESS_KEY_ID); final FileSinkConfiguration conf = new FileSinkConfiguration(c); final AwsConfiguration awsConf = new AwsConfiguration(conf); }
@Test(expected = MissingPropertyException.class) public void testConfigurationWithS3MissBucketName() { final Configuration c = initS3MissConfig("S3", FileSinkConfiguration.BUCKET_NAME); final FileSinkConfiguration conf = new FileSinkConfiguration(c); final AwsConfiguration awsConf = new AwsConfiguration(conf); }
@Test(expected = MissingPropertyException.class) public void testConfigurationWithS3MissObjectKey() { final Configuration c = initS3MissConfig("S3", FileSinkConfiguration.OBJECT_KEY); final FileSinkConfiguration conf = new FileSinkConfiguration(c); final AwsConfiguration awsConf = new AwsConfiguration(conf); } }
private String testWriteToMockS3General(@NonNull final Configuration conf) throws IOException { final JavaRDD<AvroPayload> testData = AvroPayloadUtil.generateTestDataNew(this.jsc.get(), NUM_RECORD, StringTypes.EMPTY); final FileSinkDataConverter converter = new FileSinkDataConverter(conf, new ErrorExtractor()); final FileSinkConfiguration fileConf = new FileSinkConfiguration(conf); final AwsConfiguration awsConf = new AwsConfiguration(fileConf); final MockAwsFileSink awsMockSink = spy(new MockAwsFileSink(fileConf, converter)); awsMockSink.write(testData); final AmazonS3 MockClient = awsMockSink.getS3Client(); verify(awsMockSink, times(EXPECTED_INVOCATIONS)).write(Matchers.any(JavaRDD.class)); verify(MockClient, times(EXPECTED_PARTITION_NUM)).putObject(Matchers.any(PutObjectRequest.class)); assertTrue(MockClient.doesBucketExistV2(fileConf.getBucketName().get())); for (int i = 0 ; i < EXPECTED_PARTITION_NUM ; i++) { final Boolean objectExist = MockClient.doesObjectExist(fileConf.getBucketName().get(), awsConf.getS3FilePrefix()+ "_0000" + i); assertTrue(objectExist); } return awsConf.getS3FilePrefix(); }
@Test public void testConfigurationWithS3() { final Configuration c = initS3("S3"); final FileSinkConfiguration conf = new FileSinkConfiguration(c); final AwsConfiguration awsConf = new AwsConfiguration(conf); Assert.assertEquals(conf.getSinkType().name(), "S3" ); Assert.assertEquals(conf.getAwsLocal(), "/aws_test"); Assert.assertEquals(awsConf.getBucketName(), "aws-test" ); Assert.assertEquals(awsConf.getObjectKey(), "marmaray_test/test1" ); Assert.assertEquals(awsConf.getAwsAccessKeyId(), "username"); Assert.assertEquals(awsConf.getAwsSecretAccessKey(), "password"); Assert.assertEquals(awsConf.getRegion(), "us-east-1"); Assert.assertEquals(awsConf.getSourcePath(), this.fileSystem.get().getWorkingDirectory()+"/aws_test"); }