public FileSinkDataConverter(@NonNull final Configuration conf, @NonNull final ErrorExtractor errorExtractor) { super(conf, errorExtractor); final FileSinkConfiguration fsConf = new FileSinkConfiguration(conf); this.fileType = fsConf.getFileType(); this.separator = fsConf.getSeparator(); }
@Test(expected = MissingPropertyException.class) public void testWithConfMissPartitionType() { final Configuration c = initFileNameAndPathMissConfig(FileSinkConfiguration.PATH_PREFIX); final FileSinkConfiguration conf = new FileSinkConfiguration(c); }
@Test(expected = UnsupportedOperationException.class) public void testConfWithS3DispersalTypeNotSupported() { final Configuration c = initCommon("/newpath/newtest", "csv", 1500, " ", "hdfs", "unknownType"); final FileSinkConfiguration conf = new FileSinkConfiguration(c); } }
@Test(expected = UnsupportedOperationException.class) public void testWithConfSeparatorNotSupported() { final Configuration c = initCommon("/newpath/newtest", "json", 1500, " ,", "HDFS", "version"); final FileSinkConfiguration conf = new FileSinkConfiguration(c); }
@Test(expected = MissingPropertyException.class) public void testWithConfMissSourceDataPath() { final Configuration c = initFileNameAndPathMissConfig(FileSinkConfiguration.SOURCE_NAME_PREFIX); final FileSinkConfiguration conf = new FileSinkConfiguration(c); }
@Test(expected = MissingPropertyException.class) public void testWithConfMissTimeStamp() { final Configuration c = initFileNameAndPathMissConfig(FileSinkConfiguration.TIMESTAMP); final FileSinkConfiguration conf = new FileSinkConfiguration(c); }
@Test(expected = MissingPropertyException.class) public void testWithConfMissSourceType() { final Configuration c = initFileNameAndPathMissConfig(FileSinkConfiguration.SOURCE_TYPE); final FileSinkConfiguration conf = new FileSinkConfiguration(c); }
@Test(expected = MissingPropertyException.class) public void testWithPartitionTypeButMissSourceSubPath() { final Configuration c = initFileNameAndPath("date", false, "version"); final FileSinkConfiguration conf = new FileSinkConfiguration(c); }
@Test(expected = UnsupportedOperationException.class) public void testConfWithS3SinkTypeNotSupported() { final Configuration c = initCommon("/newpath/newtest", "csv", 1500, " ", "LOCAL", "version"); final FileSinkConfiguration conf = new FileSinkConfiguration(c); }
@Test(expected = MissingPropertyException.class) public void testConfigurationWithS3MissRegion() { final Configuration c = initS3MissConfig("S3", FileSinkConfiguration.AWS_REGION); final FileSinkConfiguration conf = new FileSinkConfiguration(c); final AwsConfiguration awsConf = new AwsConfiguration(conf); }
@Test(expected = MissingPropertyException.class) public void testConfigurationWithS3MissCredentials() { final Configuration c = initS3MissConfig("S3", FileSinkConfiguration.AWS_ACCESS_KEY_ID); final FileSinkConfiguration conf = new FileSinkConfiguration(c); final AwsConfiguration awsConf = new AwsConfiguration(conf); }
@Test(expected = UnsupportedOperationException.class) public void testWithPartitionTypeNoneButPartitionKeyStr() { final Configuration c = initFileNameAndPath("none", false, "version"); c.setProperty(HiveConfiguration.PARTITION_KEY_NAME, "datestr"); final FileSinkConfiguration conf = new FileSinkConfiguration(c); }
@Test(expected = MissingPropertyException.class) public void testConfigurationWithS3MissBucketName() { final Configuration c = initS3MissConfig("S3", FileSinkConfiguration.BUCKET_NAME); final FileSinkConfiguration conf = new FileSinkConfiguration(c); final AwsConfiguration awsConf = new AwsConfiguration(conf); }
@Test(expected = MissingPropertyException.class) public void testConfigurationWithS3MissObjectKey() { final Configuration c = initS3MissConfig("S3", FileSinkConfiguration.OBJECT_KEY); final FileSinkConfiguration conf = new FileSinkConfiguration(c); final AwsConfiguration awsConf = new AwsConfiguration(conf); } }
private void testWriteGeneral(@NonNull final JavaRDD<AvroPayload> testData, @NonNull final Configuration conf) throws IOException { final FileSinkDataConverter converter = new FileSinkDataConverter(conf, new ErrorExtractor()); final FileSinkConfiguration fileConf = new FileSinkConfiguration(conf); final FileSink awsSink = new AwsFileSink(fileConf, converter); awsSink.write(testData); }
@Before public void setupTest() { super.setupTest(); this.testData1 = AvroPayloadUtil.generateTestData(this.jsc.get(), NUM_RECORD1, StringTypes.EMPTY); this.testData2 = AvroPayloadUtil.generateTestDataNew(this.jsc.get(), NUM_RECORD2, StringTypes.EMPTY); this.conf = initConfig(pathPrefix, PATH1, COMMA_SEPARATOR, TIMESTAMP1, SOURCE_SUB_PATH1, VERSION); this.converter = new FileSinkDataConverter(conf, new ErrorExtractor()); final FileSinkConfiguration fileConf = new FileSinkConfiguration(conf); this.fileSink = spy(new HdfsFileSink(fileConf, converter)); this.convertedData1 = this.converter.convertAll(this.testData1); this.convertedData2 = this.converter.convertAll(this.testData2); }
@Test public void testConfigurationWithS3() { final Configuration c = initS3("S3"); //Aws Property final FileSinkConfiguration conf = new FileSinkConfiguration(c); Assert.assertEquals(conf.getSinkType().name(), "S3" ); Assert.assertEquals(conf.getAwsLocal(), "/aws_test"); Assert.assertEquals(conf.getBucketName().get(), "aws-test" ); Assert.assertEquals(conf.getObjectKey().get(), "marmaray_test/test1" ); Assert.assertEquals(conf.getFullPath(), this.fileSystem.get().getWorkingDirectory()+"/aws_test"); }
@Test public void testDefaultConfigurationInitialized() { final Configuration c = initFileNameAndPath("date", true, "version"); final FileSinkConfiguration conf = new FileSinkConfiguration(c); Assert.assertEquals(conf.getSeparator(), conf.DEFAULT_SEPARATOR); Assert.assertEquals(conf.getFileSizeMegaBytes(), conf.DEFAULT_FILE_SIZE, 0); Assert.assertEquals(conf.getFileType(), conf.DEFAULT_FILE_TYPE); }
@Test public void testDefinedConfigurationInitialized() { final Configuration c = initCommon("/newpath/newtest", "json", 1500, " ", "HDFS", "version"); final FileSinkConfiguration conf = new FileSinkConfiguration(c); Assert.assertEquals(conf.getSeparator(), ' '); Assert.assertEquals(conf.getFileSizeMegaBytes(), 1500, 0); Assert.assertEquals(conf.getFileType(), "json"); Assert.assertEquals(conf.getSourcePartitionPath().get(), "2018/08/01"); Assert.assertEquals(conf.getSourceType(), "hive"); Assert.assertEquals(conf.getSourceNamePrefix(), "test.db_test_trip.table"); Assert.assertEquals(conf.getPartitionType().name(), "DATE"); Assert.assertEquals(conf.getFileNamePrefix(), "marmaray_hive_test.db_test_trip.table_201808011025"); Assert.assertEquals(conf.getFullPath(), this.fileSystem.get().getWorkingDirectory() + "/newpath/newtest/2018/08/01/" + conf.getFileNamePrefix() ); Assert.assertEquals(conf.getPathHdfs(), this.fileSystem.get().getWorkingDirectory() + "/newpath/newtest/2018/08/01"); }
@Test public void testConfigurationWithS3() { final Configuration c = initS3("S3"); final FileSinkConfiguration conf = new FileSinkConfiguration(c); final AwsConfiguration awsConf = new AwsConfiguration(conf); Assert.assertEquals(conf.getSinkType().name(), "S3" ); Assert.assertEquals(conf.getAwsLocal(), "/aws_test"); Assert.assertEquals(awsConf.getBucketName(), "aws-test" ); Assert.assertEquals(awsConf.getObjectKey(), "marmaray_test/test1" ); Assert.assertEquals(awsConf.getAwsAccessKeyId(), "username"); Assert.assertEquals(awsConf.getAwsSecretAccessKey(), "password"); Assert.assertEquals(awsConf.getRegion(), "us-east-1"); Assert.assertEquals(awsConf.getSourcePath(), this.fileSystem.get().getWorkingDirectory()+"/aws_test"); }