final JavaRDD<String> convertedData = this.converter.convertAll(data); final int partNum = getRepartitionNum(convertedData); final int desiredDigit = (int) Math.floor(Math.log10(partNum) + 1);
@Test(expected = SparkException.class) public void testConvertAllWithJsonNotSupported() { log.info("Starts Test convert all with json"); final String separator = ","; final JavaRDD<AvroPayload> payloadData = AvroPayloadUtil.generateTestData(this.jsc.get(), 10, StringTypes.EMPTY); final Configuration conf = initConf(separator, "json"); log.info("Starting to convert data."); final FileSinkDataConverter converter = new FileSinkDataConverter(conf, new ErrorExtractor()); final JavaRDD<String> dataConverted = converter.convertAll(payloadData); int i = 1; for (String line: dataConverted.collect()){ Assert.assertEquals(String.valueOf(i) + separator + String.valueOf(i) + separator + "true", line ); i = i + 1; } }
@Test public void testConvertAllWithCsv() { log.info("Starts Test convert all with csv"); final String separator = " "; final JavaRDD<AvroPayload> payloadData = AvroPayloadUtil.generateTestData(this.jsc.get(), 10, StringTypes.EMPTY); final Configuration conf = initConf(separator, "csv"); log.info("Starting to convert data."); final FileSinkDataConverter converter = new FileSinkDataConverter(conf, new ErrorExtractor()); final JavaRDD<String> dataConverted = converter.convertAll(payloadData); int i = 1; for (String line: dataConverted.collect()){ Assert.assertEquals(String.valueOf(i) + separator + String.valueOf(i) + separator + "true", line ); i = i + 1; } }
@Test public void testConvertAllWithCsvSpecialChar() { log.info("Starts Test convert all with csv"); final String separator = ","; final JavaRDD<AvroPayload> payloadData = AvroPayloadUtil.generateTestDataNew(this.jsc.get(), 10, StringTypes.EMPTY); final Configuration conf = initConf(separator, "csv"); log.info("Starting to convert data."); final FileSinkDataConverter converter = new FileSinkDataConverter(conf, new ErrorExtractor()); final JavaRDD<String> dataConverted = converter.convertAll(payloadData); int i = 1; for (String line: dataConverted.collect()){ Assert.assertEquals(String.valueOf(i) + separator + "\"" + String.valueOf(i) + "\\\",try\\\\\"" + separator + "true", line); i = i + 1; } }
@Before public void setupTest() { super.setupTest(); this.testData1 = AvroPayloadUtil.generateTestData(this.jsc.get(), NUM_RECORD1, StringTypes.EMPTY); this.testData2 = AvroPayloadUtil.generateTestDataNew(this.jsc.get(), NUM_RECORD2, StringTypes.EMPTY); this.conf = initConfig(pathPrefix, PATH1, COMMA_SEPARATOR, TIMESTAMP1, SOURCE_SUB_PATH1, VERSION); this.converter = new FileSinkDataConverter(conf, new ErrorExtractor()); final FileSinkConfiguration fileConf = new FileSinkConfiguration(conf); this.fileSink = spy(new HdfsFileSink(fileConf, converter)); this.convertedData1 = this.converter.convertAll(this.testData1); this.convertedData2 = this.converter.convertAll(this.testData2); }