@Test public void testGetHeaderWithCsv() { final String separator = ","; final JavaRDD<AvroPayload> payloadData = AvroPayloadUtil.generateTestData(this.jsc.get(), 10, StringTypes.EMPTY); final Configuration conf = initConf(separator, "csv"); log.info("Starting to get data header."); final FileSinkDataConverter converter = new FileSinkDataConverter(conf, new ErrorExtractor()); final String header = converter.getHeader(payloadData); final String resultHeader = "int_field,string_field,boolean_field"; Assert.assertEquals(resultHeader, header); log.info("Header: {}", header); }
@Test(expected = SparkException.class) public void testConvertAllWithJsonNotSupported() { log.info("Starts Test convert all with json"); final String separator = ","; final JavaRDD<AvroPayload> payloadData = AvroPayloadUtil.generateTestData(this.jsc.get(), 10, StringTypes.EMPTY); final Configuration conf = initConf(separator, "json"); log.info("Starting to convert data."); final FileSinkDataConverter converter = new FileSinkDataConverter(conf, new ErrorExtractor()); final JavaRDD<String> dataConverted = converter.convertAll(payloadData); int i = 1; for (String line: dataConverted.collect()){ Assert.assertEquals(String.valueOf(i) + separator + String.valueOf(i) + separator + "true", line ); i = i + 1; } }
@Test public void testConvertAllWithCsv() { log.info("Starts Test convert all with csv"); final String separator = " "; final JavaRDD<AvroPayload> payloadData = AvroPayloadUtil.generateTestData(this.jsc.get(), 10, StringTypes.EMPTY); final Configuration conf = initConf(separator, "csv"); log.info("Starting to convert data."); final FileSinkDataConverter converter = new FileSinkDataConverter(conf, new ErrorExtractor()); final JavaRDD<String> dataConverted = converter.convertAll(payloadData); int i = 1; for (String line: dataConverted.collect()){ Assert.assertEquals(String.valueOf(i) + separator + String.valueOf(i) + separator + "true", line ); i = i + 1; } }
@Test public void testConvertAllWithCsvSpecialChar() { log.info("Starts Test convert all with csv"); final String separator = ","; final JavaRDD<AvroPayload> payloadData = AvroPayloadUtil.generateTestDataNew(this.jsc.get(), 10, StringTypes.EMPTY); final Configuration conf = initConf(separator, "csv"); log.info("Starting to convert data."); final FileSinkDataConverter converter = new FileSinkDataConverter(conf, new ErrorExtractor()); final JavaRDD<String> dataConverted = converter.convertAll(payloadData); int i = 1; for (String line: dataConverted.collect()){ Assert.assertEquals(String.valueOf(i) + separator + "\"" + String.valueOf(i) + "\\\",try\\\\\"" + separator + "true", line); i = i + 1; } }