@Test public void testBasic() throws Exception { split(Arrays.asList("hello", "world"), "hello,world", ',', false, true); split(Arrays.asList(" hello", " world "), " hello| world ", '|', false, true); split(Arrays.asList("", "hello", "world", ""), "|hello|world|", '|', false, true); split(Arrays.asList(""), "", '|', false, true); split(Arrays.asList("", "", "x"), "||x", '|', false, true); split(Lists.newArrayList(), "", '|', false, false); split(Arrays.asList("x"), "x", '|', false, false); split(Arrays.asList(null,"x"), "|x", '|', false, false); Record record = new Record(); CSVTokenizer tokenizer = new SimpleCSVTokenizer(',', true, true, new ArrayList<String>()); tokenizer.tokenizeLine(" x ", null, record); assertEquals(Arrays.asList("x"), record.get("column0")); }
private void split(List expected, String line, char separator, boolean isQuoted, boolean addEmptyStrings) throws IOException { Record record = new Record(); CSVTokenizer tokenizer; if (isQuoted) { tokenizer = new QuotedCSVTokenizer(separator, false, addEmptyStrings, new ArrayList<String>(), 1000, false, '"'); tokenizer.tokenizeLine(line, new BufferedReader(new StringReader("")), record); } else { tokenizer = new SimpleCSVTokenizer(separator, false, addEmptyStrings, new ArrayList<String>()); tokenizer.tokenizeLine(line, null, record); } for (int i = 0; i < expected.size(); i++) { assertEquals(expected.get(i), record.getFirstValue("column" + i)); } assertTrue(record.getFields().asMap().size() <= expected.size()); }
@Test public void testQuoted() throws Exception { char quote = '"'; String doubleQuote = String.valueOf(quote) + String.valueOf(quote); split( Arrays.asList("5", "orange", "This is a\nmulti, line text", "no\""), "5,orange,\"This is a\nmulti, line text\",no\"\"", ',', true, true); split(Arrays.asList("x", ""), "x|", '|', true, true); split(Arrays.asList("x", ""), "x|" + doubleQuote, '|', true, true); split(Lists.newArrayList(), doubleQuote, '|', true, true); split(Lists.newArrayList(), "", '|', true, true); split(Lists.newArrayList(), "", '|', true, false); split(Arrays.asList("x"), "x", '|', true, false); split(Arrays.asList(null,"x"), "|x", '|', true, false); split(Arrays.asList(null,"x"), doubleQuote + "|" + quote + "x" + quote, '|', true, false); }
@Test public void testIgnoreRecordTooLong() throws Exception { boolean ignoreTooLongRecords = true; int maxCharactersPerRecord = 10; CSVTokenizer tokenizer = new QuotedCSVTokenizer( ',', false, false, new ArrayList<String>(), maxCharactersPerRecord, ignoreTooLongRecords, '"'); assertFalse(tokenizer.tokenizeLine( "\"", new BufferedReader(new StringReader("line tooooooooo long\"")), new Record())); }
@Test(expected=IllegalStateException.class) public void testIllegalQuotedState() throws Exception { split(Lists.newArrayList(), "foo,maybe\"", ',', true, true); }