@Test public void testBasicQuoted5() { final String input = "a: 'b'x'c' :d"; final StrTokenizer tok = new StrTokenizer(input, ':', '\''); tok.setTrimmerMatcher(StrMatcher.trimMatcher()); tok.setIgnoreEmptyTokens(false); tok.setEmptyTokenAsNull(true); assertEquals("a", tok.next()); assertEquals("bxc", tok.next()); assertEquals("d", tok.next()); assertFalse(tok.hasNext()); }
@Test public void testBasicQuotedTrimmed1() { final String input = "a: 'b' :"; final StrTokenizer tok = new StrTokenizer(input, ':', '\''); tok.setTrimmerMatcher(StrMatcher.trimMatcher()); tok.setIgnoreEmptyTokens(false); tok.setEmptyTokenAsNull(true); assertEquals("a", tok.next()); assertEquals("b", tok.next()); assertNull(tok.next()); assertFalse(tok.hasNext()); }
@Test public void testBasicQuoted4() { final String input = "a: 'b' 'c' :d"; final StrTokenizer tok = new StrTokenizer(input, ':', '\''); tok.setTrimmerMatcher(StrMatcher.trimMatcher()); tok.setIgnoreEmptyTokens(false); tok.setEmptyTokenAsNull(true); assertEquals("a", tok.next()); assertEquals("b c", tok.next()); assertEquals("d", tok.next()); assertFalse(tok.hasNext()); }
@Test public void testBasicTrimmed1() { final String input = "a: b : "; final StrTokenizer tok = new StrTokenizer(input, ':'); tok.setTrimmerMatcher(StrMatcher.trimMatcher()); tok.setIgnoreEmptyTokens(false); tok.setEmptyTokenAsNull(true); assertEquals("a", tok.next()); assertEquals("b", tok.next()); assertNull(tok.next()); assertFalse(tok.hasNext()); }
@Test public void testBasicTrimmed2() { final String input = "a: b :"; final StrTokenizer tok = new StrTokenizer(input, ':'); tok.setTrimmerMatcher(StrMatcher.stringMatcher(" ")); tok.setIgnoreEmptyTokens(false); tok.setEmptyTokenAsNull(true); assertEquals("a", tok.next()); assertEquals("b", tok.next()); assertNull(tok.next()); assertFalse(tok.hasNext()); }
@Test public void testBasicIgnoreTrimmed4() { final String input = "IGNOREaIGNORE: IGNORE 'bIGNOREc'IGNORE'd' IGNORE : IGNORE "; final StrTokenizer tok = new StrTokenizer(input, ':', '\''); tok.setIgnoredMatcher(StrMatcher.stringMatcher("IGNORE")); tok.setTrimmerMatcher(StrMatcher.trimMatcher()); tok.setIgnoreEmptyTokens(false); tok.setEmptyTokenAsNull(true); assertEquals("a", tok.next()); assertEquals("bIGNOREcd", tok.next()); assertNull(tok.next()); assertFalse(tok.hasNext()); }
@Test public void testBasicIgnoreTrimmed1() { final String input = "a: bIGNOREc : "; final StrTokenizer tok = new StrTokenizer(input, ':'); tok.setIgnoredMatcher(StrMatcher.stringMatcher("IGNORE")); tok.setTrimmerMatcher(StrMatcher.trimMatcher()); tok.setIgnoreEmptyTokens(false); tok.setEmptyTokenAsNull(true); assertEquals("a", tok.next()); assertEquals("bc", tok.next()); assertNull(tok.next()); assertFalse(tok.hasNext()); }
@Test public void testBasicIgnoreTrimmed2() { final String input = "IGNOREaIGNORE: IGNORE bIGNOREc IGNORE : IGNORE "; final StrTokenizer tok = new StrTokenizer(input, ':'); tok.setIgnoredMatcher(StrMatcher.stringMatcher("IGNORE")); tok.setTrimmerMatcher(StrMatcher.trimMatcher()); tok.setIgnoreEmptyTokens(false); tok.setEmptyTokenAsNull(true); assertEquals("a", tok.next()); assertEquals("bc", tok.next()); assertNull(tok.next()); assertFalse(tok.hasNext()); }
@Test public void testChaining() { final StrTokenizer tok = new StrTokenizer(); assertEquals(tok, tok.reset()); assertEquals(tok, tok.reset("")); assertEquals(tok, tok.reset(new char[0])); assertEquals(tok, tok.setDelimiterChar(' ')); assertEquals(tok, tok.setDelimiterString(" ")); assertEquals(tok, tok.setDelimiterMatcher(null)); assertEquals(tok, tok.setQuoteChar(' ')); assertEquals(tok, tok.setQuoteMatcher(null)); assertEquals(tok, tok.setIgnoredChar(' ')); assertEquals(tok, tok.setIgnoredMatcher(null)); assertEquals(tok, tok.setTrimmerMatcher(null)); assertEquals(tok, tok.setEmptyTokenAsNull(false)); assertEquals(tok, tok.setIgnoreEmptyTokens(false)); }
/** Split string x in tokens. Effectively just a friendly wrapper around StrTokenizer. * Use *single* quotes for avoiding splitting. */ public static ArrayList<String> tokenize(String x, String delimiterString){ if(x == null){ return null; } // This is a hack to allow empty tokens to be passed at the command line. // An empty x= x.replace("''", "' '"); // See also http://stackoverflow.com/questions/38161437/inconsistent-behaviour-of-strtokenizer-to-split-string StrTokenizer str= new StrTokenizer(x); str.setTrimmerMatcher(StrMatcher.spaceMatcher()); str.setDelimiterString(delimiterString); str.setQuoteChar('\''); // str.setIgnoreEmptyTokens(false); ArrayList<String> tokens= (ArrayList<String>) str.getTokenList(); for(int i= 0; i < tokens.size(); i++){ String tok= tokens.get(i).trim(); tokens.set(i, tok); } return tokens; }
private String[] splitList(String commaDelimitedList) { return new StrTokenizer(commaDelimitedList, StrMatcher.commaMatcher(), StrMatcher.quoteMatcher()) .setTrimmerMatcher(StrMatcher.trimMatcher()) .getTokenArray(); }