@Test public void testBasicQuoted3() { final String input = "a:'b''c'"; final StrTokenizer tok = new StrTokenizer(input, ':', '\''); tok.setIgnoreEmptyTokens(false); tok.setEmptyTokenAsNull(true); assertEquals("a", tok.next()); assertEquals("b'c", tok.next()); assertFalse(tok.hasNext()); }
@Test public void testBasicQuoted2() { final String input = "a:'b':"; final StrTokenizer tok = new StrTokenizer(input, ':', '\''); tok.setIgnoreEmptyTokens(false); tok.setEmptyTokenAsNull(true); assertEquals("a", tok.next()); assertEquals("b", tok.next()); assertNull(tok.next()); assertFalse(tok.hasNext()); }
@Test public void test3() { final String input = "a;b; c;\"d;\"\"e\";f; ; ;"; final StrTokenizer tok = new StrTokenizer(input); tok.setDelimiterChar(';'); tok.setQuoteChar('"'); tok.setIgnoredMatcher(StrMatcher.noneMatcher()); tok.setIgnoreEmptyTokens(false); final String tokens[] = tok.getTokenArray(); final String expected[] = new String[]{"a", "b", " c", "d;\"e", "f", " ", " ", "",}; assertEquals(ArrayUtils.toString(tokens), expected.length, tokens.length); for (int i = 0; i < expected.length; i++) { assertEquals("token[" + i + "] was '" + tokens[i] + "' but was expected to be '" + expected[i] + "'", expected[i], tokens[i]); } }
@Test public void test4() { final String input = "a;b; c;\"d;\"\"e\";f; ; ;"; final StrTokenizer tok = new StrTokenizer(input); tok.setDelimiterChar(';'); tok.setQuoteChar('"'); tok.setIgnoredMatcher(StrMatcher.trimMatcher()); tok.setIgnoreEmptyTokens(true); final String tokens[] = tok.getTokenArray(); final String expected[] = new String[]{"a", "b", "c", "d;\"e", "f",}; assertEquals(ArrayUtils.toString(tokens), expected.length, tokens.length); for (int i = 0; i < expected.length; i++) { assertEquals("token[" + i + "] was '" + tokens[i] + "' but was expected to be '" + expected[i] + "'", expected[i], tokens[i]); } }
@Test public void test1() { final String input = "a;b;c;\"d;\"\"e\";f; ; ; "; final StrTokenizer tok = new StrTokenizer(input); tok.setDelimiterChar(';'); tok.setQuoteChar('"'); tok.setIgnoredMatcher(StrMatcher.trimMatcher()); tok.setIgnoreEmptyTokens(false); final String tokens[] = tok.getTokenArray(); final String expected[] = new String[]{"a", "b", "c", "d;\"e", "f", "", "", "",}; assertEquals(ArrayUtils.toString(tokens), expected.length, tokens.length); for (int i = 0; i < expected.length; i++) { assertEquals("token[" + i + "] was '" + tokens[i] + "' but was expected to be '" + expected[i] + "'", expected[i], tokens[i]); } }
@Test public void test2() { final String input = "a;b;c ;\"d;\"\"e\";f; ; ;"; final StrTokenizer tok = new StrTokenizer(input); tok.setDelimiterChar(';'); tok.setQuoteChar('"'); tok.setIgnoredMatcher(StrMatcher.noneMatcher()); tok.setIgnoreEmptyTokens(false); final String tokens[] = tok.getTokenArray(); final String expected[] = new String[]{"a", "b", "c ", "d;\"e", "f", " ", " ", "",}; assertEquals(ArrayUtils.toString(tokens), expected.length, tokens.length); for (int i = 0; i < expected.length; i++) { assertEquals("token[" + i + "] was '" + tokens[i] + "' but was expected to be '" + expected[i] + "'", expected[i], tokens[i]); } }
@Test public void testBasicEmpty1() { final String input = "a b c"; final StrTokenizer tok = new StrTokenizer(input); tok.setIgnoreEmptyTokens(false); assertEquals("a", tok.next()); assertEquals("", tok.next()); assertEquals("b", tok.next()); assertEquals("c", tok.next()); assertFalse(tok.hasNext()); }
@Test public void testBasicEmpty2() { final String input = "a b c"; final StrTokenizer tok = new StrTokenizer(input); tok.setIgnoreEmptyTokens(false); tok.setEmptyTokenAsNull(true); assertEquals("a", tok.next()); assertNull(tok.next()); assertEquals("b", tok.next()); assertEquals("c", tok.next()); assertFalse(tok.hasNext()); }
@Test public void testBasicQuoted5() { final String input = "a: 'b'x'c' :d"; final StrTokenizer tok = new StrTokenizer(input, ':', '\''); tok.setTrimmerMatcher(StrMatcher.trimMatcher()); tok.setIgnoreEmptyTokens(false); tok.setEmptyTokenAsNull(true); assertEquals("a", tok.next()); assertEquals("bxc", tok.next()); assertEquals("d", tok.next()); assertFalse(tok.hasNext()); }
@Test public void testBasicQuotedTrimmed1() { final String input = "a: 'b' :"; final StrTokenizer tok = new StrTokenizer(input, ':', '\''); tok.setTrimmerMatcher(StrMatcher.trimMatcher()); tok.setIgnoreEmptyTokens(false); tok.setEmptyTokenAsNull(true); assertEquals("a", tok.next()); assertEquals("b", tok.next()); assertNull(tok.next()); assertFalse(tok.hasNext()); }
@Test public void testBasicQuoted4() { final String input = "a: 'b' 'c' :d"; final StrTokenizer tok = new StrTokenizer(input, ':', '\''); tok.setTrimmerMatcher(StrMatcher.trimMatcher()); tok.setIgnoreEmptyTokens(false); tok.setEmptyTokenAsNull(true); assertEquals("a", tok.next()); assertEquals("b c", tok.next()); assertEquals("d", tok.next()); assertFalse(tok.hasNext()); }
@Test public void testBasicTrimmed1() { final String input = "a: b : "; final StrTokenizer tok = new StrTokenizer(input, ':'); tok.setTrimmerMatcher(StrMatcher.trimMatcher()); tok.setIgnoreEmptyTokens(false); tok.setEmptyTokenAsNull(true); assertEquals("a", tok.next()); assertEquals("b", tok.next()); assertNull(tok.next()); assertFalse(tok.hasNext()); }
@Test public void testBasicTrimmed2() { final String input = "a: b :"; final StrTokenizer tok = new StrTokenizer(input, ':'); tok.setTrimmerMatcher(StrMatcher.stringMatcher(" ")); tok.setIgnoreEmptyTokens(false); tok.setEmptyTokenAsNull(true); assertEquals("a", tok.next()); assertEquals("b", tok.next()); assertNull(tok.next()); assertFalse(tok.hasNext()); }
@Test public void testBasicIgnoreTrimmed3() { final String input = "IGNOREaIGNORE: IGNORE bIGNOREc IGNORE : IGNORE "; final StrTokenizer tok = new StrTokenizer(input, ':'); tok.setIgnoredMatcher(StrMatcher.stringMatcher("IGNORE")); tok.setIgnoreEmptyTokens(false); tok.setEmptyTokenAsNull(true); assertEquals("a", tok.next()); assertEquals(" bc ", tok.next()); assertEquals(" ", tok.next()); assertFalse(tok.hasNext()); }
@Test public void test5() { final String input = "a;b; c;\"d;\"\"e\";f; ; ;"; final StrTokenizer tok = new StrTokenizer(input); tok.setDelimiterChar(';'); tok.setQuoteChar('"'); tok.setIgnoredMatcher(StrMatcher.trimMatcher()); tok.setIgnoreEmptyTokens(false); tok.setEmptyTokenAsNull(true); final String tokens[] = tok.getTokenArray(); final String expected[] = new String[]{"a", "b", "c", "d;\"e", "f", null, null, null,}; assertEquals(ArrayUtils.toString(tokens), expected.length, tokens.length); for (int i = 0; i < expected.length; i++) { assertEquals("token[" + i + "] was '" + tokens[i] + "' but was expected to be '" + expected[i] + "'", expected[i], tokens[i]); } }
@Test public void test8() { final String input = "a b c \"d e\" f "; final StrTokenizer tok = new StrTokenizer(input); tok.setDelimiterMatcher(StrMatcher.spaceMatcher()); tok.setQuoteMatcher(StrMatcher.doubleQuoteMatcher()); tok.setIgnoredMatcher(StrMatcher.noneMatcher()); tok.setIgnoreEmptyTokens(true); final String tokens[] = tok.getTokenArray(); final String expected[] = new String[]{"a", "b", "c", "d e", "f",}; assertEquals(ArrayUtils.toString(tokens), expected.length, tokens.length); for (int i = 0; i < expected.length; i++) { assertEquals("token[" + i + "] was '" + tokens[i] + "' but was expected to be '" + expected[i] + "'", expected[i], tokens[i]); } }
@Test public void test7() { final String input = "a b c \"d e\" f "; final StrTokenizer tok = new StrTokenizer(input); tok.setDelimiterMatcher(StrMatcher.spaceMatcher()); tok.setQuoteMatcher(StrMatcher.doubleQuoteMatcher()); tok.setIgnoredMatcher(StrMatcher.noneMatcher()); tok.setIgnoreEmptyTokens(false); final String tokens[] = tok.getTokenArray(); final String expected[] = new String[]{"a", "", "", "b", "c", "d e", "f", "",}; assertEquals(ArrayUtils.toString(tokens), expected.length, tokens.length); for (int i = 0; i < expected.length; i++) { assertEquals("token[" + i + "] was '" + tokens[i] + "' but was expected to be '" + expected[i] + "'", expected[i], tokens[i]); } }
@Test public void testBasicIgnoreTrimmed4() { final String input = "IGNOREaIGNORE: IGNORE 'bIGNOREc'IGNORE'd' IGNORE : IGNORE "; final StrTokenizer tok = new StrTokenizer(input, ':', '\''); tok.setIgnoredMatcher(StrMatcher.stringMatcher("IGNORE")); tok.setTrimmerMatcher(StrMatcher.trimMatcher()); tok.setIgnoreEmptyTokens(false); tok.setEmptyTokenAsNull(true); assertEquals("a", tok.next()); assertEquals("bIGNOREcd", tok.next()); assertNull(tok.next()); assertFalse(tok.hasNext()); }
@Test public void testBasicIgnoreTrimmed1() { final String input = "a: bIGNOREc : "; final StrTokenizer tok = new StrTokenizer(input, ':'); tok.setIgnoredMatcher(StrMatcher.stringMatcher("IGNORE")); tok.setTrimmerMatcher(StrMatcher.trimMatcher()); tok.setIgnoreEmptyTokens(false); tok.setEmptyTokenAsNull(true); assertEquals("a", tok.next()); assertEquals("bc", tok.next()); assertNull(tok.next()); assertFalse(tok.hasNext()); }
@Test public void testBasicIgnoreTrimmed2() { final String input = "IGNOREaIGNORE: IGNORE bIGNOREc IGNORE : IGNORE "; final StrTokenizer tok = new StrTokenizer(input, ':'); tok.setIgnoredMatcher(StrMatcher.stringMatcher("IGNORE")); tok.setTrimmerMatcher(StrMatcher.trimMatcher()); tok.setIgnoreEmptyTokens(false); tok.setEmptyTokenAsNull(true); assertEquals("a", tok.next()); assertEquals("bc", tok.next()); assertNull(tok.next()); assertFalse(tok.hasNext()); }