@Test public void testListArray() { final String input = "a b c"; final StrTokenizer tok = new StrTokenizer(input); final String[] array = tok.getTokenArray(); final List<?> list = tok.getTokenList(); assertEquals(Arrays.asList(array), list); assertEquals(3, list.size()); }
@Test public void test3() { final String input = "a;b; c;\"d;\"\"e\";f; ; ;"; final StrTokenizer tok = new StrTokenizer(input); tok.setDelimiterChar(';'); tok.setQuoteChar('"'); tok.setIgnoredMatcher(StrMatcher.noneMatcher()); tok.setIgnoreEmptyTokens(false); final String tokens[] = tok.getTokenArray(); final String expected[] = new String[]{"a", "b", " c", "d;\"e", "f", " ", " ", "",}; assertEquals(ArrayUtils.toString(tokens), expected.length, tokens.length); for (int i = 0; i < expected.length; i++) { assertEquals("token[" + i + "] was '" + tokens[i] + "' but was expected to be '" + expected[i] + "'", expected[i], tokens[i]); } }
@Test public void test4() { final String input = "a;b; c;\"d;\"\"e\";f; ; ;"; final StrTokenizer tok = new StrTokenizer(input); tok.setDelimiterChar(';'); tok.setQuoteChar('"'); tok.setIgnoredMatcher(StrMatcher.trimMatcher()); tok.setIgnoreEmptyTokens(true); final String tokens[] = tok.getTokenArray(); final String expected[] = new String[]{"a", "b", "c", "d;\"e", "f",}; assertEquals(ArrayUtils.toString(tokens), expected.length, tokens.length); for (int i = 0; i < expected.length; i++) { assertEquals("token[" + i + "] was '" + tokens[i] + "' but was expected to be '" + expected[i] + "'", expected[i], tokens[i]); } }
@Test public void test1() { final String input = "a;b;c;\"d;\"\"e\";f; ; ; "; final StrTokenizer tok = new StrTokenizer(input); tok.setDelimiterChar(';'); tok.setQuoteChar('"'); tok.setIgnoredMatcher(StrMatcher.trimMatcher()); tok.setIgnoreEmptyTokens(false); final String tokens[] = tok.getTokenArray(); final String expected[] = new String[]{"a", "b", "c", "d;\"e", "f", "", "", "",}; assertEquals(ArrayUtils.toString(tokens), expected.length, tokens.length); for (int i = 0; i < expected.length; i++) { assertEquals("token[" + i + "] was '" + tokens[i] + "' but was expected to be '" + expected[i] + "'", expected[i], tokens[i]); } }
@Test public void test2() { final String input = "a;b;c ;\"d;\"\"e\";f; ; ;"; final StrTokenizer tok = new StrTokenizer(input); tok.setDelimiterChar(';'); tok.setQuoteChar('"'); tok.setIgnoredMatcher(StrMatcher.noneMatcher()); tok.setIgnoreEmptyTokens(false); final String tokens[] = tok.getTokenArray(); final String expected[] = new String[]{"a", "b", "c ", "d;\"e", "f", " ", " ", "",}; assertEquals(ArrayUtils.toString(tokens), expected.length, tokens.length); for (int i = 0; i < expected.length; i++) { assertEquals("token[" + i + "] was '" + tokens[i] + "' but was expected to be '" + expected[i] + "'", expected[i], tokens[i]); } }
@Test public void test5() { final String input = "a;b; c;\"d;\"\"e\";f; ; ;"; final StrTokenizer tok = new StrTokenizer(input); tok.setDelimiterChar(';'); tok.setQuoteChar('"'); tok.setIgnoredMatcher(StrMatcher.trimMatcher()); tok.setIgnoreEmptyTokens(false); tok.setEmptyTokenAsNull(true); final String tokens[] = tok.getTokenArray(); final String expected[] = new String[]{"a", "b", "c", "d;\"e", "f", null, null, null,}; assertEquals(ArrayUtils.toString(tokens), expected.length, tokens.length); for (int i = 0; i < expected.length; i++) { assertEquals("token[" + i + "] was '" + tokens[i] + "' but was expected to be '" + expected[i] + "'", expected[i], tokens[i]); } }
@Test public void test8() { final String input = "a b c \"d e\" f "; final StrTokenizer tok = new StrTokenizer(input); tok.setDelimiterMatcher(StrMatcher.spaceMatcher()); tok.setQuoteMatcher(StrMatcher.doubleQuoteMatcher()); tok.setIgnoredMatcher(StrMatcher.noneMatcher()); tok.setIgnoreEmptyTokens(true); final String tokens[] = tok.getTokenArray(); final String expected[] = new String[]{"a", "b", "c", "d e", "f",}; assertEquals(ArrayUtils.toString(tokens), expected.length, tokens.length); for (int i = 0; i < expected.length; i++) { assertEquals("token[" + i + "] was '" + tokens[i] + "' but was expected to be '" + expected[i] + "'", expected[i], tokens[i]); } }
@Test public void test7() { final String input = "a b c \"d e\" f "; final StrTokenizer tok = new StrTokenizer(input); tok.setDelimiterMatcher(StrMatcher.spaceMatcher()); tok.setQuoteMatcher(StrMatcher.doubleQuoteMatcher()); tok.setIgnoredMatcher(StrMatcher.noneMatcher()); tok.setIgnoreEmptyTokens(false); final String tokens[] = tok.getTokenArray(); final String expected[] = new String[]{"a", "", "", "b", "c", "d e", "f", "",}; assertEquals(ArrayUtils.toString(tokens), expected.length, tokens.length); for (int i = 0; i < expected.length; i++) { assertEquals("token[" + i + "] was '" + tokens[i] + "' but was expected to be '" + expected[i] + "'", expected[i], tokens[i]); } }
final StrTokenizer t = b.asTokenizer(); final String[] tokens1 = t.getTokenArray(); assertEquals(2, tokens1.length); assertEquals("a", tokens1[0]); final String[] tokens2 = t.getTokenArray(); assertEquals(2, tokens2.length); assertEquals("a", tokens2[0]); final String[] tokens3 = t.getTokenArray(); assertEquals(4, tokens3.length); assertEquals("a", tokens3[0]);
@Test public void test6() { final String input = "a;b; c;\"d;\"\"e\";f; ; ;"; final StrTokenizer tok = new StrTokenizer(input); tok.setDelimiterChar(';'); tok.setQuoteChar('"'); tok.setIgnoredMatcher(StrMatcher.trimMatcher()); tok.setIgnoreEmptyTokens(false); // tok.setTreatingEmptyAsNull(true); final String tokens[] = tok.getTokenArray(); final String expected[] = new String[]{"a", "b", " c", "d;\"e", "f", null, null, null,}; int nextCount = 0; while (tok.hasNext()) { tok.next(); nextCount++; } int prevCount = 0; while (tok.hasPrevious()) { tok.previous(); prevCount++; } assertEquals(ArrayUtils.toString(tokens), expected.length, tokens.length); assertTrue("could not cycle through entire token list" + " using the 'hasNext' and 'next' methods", nextCount == expected.length); assertTrue("could not cycle through entire token list" + " using the 'hasPrevious' and 'previous' methods", prevCount == expected.length); }
new StrTokenizer(pattern, pathSeparator).setIgnoreEmptyTokens(true).getTokenArray(); final String[] pathParts = new StrTokenizer(path, pathSeparator).setIgnoreEmptyTokens(true).getTokenArray();
new StrTokenizer(pattern, pathSeparator).setIgnoreEmptyTokens(true).getTokenArray(); final String[] pathDirs = new StrTokenizer(path, pathSeparator).setIgnoreEmptyTokens(true).getTokenArray();
String params = "\"John Smith\" Ted Barry" // Initialize tokenizer with input string, delimiter character, quote character StrTokenizer tokenizer = new StrTokenizer(params, ' ', '"'); for (String token : tokenizer.getTokenArray()) { System.out.println(token); }
String input = "A||B||||D||E"; StrTokenizer tokenizer = new StrTokenizer(input, "||"); tokenizer.setIgnoreEmptyTokens(false); for (int i = 0; i < tokenizer .getTokenArray().length; i++) { System.out.println("result[" + i + "]:" + tokenizer.getTokenArray()[i]); }
StrTokenizer tokenizer = StrTokenizer.getCSVInstance(); while (...) { tokenizer.reset(dataLine); String tokens[] = tokenizer.getTokenArray(); ... }
StrTokenizer tokenizer = StrTokenizer.getCSVInstance(); for (int i=0;i<50000;i++) { tokenizer.reset("a,b,c,d,e,f,g,h,i,j,k,l,m,n,o,p,q,r,s,t,u,v,w,x,y,z"); String toks[] = tokenizer.getTokenArray(); }
StrTokenizer tokenizer = StrTokenizer.getTSVInstance(); while ( (line = ...) != null) { tokenizer.reset(line); String tokArray[] = tokenizer.getTokenArray(); }
// get a csv instance (which is cloned, so we can customize it) StrTokenizer tokenizer = StrTokenizer.getCSVInstance(); // Set delimiter char tokenizer.setDelimiterChar('|'); Scanner scanner = new Scanner(new File("file.psv")); while (scanner.hasNextLine()) { // set the input on the tokenizer tokenizer.reset(scanner.nextLine()); // get the tokens String toks[] = tokenizer.getTokenArray(); }
private List<Column> initColumns(final StrTokenizer st, final String headerLine) { st.reset(headerLine); String[] headers = st.getTokenArray(); List<Column> columns = newArrayListWithCapacity(headers.length); for (String header : headers) { columns.add(new Column(header)); } return columns; }
private String[] splitList(String commaDelimitedList) { return new StrTokenizer(commaDelimitedList, StrMatcher.commaMatcher(), StrMatcher.quoteMatcher()) .setTrimmerMatcher(StrMatcher.trimMatcher()) .getTokenArray(); }