/** * Constructs a tokenizer splitting on the specified delimiter character * and handling quotes using the specified quote character. * * @param input the string which is to be parsed, not cloned * @param delim the field delimiter character * @param quote the field quoted string character */ public StrTokenizer(final char[] input, final char delim, final char quote) { this(input, delim); setQuoteChar(quote); }
/** * Constructs a tokenizer splitting on the specified delimiter character * and handling quotes using the specified quote character. * * @param input the string which is to be parsed * @param delim the field delimiter character * @param quote the field quoted string character */ public StrTokenizer(final String input, final char delim, final char quote) { this(input, delim); setQuoteChar(quote); }
@Test public void test4() { final String input = "a;b; c;\"d;\"\"e\";f; ; ;"; final StrTokenizer tok = new StrTokenizer(input); tok.setDelimiterChar(';'); tok.setQuoteChar('"'); tok.setIgnoredMatcher(StrMatcher.trimMatcher()); tok.setIgnoreEmptyTokens(true); final String tokens[] = tok.getTokenArray(); final String expected[] = new String[]{"a", "b", "c", "d;\"e", "f",}; assertEquals(ArrayUtils.toString(tokens), expected.length, tokens.length); for (int i = 0; i < expected.length; i++) { assertEquals("token[" + i + "] was '" + tokens[i] + "' but was expected to be '" + expected[i] + "'", expected[i], tokens[i]); } }
@Test public void test3() { final String input = "a;b; c;\"d;\"\"e\";f; ; ;"; final StrTokenizer tok = new StrTokenizer(input); tok.setDelimiterChar(';'); tok.setQuoteChar('"'); tok.setIgnoredMatcher(StrMatcher.noneMatcher()); tok.setIgnoreEmptyTokens(false); final String tokens[] = tok.getTokenArray(); final String expected[] = new String[]{"a", "b", " c", "d;\"e", "f", " ", " ", "",}; assertEquals(ArrayUtils.toString(tokens), expected.length, tokens.length); for (int i = 0; i < expected.length; i++) { assertEquals("token[" + i + "] was '" + tokens[i] + "' but was expected to be '" + expected[i] + "'", expected[i], tokens[i]); } }
@Test public void test1() { final String input = "a;b;c;\"d;\"\"e\";f; ; ; "; final StrTokenizer tok = new StrTokenizer(input); tok.setDelimiterChar(';'); tok.setQuoteChar('"'); tok.setIgnoredMatcher(StrMatcher.trimMatcher()); tok.setIgnoreEmptyTokens(false); final String tokens[] = tok.getTokenArray(); final String expected[] = new String[]{"a", "b", "c", "d;\"e", "f", "", "", "",}; assertEquals(ArrayUtils.toString(tokens), expected.length, tokens.length); for (int i = 0; i < expected.length; i++) { assertEquals("token[" + i + "] was '" + tokens[i] + "' but was expected to be '" + expected[i] + "'", expected[i], tokens[i]); } }
@Test public void test2() { final String input = "a;b;c ;\"d;\"\"e\";f; ; ;"; final StrTokenizer tok = new StrTokenizer(input); tok.setDelimiterChar(';'); tok.setQuoteChar('"'); tok.setIgnoredMatcher(StrMatcher.noneMatcher()); tok.setIgnoreEmptyTokens(false); final String tokens[] = tok.getTokenArray(); final String expected[] = new String[]{"a", "b", "c ", "d;\"e", "f", " ", " ", "",}; assertEquals(ArrayUtils.toString(tokens), expected.length, tokens.length); for (int i = 0; i < expected.length; i++) { assertEquals("token[" + i + "] was '" + tokens[i] + "' but was expected to be '" + expected[i] + "'", expected[i], tokens[i]); } }
@Test public void test5() { final String input = "a;b; c;\"d;\"\"e\";f; ; ;"; final StrTokenizer tok = new StrTokenizer(input); tok.setDelimiterChar(';'); tok.setQuoteChar('"'); tok.setIgnoredMatcher(StrMatcher.trimMatcher()); tok.setIgnoreEmptyTokens(false); tok.setEmptyTokenAsNull(true); final String tokens[] = tok.getTokenArray(); final String expected[] = new String[]{"a", "b", "c", "d;\"e", "f", null, null, null,}; assertEquals(ArrayUtils.toString(tokens), expected.length, tokens.length); for (int i = 0; i < expected.length; i++) { assertEquals("token[" + i + "] was '" + tokens[i] + "' but was expected to be '" + expected[i] + "'", expected[i], tokens[i]); } }
@Test public void test6() { final String input = "a;b; c;\"d;\"\"e\";f; ; ;"; final StrTokenizer tok = new StrTokenizer(input); tok.setDelimiterChar(';'); tok.setQuoteChar('"'); tok.setIgnoredMatcher(StrMatcher.trimMatcher()); tok.setIgnoreEmptyTokens(false); // tok.setTreatingEmptyAsNull(true); final String tokens[] = tok.getTokenArray(); final String expected[] = new String[]{"a", "b", " c", "d;\"e", "f", null, null, null,}; int nextCount = 0; while (tok.hasNext()) { tok.next(); nextCount++; } int prevCount = 0; while (tok.hasPrevious()) { tok.previous(); prevCount++; } assertEquals(ArrayUtils.toString(tokens), expected.length, tokens.length); assertTrue("could not cycle through entire token list" + " using the 'hasNext' and 'next' methods", nextCount == expected.length); assertTrue("could not cycle through entire token list" + " using the 'hasPrevious' and 'previous' methods", prevCount == expected.length); }
@Test public void testChaining() { final StrTokenizer tok = new StrTokenizer(); assertEquals(tok, tok.reset()); assertEquals(tok, tok.reset("")); assertEquals(tok, tok.reset(new char[0])); assertEquals(tok, tok.setDelimiterChar(' ')); assertEquals(tok, tok.setDelimiterString(" ")); assertEquals(tok, tok.setDelimiterMatcher(null)); assertEquals(tok, tok.setQuoteChar(' ')); assertEquals(tok, tok.setQuoteMatcher(null)); assertEquals(tok, tok.setIgnoredChar(' ')); assertEquals(tok, tok.setIgnoredMatcher(null)); assertEquals(tok, tok.setTrimmerMatcher(null)); assertEquals(tok, tok.setEmptyTokenAsNull(false)); assertEquals(tok, tok.setIgnoreEmptyTokens(false)); }
/** * Constructs a tokenizer splitting on the specified delimiter character * and handling quotes using the specified quote character. * * @param input the string which is to be parsed * @param delim the field delimiter character * @param quote the field quoted string character */ public StrTokenizer(final String input, final char delim, final char quote) { this(input, delim); setQuoteChar(quote); }
/** * Constructs a tokenizer splitting on the specified delimiter character * and handling quotes using the specified quote character. * * @param input the string which is to be parsed * @param delim the field delimiter character * @param quote the field quoted string character */ public StrTokenizer(final String input, final char delim, final char quote) { this(input, delim); setQuoteChar(quote); }
/** * Constructs a tokenizer splitting on the specified delimiter character * and handling quotes using the specified quote character. * * @param input the string which is to be parsed, not cloned * @param delim the field delimiter character * @param quote the field quoted string character */ public StrTokenizer(final char[] input, final char delim, final char quote) { this(input, delim); setQuoteChar(quote); }
/** * Constructs a tokenizer splitting on the specified delimiter character * and handling quotes using the specified quote character. * * @param input the string which is to be parsed, not cloned * @param delim the field delimiter character * @param quote the field quoted string character */ public StrTokenizer(final char[] input, final char delim, final char quote) { this(input, delim); setQuoteChar(quote); }
/** * Constructs a tokenizer splitting on the specified delimiter character * and handling quotes using the specified quote character. * * @param input the string which is to be parsed * @param delim the field delimiter character * @param quote the field quoted string character */ public StrTokenizer(final String input, final char delim, final char quote) { this(input, delim); setQuoteChar(quote); }
/** * Constructs a tokenizer splitting on the specified delimiter character * and handling quotes using the specified quote character. * * @param input the string which is to be parsed, not cloned * @param delim the field delimiter character * @param quote the field quoted string character */ public StrTokenizer(final char[] input, final char delim, final char quote) { this(input, delim); setQuoteChar(quote); }
@Override public LineEntityParser makeParser(List<String> header) { assert header.size() == getHeaderLines(); if (usesHeader() && labeledColumns != null) { assert header.size() == 1; List<TypedName<?>> cols = new ArrayList<>(); StrTokenizer tok = new StrTokenizer(header.get(0), delimiter); tok.setQuoteChar('"'); while (tok.hasNext()) { String label = tok.next(); cols.add(labeledColumns.get(label)); } return new OrderedParser(cols, tok); } else { Preconditions.checkState(columns != null, "no columns specified"); StrTokenizer tok = new StrTokenizer("", delimiter); tok.setQuoteChar('"'); return new OrderedParser(columns, tok); } }
/** Split string x in tokens. Effectively just a friendly wrapper around StrTokenizer. * Use *single* quotes for avoiding splitting. */ public static ArrayList<String> tokenize(String x, String delimiterString){ if(x == null){ return null; } // This is a hack to allow empty tokens to be passed at the command line. // An empty x= x.replace("''", "' '"); // See also http://stackoverflow.com/questions/38161437/inconsistent-behaviour-of-strtokenizer-to-split-string StrTokenizer str= new StrTokenizer(x); str.setTrimmerMatcher(StrMatcher.spaceMatcher()); str.setDelimiterString(delimiterString); str.setQuoteChar('\''); // str.setIgnoreEmptyTokens(false); ArrayList<String> tokens= (ArrayList<String>) str.getTokenList(); for(int i= 0; i < tokens.size(); i++){ String tok= tokens.get(i).trim(); tokens.set(i, tok); } return tokens; }
st.setDelimiterString(delimiter); if (quoteChar != '\0') { st.setQuoteChar(quoteChar); } else { st.setQuoteMatcher(StrMatcher.noneMatcher());