@Test public void testBasicQuoted3() { final String input = "a:'b''c'"; final StrTokenizer tok = new StrTokenizer(input, ':', '\''); tok.setIgnoreEmptyTokens(false); tok.setEmptyTokenAsNull(true); assertEquals("a", tok.next()); assertEquals("b'c", tok.next()); assertFalse(tok.hasNext()); }
@Test public void test5() { final String input = "a;b; c;\"d;\"\"e\";f; ; ;"; final StrTokenizer tok = new StrTokenizer(input); tok.setDelimiterChar(';'); tok.setQuoteChar('"'); tok.setIgnoredMatcher(StrMatcher.trimMatcher()); tok.setIgnoreEmptyTokens(false); tok.setEmptyTokenAsNull(true); final String tokens[] = tok.getTokenArray(); final String expected[] = new String[]{"a", "b", "c", "d;\"e", "f", null, null, null,}; assertEquals(ArrayUtils.toString(tokens), expected.length, tokens.length); for (int i = 0; i < expected.length; i++) { assertEquals("token[" + i + "] was '" + tokens[i] + "' but was expected to be '" + expected[i] + "'", expected[i], tokens[i]); } }
@Test public void testListArray() { final String input = "a b c"; final StrTokenizer tok = new StrTokenizer(input); final String[] array = tok.getTokenArray(); final List<?> list = tok.getTokenList(); assertEquals(Arrays.asList(array), list); assertEquals(3, list.size()); }
/** * Gets a new tokenizer instance which parses Tab Separated Value strings. * The default for CSV processing will be trim whitespace from both ends * (which can be overridden with the setTrimmer method). * @param input the string to parse * @return a new tokenizer instance which parses Tab Separated Value strings. */ public static StrTokenizer getTSVInstance(final char[] input) { final StrTokenizer tok = getTSVClone(); tok.reset(input); return tok; }
/** * Gets a new tokenizer instance which parses Comma Separated Value strings * initializing it with the given input. The default for CSV processing * will be trim whitespace from both ends (which can be overridden with * the setTrimmer method). * * @param input the text to parse * @return a new tokenizer instance which parses Comma Separated Value strings */ public static StrTokenizer getCSVInstance(final String input) { final StrTokenizer tok = getCSVClone(); tok.reset(input); return tok; }
@Test public void test6() { final String input = "a;b; c;\"d;\"\"e\";f; ; ;"; final StrTokenizer tok = new StrTokenizer(input); tok.setDelimiterChar(';'); tok.setQuoteChar('"'); tok.setIgnoredMatcher(StrMatcher.trimMatcher()); tok.setIgnoreEmptyTokens(false); // tok.setTreatingEmptyAsNull(true); final String tokens[] = tok.getTokenArray(); final String expected[] = new String[]{"a", "b", " c", "d;\"e", "f", null, null, null,}; int nextCount = 0; while (tok.hasNext()) { tok.next(); nextCount++; } int prevCount = 0; while (tok.hasPrevious()) { tok.previous(); prevCount++; } assertEquals(ArrayUtils.toString(tokens), expected.length, tokens.length); assertTrue("could not cycle through entire token list" + " using the 'hasNext' and 'next' methods", nextCount == expected.length); assertTrue("could not cycle through entire token list" + " using the 'hasPrevious' and 'previous' methods", prevCount == expected.length); }
@Test public void testChaining() { final StrTokenizer tok = new StrTokenizer(); assertEquals(tok, tok.reset()); assertEquals(tok, tok.reset("")); assertEquals(tok, tok.reset(new char[0])); assertEquals(tok, tok.setDelimiterChar(' ')); assertEquals(tok, tok.setDelimiterString(" ")); assertEquals(tok, tok.setDelimiterMatcher(null)); assertEquals(tok, tok.setQuoteChar(' ')); assertEquals(tok, tok.setQuoteMatcher(null)); assertEquals(tok, tok.setIgnoredChar(' ')); assertEquals(tok, tok.setIgnoredMatcher(null)); assertEquals(tok, tok.setTrimmerMatcher(null)); assertEquals(tok, tok.setEmptyTokenAsNull(false)); assertEquals(tok, tok.setIgnoreEmptyTokens(false)); }
/** * Parase a fully qualified dotted schema path to list of strings. * a.b.`c.json` -> [a,b,c.json] * a.b.`c-1` -> [a,b,c-1] * a.b.c -> [a,b,c] * @param path dotted schema path * @return list of path components. */ public static List<String> parseFullPath(final String path) { final StrTokenizer tokenizer = new StrTokenizer(path, PATH_DELIMITER, SqlUtils.QUOTE).setIgnoreEmptyTokens(true); return tokenizer.getTokenList(); }
@Test public void testConstructor_String_char() { StrTokenizer tok = new StrTokenizer("a b", ' '); assertEquals(1, tok.getDelimiterMatcher().isMatch(" ".toCharArray(), 0, 0, 1)); assertEquals("a", tok.next()); assertEquals("b", tok.next()); assertFalse(tok.hasNext()); tok = new StrTokenizer("", ' '); assertFalse(tok.hasNext()); tok = new StrTokenizer((String) null, ' '); assertFalse(tok.hasNext()); }
@Test public void testConstructor_String() { StrTokenizer tok = new StrTokenizer("a b"); assertEquals("a", tok.next()); assertEquals("b", tok.next()); assertFalse(tok.hasNext()); tok = new StrTokenizer(""); assertFalse(tok.hasNext()); tok = new StrTokenizer((String) null); assertFalse(tok.hasNext()); }
@Test public void test8() { final String input = "a b c \"d e\" f "; final StrTokenizer tok = new StrTokenizer(input); tok.setDelimiterMatcher(StrMatcher.spaceMatcher()); tok.setQuoteMatcher(StrMatcher.doubleQuoteMatcher()); tok.setIgnoredMatcher(StrMatcher.noneMatcher()); tok.setIgnoreEmptyTokens(true); final String tokens[] = tok.getTokenArray(); final String expected[] = new String[]{"a", "b", "c", "d e", "f",}; assertEquals(ArrayUtils.toString(tokens), expected.length, tokens.length); for (int i = 0; i < expected.length; i++) { assertEquals("token[" + i + "] was '" + tokens[i] + "' but was expected to be '" + expected[i] + "'", expected[i], tokens[i]); } }
cAr[i] = new StrTokenizer(((DoesRenderToWidth)content).render(length)) .setDelimiterChar('\n') .setIgnoreEmptyTokens(false) .getTokenArray()
new StrTokenizer(pattern, pathSeparator).setIgnoreEmptyTokens(true).getTokenArray(); final String[] pathParts = new StrTokenizer(path, pathSeparator).setIgnoreEmptyTokens(true).getTokenArray();
@Test public void testCloneNull() { final StrTokenizer tokenizer = new StrTokenizer((char[]) null); // Start sanity check assertNull(tokenizer.nextToken()); tokenizer.reset(); assertNull(tokenizer.nextToken()); // End sanity check final StrTokenizer clonedTokenizer = (StrTokenizer) tokenizer.clone(); tokenizer.reset(); assertNull(tokenizer.nextToken()); assertNull(clonedTokenizer.nextToken()); }
private ListMultimap<ProcessKey, LoadProfileEvent> readLoadProfileEvents(final Element testplan) throws IOException { ListMultimap<ProcessKey, LoadProfileEvent> eventsByProcess = ArrayListMultimap.create(); String loadProfile = testplan.elementTextTrim("loadProfile"); // relative to testplan File loadProfileConfigFile = new File(new File(testplanFile.getParentFile(), "loadprofiles"), loadProfile); try (BufferedReader br = new BufferedReader(new InputStreamReader(new FileInputStream(loadProfileConfigFile), "UTF-8"))) { StrTokenizer st = StrTokenizer.getCSVInstance(); st.setDelimiterChar(';'); for (String line = null; (line = br.readLine()) != null;) { // ignore line that are blank, commented out, or represent markers if (isBlank(line) || startsWith(line, "#") || MARKER_PATTERN.matcher(line).matches()) { continue; } st.reset(line); String[] tokens = st.getTokenArray(); long startTime = Long.parseLong(tokens[0]); String operation = tokens[1]; String target = tokens[2]; int daemonId = Integer.parseInt(tokens[3]); int processId = Integer.parseInt(tokens[4]); eventsByProcess.put(new ProcessKey(daemonId, processId), new LoadProfileEvent(startTime, operation, target, daemonId, processId)); } } return eventsByProcess; }
/** Split string x in tokens. Effectively just a friendly wrapper around StrTokenizer. * Use *single* quotes for avoiding splitting. */ public static ArrayList<String> tokenize(String x, String delimiterString){ if(x == null){ return null; } // This is a hack to allow empty tokens to be passed at the command line. // An empty x= x.replace("''", "' '"); // See also http://stackoverflow.com/questions/38161437/inconsistent-behaviour-of-strtokenizer-to-split-string StrTokenizer str= new StrTokenizer(x); str.setTrimmerMatcher(StrMatcher.spaceMatcher()); str.setDelimiterString(delimiterString); str.setQuoteChar('\''); // str.setIgnoreEmptyTokens(false); ArrayList<String> tokens= (ArrayList<String>) str.getTokenList(); for(int i= 0; i < tokens.size(); i++){ String tok= tokens.get(i).trim(); tokens.set(i, tok); } return tokens; }
StrTokenizer tokenizer = StrTokenizer.getTSVInstance(); while ( (line = ...) != null) { tokenizer.reset(line); String tokArray[] = tokenizer.getTokenArray(); }
List<String> inputLines = CharStreams.readLines(reader); StrTokenizer st = StrTokenizer.getCSVInstance(); st.setDelimiterString(delimiter); if (quoteChar != '\0') { st.setQuoteChar(quoteChar); } else { st.setQuoteMatcher(StrMatcher.noneMatcher()); List<Column> columns = initColumns(st, headerLine); for (String line : inputLines) { st.reset(line); String[] colArray = st.getTokenArray(); int len = colArray.length; checkState(len == columns.size(), "Mismatch between number of header columns and number of line columns.");
@Override public LineEntityParser makeParser(List<String> header) { assert header.size() == getHeaderLines(); if (usesHeader() && labeledColumns != null) { assert header.size() == 1; List<TypedName<?>> cols = new ArrayList<>(); StrTokenizer tok = new StrTokenizer(header.get(0), delimiter); tok.setQuoteChar('"'); while (tok.hasNext()) { String label = tok.next(); cols.add(labeledColumns.get(label)); } return new OrderedParser(cols, tok); } else { Preconditions.checkState(columns != null, "no columns specified"); StrTokenizer tok = new StrTokenizer("", delimiter); tok.setQuoteChar('"'); return new OrderedParser(columns, tok); } }