@Override public void process(char token) { String cword = getCurrent().getWord(); // let's see if this is a contraction. if (cword.equals("'")) { String word = getNextWord(); if (Contractions.contains(word)) { // just change the state type to text, this will end up being a // word. getCurrent().stateindex = TokenType.TEXT.ordinal(); state = getCurrent().stateindex; return; } } else if (cword.equals(".") && Character.isDigit(token)) { // This is a decimal number (probably), just keep the current state and // make it a word token getCurrent().stateindex = TokenType.TEXT.ordinal(); state = getCurrent().stateindex; return; } pop(current); push(new State(TokenizerState.IN_WORD), current); } },
@Override public void process(char token) { String cword = getCurrent().getWord(); // let's see if this is a contraction. if (cword.equals("'")) { String word = getNextWord(); if (Contractions.contains(word)) { // just change the state type to text, this will end up being a // word. getCurrent().stateindex = TokenType.TEXT.ordinal(); state = getCurrent().stateindex; return; } } else if (cword.equals(".") && Character.isDigit(token)) { // This is a decimal number (probably), just keep the current state and // make it a word token getCurrent().stateindex = TokenType.TEXT.ordinal(); state = getCurrent().stateindex; return; } pop(current); push(new State(TokenizerState.IN_WORD), current); } },
@Override public void process(char token) { String cword = getCurrent().getWord(); // let's see if this is a contraction. if (cword.equals("'")) { String word = getNextWord(); if (Contractions.contains(word)) { // just change the state type to text, this will end up being a // word. getCurrent().stateindex = TokenType.TEXT.ordinal(); state = getCurrent().stateindex; return; } } else if (cword.equals(".") && Character.isDigit(token)) { // This is a decimal number (probably), just keep the current state and // make it a word token getCurrent().stateindex = TokenType.TEXT.ordinal(); state = getCurrent().stateindex; return; } pop(current); push(new State(TokenizerState.IN_WORD), current); } },