@Override public void process(char token) { pop(current); } }}, {
@Override public void process(char token) { pop(current); } }}, {
@Override public void process(char token) { pop(current); } },
@Override public void process(char token) { pop(current); } }}};
@Override public void process(char token) { pop(current); } },
@Override public void process(char token) { pop(current); } }}};
@Override public void process(char token) { pop(current); } },
@Override public void process(char token) { pop(current); } }}, {
@Override public void process(char token) { pop(current); } }}};
@Override public void process(char token) { pop(current); // pop the special off the stack. State state = completed.get(completed.size() - 1); if (state != null) { char c = text[state.start]; switch (c) { case '.': case '!': case '?': if (!isContinue()) { pop(current); // the sentence } break; case '"': case '\'': // if the current special is a quote, see if there is a line // ender inside the quote if (completed.size() > 2) { state = completed.get(completed.size() - 2); c = text[state.start]; if (c == '.' || c == '!' || c == '?') { pop(current); } } break; } } } },
@Override public void process(char token) { if (stack.get(stack.size()-1).hasMass && token == '\n' && peek(-1) == '\n' && peek(-2) != '\n') { // we are in a sentence, but we will pop it pop(current); } } }
/** * We have encountered a colon in the input data stream, check to see if it is a URL, and if it * is, advance the cursor and return true, or return false. * * @return return true if it is a url. */ protected boolean isEmail() { int start = this.getCurrent().start; String tmp = new String (text).substring(start); Matcher matcher = emailRegex.matcher(tmp); if (matcher.find()) { int end = matcher.end(); current = start + (end-1); this.pop(this.current + 1); return true; } else { return false; } }
@Override public void process(char token) { if (stack.get(stack.size()-1).hasMass && token == '\n' && peek(-1) == '\n' && peek(-2) != '\n') { // we are in a sentence, but we will pop it pop(current); } } }
@Override public void process(char token) { // we will keep like special characters together. if (peek(-1) != token) { pop(current); // the current word is finished. push(new State(TokenizerState.IN_SPECIAL), current); // No matter what // we push a new // word token. } } },
@Override public void process(char token) { // we will keep like special characters together. if (peek(-1) != token) { pop(current); // the current token is finished. if (token == '$') { Character next = peek(1); if (Character.isDigit(next) || ( next == '.' && Character.isDigit(peek(2)))) { push(new State(TokenizerState.IN_WORD), current); return; } } push(new State(TokenizerState.IN_SPECIAL), current); } } },
@Override public void process(char token) { // we will keep like special characters together. if (peek(-1) != token) { pop(current); // the current token is finished. if (token == '$') { Character next = peek(1); if (Character.isDigit(next) || ( next == '.' && Character.isDigit(peek(2)))) { push(new State(TokenizerState.IN_WORD), current); return; } } push(new State(TokenizerState.IN_SPECIAL), current); } } },
@Override public void process(char token) { String cword = getCurrent().getWord(); // let's see if this is a contraction. if (cword.equals("'")) { String word = getNextWord(); if (Contractions.contains(word)) { // just change the state type to text, this will end up being a // word. getCurrent().stateindex = TokenType.TEXT.ordinal(); state = getCurrent().stateindex; return; } } else if (cword.equals(".") && Character.isDigit(token)) { // This is a decimal number (probably), just keep the current state and // make it a word token getCurrent().stateindex = TokenType.TEXT.ordinal(); state = getCurrent().stateindex; return; } pop(current); push(new State(TokenizerState.IN_WORD), current); } },
@Override public void process(char token) { String cword = getCurrent().getWord(); // let's see if this is a contraction. if (cword.equals("'")) { String word = getNextWord(); if (Contractions.contains(word)) { // just change the state type to text, this will end up being a // word. getCurrent().stateindex = TokenType.TEXT.ordinal(); state = getCurrent().stateindex; return; } } else if (cword.equals(".") && Character.isDigit(token)) { // This is a decimal number (probably), just keep the current state and // make it a word token getCurrent().stateindex = TokenType.TEXT.ordinal(); state = getCurrent().stateindex; return; } pop(current); push(new State(TokenizerState.IN_WORD), current); } },
@Override public void process(char token) { String cword = getCurrent().getWord(); // let's see if this is a contraction. if (cword.equals("'")) { String word = getNextWord(); if (Contractions.contains(word)) { // just change the state type to text, this will end up being a // word. getCurrent().stateindex = TokenType.TEXT.ordinal(); state = getCurrent().stateindex; return; } } else if (cword.equals(".") && Character.isDigit(token)) { // This is a decimal number (probably), just keep the current state and // make it a word token getCurrent().stateindex = TokenType.TEXT.ordinal(); state = getCurrent().stateindex; return; } pop(current); push(new State(TokenizerState.IN_WORD), current); } },