@Override public Token nextToken() { if (!queue.isEmpty()) { return queue.poll(); } Token next = super.nextToken(); if (next.getType() != Word) { return next; } Token next2 = super.nextToken(); if (next2.getType() == Punctuation && next2.getText().equals(".")) { String abbrev = next.getText() + "."; if (abbreviations != null && abbreviations.contains(abbrev)) { CommonToken commonToken = new CommonToken(Abbreviation, abbrev); commonToken.setStartIndex(next.getStartIndex()); commonToken.setStopIndex(next2.getStopIndex()); commonToken.setTokenIndex(next.getTokenIndex()); commonToken.setCharPositionInLine(next.getCharPositionInLine()); commonToken.setLine(next.getLine()); return commonToken; } } queue.offer(next2); return next; }
t.setStartIndex(_tokenStartCharIndex); t.setCharPositionInLine(_tokenStartCharIndex); t.setLine(line);
private void unQuoteToken(Token token) { if (token instanceof CommonToken) { CommonToken commonToken = (CommonToken) token; commonToken.setStartIndex(commonToken.getStartIndex() + 1); commonToken.setStopIndex(commonToken.getStopIndex() - 1); } }
public static Token getTokenUnderCursor(CommonTokenStream tokens, int offset) { Comparator<Token> cmp = new Comparator<Token>() { @Override public int compare(Token a, Token b) { if ( a.getStopIndex() < b.getStartIndex() ) return -1; if ( a.getStartIndex() > b.getStopIndex() ) return 1; return 0; } }; if ( offset<0 || offset >= tokens.getTokenSource().getInputStream().size() ) return null; CommonToken key = new CommonToken(Token.INVALID_TYPE, ""); key.setStartIndex(offset); key.setStopIndex(offset); List<Token> tokenList = tokens.getTokens(); Token tokenUnderCursor = null; int i = Collections.binarySearch(tokenList, key, cmp); if ( i>=0 ) tokenUnderCursor = tokenList.get(i); return tokenUnderCursor; }
/** By default ANTLR makes the start/stop -1/-1 for invalid tokens * which is reasonable but here we want to highlight the * current position indicating that is where we lack a token. * if no input, highlight at position 0. */ protected Token getMissingSymbol(Parser recognizer) { Token missingSymbol = super.getMissingSymbol(recognizer); // alter the default missing symbol. if ( missingSymbol instanceof CommonToken) { int start, stop; Token current = recognizer.getCurrentToken(); start = current.getStartIndex(); stop = current.getStopIndex(); ((CommonToken) missingSymbol).setStartIndex(start); ((CommonToken) missingSymbol).setStopIndex(stop); } return missingSymbol; } }
t.setStartIndex(_tokenStartCharIndex); t.setCharPositionInLine(_tokenStartCharIndex); t.setLine(line);