public ImmutableList<ErrorProneToken> getTokens() { Scanner scanner = new AccessibleScanner(scannerFactory, commentSavingTokenizer); ImmutableList.Builder<ErrorProneToken> tokens = ImmutableList.builder(); do { scanner.nextToken(); tokens.add(new ErrorProneToken(scanner.token())); } while (scanner.token().kind != TokenKind.EOF); return tokens.build(); }
case FF: // (Spec 3.6) do { scanChar(); } while (ch == ' ' || ch == '\t' || ch == FF); endPos = bp; processWhiteSpace(); break; case LF: // (Spec 3.4) scanChar(); endPos = bp; processLineTerminator(); break; case CR: // (Spec 3.4) scanChar(); if (ch == LF) { scanChar(); processLineTerminator(); break; case 'A': case 'B': case 'C': case 'D': case 'E': case 'z': case '$': case '_': scanIdent(); return; case '0': scanChar(); if (ch == 'x' || ch == 'X') { scanChar();
/** Read fractional part and 'd' or 'f' suffix of floating point number. */ private void scanHexFractionAndSuffix(boolean seendigit) { this.radix = 16; assert ch == '.'; putChar(ch); scanChar(); while (digit(16) >= 0) { seendigit = true; putChar(ch); scanChar(); } if (!seendigit) lexError("invalid.hex.number"); else scanHexExponentAndSuffix(); }
while (digit(digitRadix) >= 0) { seendigit = true; putChar(ch); scanChar(); scanHexFractionAndSuffix(seendigit); } else if (seendigit && radix == 16 && (ch == 'p' || ch == 'P')) { scanHexExponentAndSuffix(); } else if (radix <= 10 && ch == '.') { putChar(ch); scanChar(); scanFractionAndSuffix(); } else if (radix <= 10 && (ch == 'e' || ch == 'E' || ch == 'f' || ch == 'F' || ch == 'd' || ch == 'D')) { scanFractionAndSuffix(); } else { if (ch == 'l' || ch == 'L') { scanChar(); token = LONGLITERAL; } else {
public Scanner newScanner(char[] input, int inputLength) { return new Scanner(this, input, inputLength); } }
/** Read longest possible sequence of special characters and convert * to token. */ private void scanOperator() { while (true) { putChar(ch); Name newname = names.fromChars(sbuf, 0, sp); if (keywords.key(newname) == IDENTIFIER) { sp--; break; } name = newname; token = keywords.key(newname); scanChar(); if (!isSpecial(ch)) break; } }
/** Read next character in comment, skipping over double '\' characters. */ private void scanCommentChar() { scanChar(); if (ch == '\\') { if (buf[bp+1] == '\\' && unicodeConversionBp != bp) { bp++; } else { convertUnicode(); } } }
/** * Called when a complete comment has been scanned. pos and endPos * will mark the comment boundary. */ protected void processComment(CommentStyle style) { if (scannerDebug) System.out.println("processComment(" + pos + "," + endPos + "," + style + ")=|" + new String(getRawCharacters(pos, endPos)) + "|"); }
@Override protected void processComment(CommentStyle style) { int prevEndPos = Math.max(prevEndPos(), endComment); int pos = pos(); int endPos = endPos(); endComment = endPos; String content = new String(getRawCharacters(pos, endPos)); StartConnection start = determineStartConnection(prevEndPos, pos); EndConnection end = determineEndConnection(endPos); CommentInfo comment = new CommentInfo(prevEndPos, pos, endPos, content, start, end); comments.append(comment); super.processComment(style); }
public void nextToken() { docComment = null; super.nextToken(); }
public Token token() { return token(0); }
public Token token(int lookahead) { if (lookahead == 0) { return token; } else { ensureLookahead(lookahead); return savedTokens.get(lookahead - 1); } } //where
/** Read next character. */ private void scanChar() { ch = buf[++bp]; if (ch == '\\') { convertUnicode(); } }
parseErrors |= (log.nerrors > initialErrorCount); if (lineDebugInfo) { tree.lineMap = scanner.getLineMap();
case FF: // (Spec 3.6) do { scanChar(); } while (ch == ' ' || ch == '\t' || ch == FF); endPos = bp; processWhiteSpace(); break; case LF: // (Spec 3.4) scanChar(); endPos = bp; processLineTerminator(); break; case CR: // (Spec 3.4) scanChar(); if (ch == LF) { scanChar(); processLineTerminator(); break; case 'A': case 'B': case 'C': case 'D': case 'E': case 'z': case '$': case '_': scanIdent(); return; case '0': scanChar(); if (ch == 'x' || ch == 'X') { scanChar();
/** Read fractional part and 'd' or 'f' suffix of floating point number. */ private void scanHexFractionAndSuffix(boolean seendigit) { this.radix = 16; assert ch == '.'; putChar(ch); scanChar(); while (digit(16) >= 0) { seendigit = true; putChar(ch); scanChar(); } if (!seendigit) lexError("invalid.hex.number"); else scanHexExponentAndSuffix(); }
while (digit(digitRadix) >= 0) { seendigit = true; putChar(ch); scanChar(); scanHexFractionAndSuffix(seendigit); } else if (seendigit && radix == 16 && (ch == 'p' || ch == 'P')) { scanHexExponentAndSuffix(); } else if (radix <= 10 && ch == '.') { putChar(ch); scanChar(); scanFractionAndSuffix(); } else if (radix <= 10 && (ch == 'e' || ch == 'E' || ch == 'f' || ch == 'F' || ch == 'd' || ch == 'D')) { scanFractionAndSuffix(); } else { if (ch == 'l' || ch == 'L') { scanChar(); token = LONGLITERAL; } else {
public Scanner newScanner(char[] input, int inputLength) { return new Scanner(this, input, inputLength); } }
/** Read longest possible sequence of special characters and convert * to token. */ private void scanOperator() { while (true) { putChar(ch); Name newname = names.fromChars(sbuf, 0, sp); if (keywords.key(newname) == IDENTIFIER) { sp--; break; } name = newname; token = keywords.key(newname); scanChar(); if (!isSpecial(ch)) break; } }