public void addToken(Token... tokens) { checkArgument(tokens.length > 0, "at least one token must be given"); Token firstToken = tokens[0]; Token firstTokenWithTrivia; // Performance optimization: no need to rebuild token, if there is no trivia if (trivia.isEmpty() && !firstToken.hasTrivia()) { firstTokenWithTrivia = firstToken; } else { firstTokenWithTrivia = Token.builder(firstToken).setTrivia(trivia).build(); trivia.clear(); } this.tokens.add(firstTokenWithTrivia); if (tokens.length > 1) { this.tokens.addAll(Arrays.asList(tokens).subList(1, tokens.length)); } }
public void addToken(Token... tokens) { if (tokens.length <= 0) { throw new IllegalArgumentException("at least one token must be given"); } Token firstToken = tokens[0]; Token firstTokenWithTrivia; // Performance optimization: no need to rebuild token, if there is no trivia if (trivia.isEmpty() && !firstToken.hasTrivia()) { firstTokenWithTrivia = firstToken; } else { firstTokenWithTrivia = Token.builder(firstToken).setTrivia(trivia).build(); trivia.clear(); } this.tokens.add(firstTokenWithTrivia); if (tokens.length > 1) { this.tokens.addAll(Arrays.asList(tokens).subList(1, tokens.length)); } }
public void addToken(Token... tokens) { if (tokens.length <= 0) { throw new IllegalArgumentException("at least one token must be given"); } Token firstToken = tokens[0]; Token firstTokenWithTrivia; // Performance optimization: no need to rebuild token, if there is no trivia if (trivia.isEmpty() && !firstToken.hasTrivia()) { firstTokenWithTrivia = firstToken; } else { firstTokenWithTrivia = Token.builder(firstToken).setTrivia(trivia).build(); trivia.clear(); } this.tokens.add(firstTokenWithTrivia); if (tokens.length > 1) { this.tokens.addAll(Arrays.asList(tokens).subList(1, tokens.length)); } }
private void processIndents(Lexer lexer, int line, int column, int indentationLevel) { if (indentationLevel > lexerState.indentationStack.peek()) { lexerState.indentationStack.push(indentationLevel); lexer.addToken(Token.builder() .setType(PythonTokenType.INDENT) .setValueAndOriginalValue(buffer.toString()) .setURI(lexer.getURI()) .setLine(line) .setColumn(column) .build()); } else if (indentationLevel < lexerState.indentationStack.peek()) { while (indentationLevel < lexerState.indentationStack.peek()) { lexerState.indentationStack.pop(); lexer.addToken(Token.builder() .setType(PythonTokenType.DEDENT) .setValueAndOriginalValue(buffer.toString()) .setURI(lexer.getURI()) .setLine(line) .setColumn(column) .build()); } } }
@Override public boolean consume(CodeReader code, Lexer lexer) { char[] next = code.peek(lookahead); for (int i = 0; i < sortedPunctuators.length; i++) { if (arraysEquals(next, sortedPunctuatorsChars[i])) { Token token = tokenBuilder .setType(sortedPunctuators[i]) .setValueAndOriginalValue(sortedPunctuators[i].getValue()) .setURI(lexer.getURI()) .setLine(code.getLinePosition()) .setColumn(code.getColumnPosition()) .build(); lexer.addToken(token); /* Advance the CodeReader stream by the length of the punctuator */ for (int j = 0; j < sortedPunctuatorsChars[i].length; j++) { code.pop(); } return true; } } return false; }
private List<Token> lex(Reader reader) { tokens = new ArrayList<>(); initPreprocessors(); CodeReader code = new CodeReader(reader, configuration); try { channelDispatcher.consume(code, this); addToken(Token.builder() .setType(EOF) .setValueAndOriginalValue("EOF") .setURI(uri) .setLine(code.getLinePosition()) .setColumn(code.getColumnPosition()) .build()); preprocess(); return getTokens(); } catch (Exception e) { throw new LexerException("Unable to lex source code at line : " + code.getLinePosition() + " and column : " + code.getColumnPosition() + " in file : " + uri, e); } }
@Override public boolean consume(CodeReader code, Lexer lexer) { if (code.popTo(matcher, tmpBuilder) > 0) { String word = tmpBuilder.toString(); String wordOriginal = word; if (!caseSensitive) { word = word.toUpperCase(); } TokenType keywordType = keywordsMap.get(word); Token token = tokenBuilder .setType(keywordType == null ? IDENTIFIER : keywordType) .setValueAndOriginalValue(word, wordOriginal) .setURI(lexer.getURI()) .setLine(code.getPreviousCursor().getLine()) .setColumn(code.getPreviousCursor().getColumn()) .build(); lexer.addToken(token); tmpBuilder.delete(0, tmpBuilder.length()); return true; } return false; }
private List<Token> lex(Reader reader) { tokens = Lists.newArrayList(); initPreprocessors(); CodeReader code = new CodeReader(reader, configuration); try { channelDispatcher.consume(code, this); addToken(Token.builder() .setType(EOF) .setValueAndOriginalValue("EOF") .setURI(uri) .setLine(code.getLinePosition()) .setColumn(code.getColumnPosition()) .build()); preprocess(); return getTokens(); } catch (Exception e) { throw new LexerException("Unable to lex source code at line : " + code.getLinePosition() + " and column : " + code.getColumnPosition() + " in file : " + uri, e); } }
@Override public boolean consume(CodeReader code, Lexer output) { int line = code.getLinePosition(); int column = code.getColumnPosition(); sb = new StringBuilder(); if (column == 0 && code.popTo(POD_START, sb) > 0) { String pod = consumePod(code); // POD token output.addToken(Token.builder() .setLine(line) .setColumn(column) .setURI(output.getURI()) .setType(GenericTokenType.COMMENT) .setValueAndOriginalValue(pod) .setGeneratedCode(true) .build()); return true; } return false; }
private List<Token> lex(Reader reader) { tokens = new ArrayList<>(); initPreprocessors(); CodeReader code = new CodeReader(reader, configuration); try { channelDispatcher.consume(code, this); addToken(Token.builder() .setType(EOF) .setValueAndOriginalValue("EOF") .setURI(uri) .setLine(code.getLinePosition()) .setColumn(code.getColumnPosition()) .build()); preprocess(); return getTokens(); } catch (Exception e) { throw new LexerException("Unable to lex source code at line : " + code.getLinePosition() + " and column : " + code.getColumnPosition() + " in file : " + uri, e); } }
private boolean processNewLine(CodeReader code, Lexer output) { if (output.getTokens().isEmpty() || (output.getTokens().get(output.getTokens().size() - 1).getType().equals(PerlTokenType.NEWLINE))) { // Blank line consumeEOL(code); return true; } // NEWLINE token output.addToken(Token.builder() .setLine(code.getLinePosition()) .setColumn(code.getColumnPosition()) .setURI(output.getURI()) .setType(PerlTokenType.NEWLINE) .setValueAndOriginalValue("\n") .setGeneratedCode(true) .build()); consumeEOL(code); return false; }
@Override public AstNode createTerminal(Input input, int startIndex, int endIndex, List<Trivia> trivias, TokenType type) { int[] lineAndColumn = input.lineAndColumnAt(startIndex); Token token = Token.builder() .setType(type == null ? UNDEFINED_TOKEN_TYPE : type) .setLine(lineAndColumn[0]) .setColumn(lineAndColumn[1] - 1) .setValueAndOriginalValue(input.substring(startIndex, endIndex)) .setURI(input.uri()) .setGeneratedCode(false) .setTrivia(trivias) .build(); AstNode astNode = new AstNode(token); astNode.setFromIndex(startIndex); astNode.setToIndex(endIndex); return astNode; }
@Override public boolean consume(CodeReader code, Lexer lexer) { if (code.peek() != -1) { char unknownChar = (char) code.pop(); Token token = tokenBuilder .setType(UNKNOWN_CHAR) .setValueAndOriginalValue(String.valueOf(unknownChar)) .setURI(lexer.getURI()) .setLine(code.getLinePosition()) .setColumn(code.getColumnPosition() - 1) .build(); lexer.addToken(token); return true; } return false; }
@Override public boolean consume(CodeReader code, Lexer lexer) { if (code.peek() != -1) { char unknownChar = (char) code.pop(); Token token = tokenBuilder .setType(UNKNOWN_CHAR) .setValueAndOriginalValue(String.valueOf(unknownChar)) .setURI(lexer.getURI()) .setLine(code.getLinePosition()) .setColumn(code.getColumnPosition() - 1) .build(); lexer.addToken(token); return true; } return false; }
@Override public AstNode createTerminal(Input input, int startIndex, int endIndex, List<Trivia> trivias, TokenType type) { int[] lineAndColumn = input.lineAndColumnAt(startIndex); Token token = Token.builder() .setType(type == null ? UNDEFINED_TOKEN_TYPE : type) .setLine(lineAndColumn[0]) .setColumn(lineAndColumn[1] - 1) .setValueAndOriginalValue(input.substring(startIndex, endIndex)) .setURI(input.uri()) .setGeneratedCode(false) .setTrivia(trivias) .build(); AstNode astNode = new AstNode(token); astNode.setFromIndex(startIndex); astNode.setToIndex(endIndex); return astNode; }
@Override public boolean consume(CodeReader code, Lexer lexer) { if (code.peek() != -1) { char unknownChar = (char) code.pop(); Token token = tokenBuilder .setType(UNKNOWN_CHAR) .setValueAndOriginalValue(String.valueOf(unknownChar)) .setURI(lexer.getURI()) .setLine(code.getLinePosition()) .setColumn(code.getColumnPosition() - 1) .build(); lexer.addToken(token); return true; } return false; }
public static Token mockToken(TokenType type, String value) { return mockTokenBuilder(type, value).build(); }
public static Token mockToken(TokenType type, String value) { return mockTokenBuilder(type, value).build(); }
private void addComment(ParseNode node) { tokenBuilder.setGeneratedCode(false); int[] lineAndColumn = input.lineAndColumnAt(node.getStartIndex()); tokenBuilder.setLine(lineAndColumn[0]); tokenBuilder.setColumn(lineAndColumn[1] - 1); tokenBuilder.setURI(input.uri()); String value = input.substring(node.getStartIndex(), node.getEndIndex()); tokenBuilder.setValueAndOriginalValue(value); tokenBuilder.setTrivia(Collections.<Trivia>emptyList()); tokenBuilder.setType(GenericTokenType.COMMENT); trivias.add(Trivia.createComment(tokenBuilder.build())); }
private void addComment(ParseNode node) { tokenBuilder.setGeneratedCode(false); int[] lineAndColumn = input.lineAndColumnAt(node.getStartIndex()); tokenBuilder.setLine(lineAndColumn[0]); tokenBuilder.setColumn(lineAndColumn[1] - 1); tokenBuilder.setURI(input.uri()); String value = input.substring(node.getStartIndex(), node.getEndIndex()); tokenBuilder.setValueAndOriginalValue(value); tokenBuilder.setTrivia(Collections.<Trivia>emptyList()); tokenBuilder.setType(GenericTokenType.COMMENT); trivias.add(Trivia.createComment(tokenBuilder.build())); }