@Override public PreprocessorAction process(List<Token> tokens) { Token token = tokens.get(0); if (token.getType().equals(GenericTokenType.EOF)) { if (lexerState.indentationStack.isEmpty()) { return PreprocessorAction.NO_OPERATION; } List<Token> tokensToInject = new ArrayList<>(); while (lexerState.indentationStack.peek() > 0) { lexerState.indentationStack.pop(); tokensToInject.add(Token.builder(token) .setURI(token.getURI()) .setType(PythonTokenType.DEDENT) .setLine(token.getLine()) .setColumn(token.getColumn()) .setValueAndOriginalValue("") .build()); } return new PreprocessorAction(0, Collections.<Trivia>emptyList(), tokensToInject); } return PreprocessorAction.NO_OPERATION; }
public static Token.Builder mockTokenBuilder(TokenType type, String value) { try { return Token.builder() .setType(type) .setValueAndOriginalValue(value) .setURI(new URI("tests://unittest")) .setLine(1) .setColumn(1); } catch (URISyntaxException e) { throw new RuntimeException(e); } }
public static Token.Builder mockTokenBuilder(TokenType type, String value) { try { return Token.builder() .setType(type) .setValueAndOriginalValue(value) .setURI(new URI("tests://unittest")) .setLine(1) .setColumn(1); } catch (URISyntaxException e) { throw new RuntimeException(e); } }
@Override public boolean consume(CodeReader code, Lexer output) { int line = code.getLinePosition(); int column = code.getColumnPosition(); index = 0; ch = code.charAt(index); if ((ch != '\'') && (ch != '\"') && (ch != '`')) { return false; } if (!read(code)) { return false; } for (int i = 0; i < index; i++) { sb.append((char) code.pop()); } output.addToken(Token.builder() .setLine(line) .setColumn(column) .setURI(output.getURI()) .setValueAndOriginalValue(sb.toString()) .setType(PerlTokenType.STRING) .build()); sb.setLength(0); return true; }
@Override public boolean consume(CodeReader code, Lexer lexer) { char[] next = code.peek(lookahead); for (int i = 0; i < sortedPunctuators.length; i++) { if (arraysEquals(next, sortedPunctuatorsChars[i])) { Token token = tokenBuilder .setType(sortedPunctuators[i]) .setValueAndOriginalValue(sortedPunctuators[i].getValue()) .setURI(lexer.getURI()) .setLine(code.getLinePosition()) .setColumn(code.getColumnPosition()) .build(); lexer.addToken(token); /* Advance the CodeReader stream by the length of the punctuator */ for (int j = 0; j < sortedPunctuatorsChars[i].length; j++) { code.pop(); } return true; } } return false; }
private void processIndents(Lexer lexer, int line, int column, int indentationLevel) { if (indentationLevel > lexerState.indentationStack.peek()) { lexerState.indentationStack.push(indentationLevel); lexer.addToken(Token.builder() .setType(PythonTokenType.INDENT) .setValueAndOriginalValue(buffer.toString()) .setURI(lexer.getURI()) .setLine(line) .setColumn(column) .build()); } else if (indentationLevel < lexerState.indentationStack.peek()) { while (indentationLevel < lexerState.indentationStack.peek()) { lexerState.indentationStack.pop(); lexer.addToken(Token.builder() .setType(PythonTokenType.DEDENT) .setValueAndOriginalValue(buffer.toString()) .setURI(lexer.getURI()) .setLine(line) .setColumn(column) .build()); } } }
@Override public boolean consume(CodeReader code, Lexer lexer) { char[] next = code.peek(lookahead); for (int i = 0; i < sortedPunctuators.length; i++) { if (arraysEquals(next, sortedPunctuatorsChars[i])) { Token token = tokenBuilder .setType(sortedPunctuators[i]) .setValueAndOriginalValue(sortedPunctuators[i].getValue()) .setURI(lexer.getURI()) .setLine(code.getLinePosition()) .setColumn(code.getColumnPosition()) .build(); lexer.addToken(token); /* Advance the CodeReader stream by the length of the punctuator */ for (int j = 0; j < sortedPunctuatorsChars[i].length; j++) { code.pop(); } return true; } } return false; }
private List<Token> lex(Reader reader) { tokens = new ArrayList<>(); initPreprocessors(); CodeReader code = new CodeReader(reader, configuration); try { channelDispatcher.consume(code, this); addToken(Token.builder() .setType(EOF) .setValueAndOriginalValue("EOF") .setURI(uri) .setLine(code.getLinePosition()) .setColumn(code.getColumnPosition()) .build()); preprocess(); return getTokens(); } catch (Exception e) { throw new LexerException("Unable to lex source code at line : " + code.getLinePosition() + " and column : " + code.getColumnPosition() + " in file : " + uri, e); } }
private List<Token> lex(Reader reader) { tokens = Lists.newArrayList(); initPreprocessors(); CodeReader code = new CodeReader(reader, configuration); try { channelDispatcher.consume(code, this); addToken(Token.builder() .setType(EOF) .setValueAndOriginalValue("EOF") .setURI(uri) .setLine(code.getLinePosition()) .setColumn(code.getColumnPosition()) .build()); preprocess(); return getTokens(); } catch (Exception e) { throw new LexerException("Unable to lex source code at line : " + code.getLinePosition() + " and column : " + code.getColumnPosition() + " in file : " + uri, e); } }
private List<Token> lex(Reader reader) { tokens = new ArrayList<>(); initPreprocessors(); CodeReader code = new CodeReader(reader, configuration); try { channelDispatcher.consume(code, this); addToken(Token.builder() .setType(EOF) .setValueAndOriginalValue("EOF") .setURI(uri) .setLine(code.getLinePosition()) .setColumn(code.getColumnPosition()) .build()); preprocess(); return getTokens(); } catch (Exception e) { throw new LexerException("Unable to lex source code at line : " + code.getLinePosition() + " and column : " + code.getColumnPosition() + " in file : " + uri, e); } }
@Override public boolean consume(CodeReader code, Lexer lexer) { if (code.popTo(matcher, tmpBuilder) > 0) { String word = tmpBuilder.toString(); String wordOriginal = word; if (!caseSensitive) { word = word.toUpperCase(); } TokenType keywordType = keywordsMap.get(word); Token token = tokenBuilder .setType(keywordType == null ? IDENTIFIER : keywordType) .setValueAndOriginalValue(word, wordOriginal) .setURI(lexer.getURI()) .setLine(code.getPreviousCursor().getLine()) .setColumn(code.getPreviousCursor().getColumn()) .build(); lexer.addToken(token); tmpBuilder.delete(0, tmpBuilder.length()); return true; } return false; }
@Override public boolean consume(CodeReader code, Lexer output) { int line = code.getLinePosition(); int column = code.getColumnPosition(); sb = new StringBuilder(); if (column == 0 && code.popTo(POD_START, sb) > 0) { String pod = consumePod(code); // POD token output.addToken(Token.builder() .setLine(line) .setColumn(column) .setURI(output.getURI()) .setType(GenericTokenType.COMMENT) .setValueAndOriginalValue(pod) .setGeneratedCode(true) .build()); return true; } return false; }
private boolean processNewLine(CodeReader code, Lexer output) { if (output.getTokens().isEmpty() || (output.getTokens().get(output.getTokens().size() - 1).getType().equals(PerlTokenType.NEWLINE))) { // Blank line consumeEOL(code); return true; } // NEWLINE token output.addToken(Token.builder() .setLine(code.getLinePosition()) .setColumn(code.getColumnPosition()) .setURI(output.getURI()) .setType(PerlTokenType.NEWLINE) .setValueAndOriginalValue("\n") .setGeneratedCode(true) .build()); consumeEOL(code); return false; }
@Override public AstNode createTerminal(Input input, int startIndex, int endIndex, List<Trivia> trivias, TokenType type) { int[] lineAndColumn = input.lineAndColumnAt(startIndex); Token token = Token.builder() .setType(type == null ? UNDEFINED_TOKEN_TYPE : type) .setLine(lineAndColumn[0]) .setColumn(lineAndColumn[1] - 1) .setValueAndOriginalValue(input.substring(startIndex, endIndex)) .setURI(input.uri()) .setGeneratedCode(false) .setTrivia(trivias) .build(); AstNode astNode = new AstNode(token); astNode.setFromIndex(startIndex); astNode.setToIndex(endIndex); return astNode; }
@Override public boolean consume(CodeReader code, Lexer lexer) { if (code.peek() != -1) { char unknownChar = (char) code.pop(); Token token = tokenBuilder .setType(UNKNOWN_CHAR) .setValueAndOriginalValue(String.valueOf(unknownChar)) .setURI(lexer.getURI()) .setLine(code.getLinePosition()) .setColumn(code.getColumnPosition() - 1) .build(); lexer.addToken(token); return true; } return false; }
@Override public boolean consume(CodeReader code, Lexer lexer) { if (code.peek() != -1) { char unknownChar = (char) code.pop(); Token token = tokenBuilder .setType(UNKNOWN_CHAR) .setValueAndOriginalValue(String.valueOf(unknownChar)) .setURI(lexer.getURI()) .setLine(code.getLinePosition()) .setColumn(code.getColumnPosition() - 1) .build(); lexer.addToken(token); return true; } return false; }
@Override public AstNode createTerminal(Input input, int startIndex, int endIndex, List<Trivia> trivias, TokenType type) { int[] lineAndColumn = input.lineAndColumnAt(startIndex); Token token = Token.builder() .setType(type == null ? UNDEFINED_TOKEN_TYPE : type) .setLine(lineAndColumn[0]) .setColumn(lineAndColumn[1] - 1) .setValueAndOriginalValue(input.substring(startIndex, endIndex)) .setURI(input.uri()) .setGeneratedCode(false) .setTrivia(trivias) .build(); AstNode astNode = new AstNode(token); astNode.setFromIndex(startIndex); astNode.setToIndex(endIndex); return astNode; }
@Override public boolean consume(CodeReader code, Lexer lexer) { if (code.peek() != -1) { char unknownChar = (char) code.pop(); Token token = tokenBuilder .setType(UNKNOWN_CHAR) .setValueAndOriginalValue(String.valueOf(unknownChar)) .setURI(lexer.getURI()) .setLine(code.getLinePosition()) .setColumn(code.getColumnPosition() - 1) .build(); lexer.addToken(token); return true; } return false; }
private void addComment(ParseNode node) { tokenBuilder.setGeneratedCode(false); int[] lineAndColumn = input.lineAndColumnAt(node.getStartIndex()); tokenBuilder.setLine(lineAndColumn[0]); tokenBuilder.setColumn(lineAndColumn[1] - 1); tokenBuilder.setURI(input.uri()); String value = input.substring(node.getStartIndex(), node.getEndIndex()); tokenBuilder.setValueAndOriginalValue(value); tokenBuilder.setTrivia(Collections.<Trivia>emptyList()); tokenBuilder.setType(GenericTokenType.COMMENT); trivias.add(Trivia.createComment(tokenBuilder.build())); }
private void addComment(ParseNode node) { tokenBuilder.setGeneratedCode(false); int[] lineAndColumn = input.lineAndColumnAt(node.getStartIndex()); tokenBuilder.setLine(lineAndColumn[0]); tokenBuilder.setColumn(lineAndColumn[1] - 1); tokenBuilder.setURI(input.uri()); String value = input.substring(node.getStartIndex(), node.getEndIndex()); tokenBuilder.setValueAndOriginalValue(value); tokenBuilder.setTrivia(Collections.<Trivia>emptyList()); tokenBuilder.setType(GenericTokenType.COMMENT); trivias.add(Trivia.createComment(tokenBuilder.build())); }