public static Token.Builder mockTokenBuilder(TokenType type, String value) { try { return Token.builder() .setType(type) .setValueAndOriginalValue(value) .setURI(new URI("tests://unittest")) .setLine(1) .setColumn(1); } catch (URISyntaxException e) { throw new RuntimeException(e); } }
public static Token.Builder mockTokenBuilder(TokenType type, String value) { try { return Token.builder() .setType(type) .setValueAndOriginalValue(value) .setURI(new URI("tests://unittest")) .setLine(1) .setColumn(1); } catch (URISyntaxException e) { throw new RuntimeException(e); } }
@Override public boolean consume(CodeReader code, Lexer lexer) { if (code.popTo(matcher, tmpBuilder) > 0) { String word = tmpBuilder.toString(); String wordOriginal = word; if (!caseSensitive) { word = word.toUpperCase(); } TokenType keywordType = keywordsMap.get(word); Token token = tokenBuilder .setType(keywordType == null ? IDENTIFIER : keywordType) .setValueAndOriginalValue(word, wordOriginal) .setURI(lexer.getURI()) .setLine(code.getPreviousCursor().getLine()) .setColumn(code.getPreviousCursor().getColumn()) .build(); lexer.addToken(token); tmpBuilder.delete(0, tmpBuilder.length()); return true; } return false; }
@Override public boolean consume(CodeReader code, Lexer output) { int line = code.getLinePosition(); int column = code.getColumnPosition(); index = 0; ch = code.charAt(index); if ((ch != '\'') && (ch != '\"') && (ch != '`')) { return false; } if (!read(code)) { return false; } for (int i = 0; i < index; i++) { sb.append((char) code.pop()); } output.addToken(Token.builder() .setLine(line) .setColumn(column) .setURI(output.getURI()) .setValueAndOriginalValue(sb.toString()) .setType(PerlTokenType.STRING) .build()); sb.setLength(0); return true; }
private List<Token> lex(Reader reader) { tokens = new ArrayList<>(); initPreprocessors(); CodeReader code = new CodeReader(reader, configuration); try { channelDispatcher.consume(code, this); addToken(Token.builder() .setType(EOF) .setValueAndOriginalValue("EOF") .setURI(uri) .setLine(code.getLinePosition()) .setColumn(code.getColumnPosition()) .build()); preprocess(); return getTokens(); } catch (Exception e) { throw new LexerException("Unable to lex source code at line : " + code.getLinePosition() + " and column : " + code.getColumnPosition() + " in file : " + uri, e); } }
private List<Token> lex(Reader reader) { tokens = new ArrayList<>(); initPreprocessors(); CodeReader code = new CodeReader(reader, configuration); try { channelDispatcher.consume(code, this); addToken(Token.builder() .setType(EOF) .setValueAndOriginalValue("EOF") .setURI(uri) .setLine(code.getLinePosition()) .setColumn(code.getColumnPosition()) .build()); preprocess(); return getTokens(); } catch (Exception e) { throw new LexerException("Unable to lex source code at line : " + code.getLinePosition() + " and column : " + code.getColumnPosition() + " in file : " + uri, e); } }
private void updateTokenPositionAndValue(ParseNode node) { TextLocation location = input.getLocation(node.getStartIndex()); if (location == null) { tokenBuilder.setGeneratedCode(true); // Godin: line, column and uri has no value for generated code, but we should bypass checks in TokenBuilder tokenBuilder.setLine(1); tokenBuilder.setColumn(0); tokenBuilder.setURI(FAKE_URI); } else { tokenBuilder.setGeneratedCode(false); tokenBuilder.setLine(location.getLine()); tokenBuilder.setColumn(location.getColumn() - 1); tokenBuilder.setURI(location.getFileURI() == null ? FAKE_URI : location.getFileURI()); tokenBuilder.notCopyBook(); } String value = getValue(node); tokenBuilder.setValueAndOriginalValue(value); }
private boolean processNewLine(CodeReader code, Lexer output) { if (output.getTokens().isEmpty() || (output.getTokens().get(output.getTokens().size() - 1).getType().equals(PerlTokenType.NEWLINE))) { // Blank line consumeEOL(code); return true; } // NEWLINE token output.addToken(Token.builder() .setLine(code.getLinePosition()) .setColumn(code.getColumnPosition()) .setURI(output.getURI()) .setType(PerlTokenType.NEWLINE) .setValueAndOriginalValue("\n") .setGeneratedCode(true) .build()); consumeEOL(code); return false; }
private List<Token> lex(Reader reader) { tokens = Lists.newArrayList(); initPreprocessors(); CodeReader code = new CodeReader(reader, configuration); try { channelDispatcher.consume(code, this); addToken(Token.builder() .setType(EOF) .setValueAndOriginalValue("EOF") .setURI(uri) .setLine(code.getLinePosition()) .setColumn(code.getColumnPosition()) .build()); preprocess(); return getTokens(); } catch (Exception e) { throw new LexerException("Unable to lex source code at line : " + code.getLinePosition() + " and column : " + code.getColumnPosition() + " in file : " + uri, e); } }
private void updateTokenPositionAndValue(ParseNode node) { TextLocation location = input.getLocation(node.getStartIndex()); if (location == null) { tokenBuilder.setGeneratedCode(true); // Godin: line, column and uri has no value for generated code, but we should bypass checks in TokenBuilder tokenBuilder.setLine(1); tokenBuilder.setColumn(0); tokenBuilder.setURI(FAKE_URI); } else { tokenBuilder.setGeneratedCode(false); tokenBuilder.setLine(location.getLine()); tokenBuilder.setColumn(location.getColumn() - 1); tokenBuilder.setURI(location.getFileURI() == null ? FAKE_URI : location.getFileURI()); tokenBuilder.notCopyBook(); } String value = getValue(node); tokenBuilder.setValueAndOriginalValue(value); }
@Override public AstNode createTerminal(Input input, int startIndex, int endIndex, List<Trivia> trivias, TokenType type) { int[] lineAndColumn = input.lineAndColumnAt(startIndex); Token token = Token.builder() .setType(type == null ? UNDEFINED_TOKEN_TYPE : type) .setLine(lineAndColumn[0]) .setColumn(lineAndColumn[1] - 1) .setValueAndOriginalValue(input.substring(startIndex, endIndex)) .setURI(input.uri()) .setGeneratedCode(false) .setTrivia(trivias) .build(); AstNode astNode = new AstNode(token); astNode.setFromIndex(startIndex); astNode.setToIndex(endIndex); return astNode; }
@Override public boolean consume(CodeReader code, Lexer output) { int line = code.getLinePosition(); int column = code.getColumnPosition(); sb = new StringBuilder(); if (column == 0 && code.popTo(POD_START, sb) > 0) { String pod = consumePod(code); // POD token output.addToken(Token.builder() .setLine(line) .setColumn(column) .setURI(output.getURI()) .setType(GenericTokenType.COMMENT) .setValueAndOriginalValue(pod) .setGeneratedCode(true) .build()); return true; } return false; }
@Override public boolean consume(CodeReader code, Lexer lexer) { if (code.peek() != -1) { char unknownChar = (char) code.pop(); Token token = tokenBuilder .setType(UNKNOWN_CHAR) .setValueAndOriginalValue(String.valueOf(unknownChar)) .setURI(lexer.getURI()) .setLine(code.getLinePosition()) .setColumn(code.getColumnPosition() - 1) .build(); lexer.addToken(token); return true; } return false; }
@Override public boolean consume(CodeReader code, Lexer lexer) { if (code.peek() != -1) { char unknownChar = (char) code.pop(); Token token = tokenBuilder .setType(UNKNOWN_CHAR) .setValueAndOriginalValue(String.valueOf(unknownChar)) .setURI(lexer.getURI()) .setLine(code.getLinePosition()) .setColumn(code.getColumnPosition() - 1) .build(); lexer.addToken(token); return true; } return false; }
@Override public AstNode createTerminal(Input input, int startIndex, int endIndex, List<Trivia> trivias, TokenType type) { int[] lineAndColumn = input.lineAndColumnAt(startIndex); Token token = Token.builder() .setType(type == null ? UNDEFINED_TOKEN_TYPE : type) .setLine(lineAndColumn[0]) .setColumn(lineAndColumn[1] - 1) .setValueAndOriginalValue(input.substring(startIndex, endIndex)) .setURI(input.uri()) .setGeneratedCode(false) .setTrivia(trivias) .build(); AstNode astNode = new AstNode(token); astNode.setFromIndex(startIndex); astNode.setToIndex(endIndex); return astNode; }
@Override public boolean consume(CodeReader code, Lexer lexer) { if (code.peek() != -1) { char unknownChar = (char) code.pop(); Token token = tokenBuilder .setType(UNKNOWN_CHAR) .setValueAndOriginalValue(String.valueOf(unknownChar)) .setURI(lexer.getURI()) .setLine(code.getLinePosition()) .setColumn(code.getColumnPosition() - 1) .build(); lexer.addToken(token); return true; } return false; }
private void updateTokenPositionAndValue(ParseNode node) { tokenBuilder.setGeneratedCode(false); int[] lineAndColumn = input.lineAndColumnAt(node.getStartIndex()); tokenBuilder.setLine(lineAndColumn[0]); tokenBuilder.setColumn(lineAndColumn[1] - 1); tokenBuilder.setURI(input.uri()); String value = input.substring(node.getStartIndex(), node.getEndIndex()); tokenBuilder.setValueAndOriginalValue(value); }
private void addComment(ParseNode node) { tokenBuilder.setGeneratedCode(false); int[] lineAndColumn = input.lineAndColumnAt(node.getStartIndex()); tokenBuilder.setLine(lineAndColumn[0]); tokenBuilder.setColumn(lineAndColumn[1] - 1); tokenBuilder.setURI(input.uri()); String value = input.substring(node.getStartIndex(), node.getEndIndex()); tokenBuilder.setValueAndOriginalValue(value); tokenBuilder.setTrivia(Collections.<Trivia>emptyList()); tokenBuilder.setType(GenericTokenType.COMMENT); trivias.add(Trivia.createComment(tokenBuilder.build())); }
private void updateTokenPositionAndValue(ParseNode node) { tokenBuilder.setGeneratedCode(false); int[] lineAndColumn = input.lineAndColumnAt(node.getStartIndex()); tokenBuilder.setLine(lineAndColumn[0]); tokenBuilder.setColumn(lineAndColumn[1] - 1); tokenBuilder.setURI(input.uri()); String value = input.substring(node.getStartIndex(), node.getEndIndex()); tokenBuilder.setValueAndOriginalValue(value); }
private void addComment(ParseNode node) { tokenBuilder.setGeneratedCode(false); int[] lineAndColumn = input.lineAndColumnAt(node.getStartIndex()); tokenBuilder.setLine(lineAndColumn[0]); tokenBuilder.setColumn(lineAndColumn[1] - 1); tokenBuilder.setURI(input.uri()); String value = input.substring(node.getStartIndex(), node.getEndIndex()); tokenBuilder.setValueAndOriginalValue(value); tokenBuilder.setTrivia(Collections.<Trivia>emptyList()); tokenBuilder.setType(GenericTokenType.COMMENT); trivias.add(Trivia.createComment(tokenBuilder.build())); }