private boolean processNewLine(CodeReader code, Lexer output) { if (output.getTokens().isEmpty() || (output.getTokens().get(output.getTokens().size() - 1).getType().equals(PerlTokenType.NEWLINE))) { // Blank line consumeEOL(code); return true; } // NEWLINE token output.addToken(Token.builder() .setLine(code.getLinePosition()) .setColumn(code.getColumnPosition()) .setURI(output.getURI()) .setType(PerlTokenType.NEWLINE) .setValueAndOriginalValue("\n") .setGeneratedCode(true) .build()); consumeEOL(code); return false; }
private boolean processNewLine(CodeReader code, Lexer output) { if (isImplicitLineJoining()) { // Implicit line joining joinLines(code); return true; } if (output.getTokens().isEmpty() || (output.getTokens().get(output.getTokens().size() - 1).getType().equals(PythonTokenType.NEWLINE))) { // Blank line consumeEOL(code); return true; } // NEWLINE token output.addToken(Token.builder() .setLine(code.getLinePosition()) .setColumn(code.getColumnPosition()) .setURI(output.getURI()) .setType(PythonTokenType.NEWLINE) .setValueAndOriginalValue("\n") .setGeneratedCode(true) .build()); consumeEOL(code); return false; }
private boolean processNewLine(CodeReader code, Lexer output) { if (isImplicitLineJoining()) { // Implicit line joining joinLines(code); return true; } if (output.getTokens().isEmpty() || (output.getTokens().get(output.getTokens().size() - 1).getType().equals(PythonTokenType.NEWLINE))) { // Blank line consumeEOL(code); return true; } // NEWLINE token output.addToken(Token.builder() .setLine(code.getLinePosition()) .setColumn(code.getColumnPosition()) .setURI(output.getURI()) .setType(PythonTokenType.NEWLINE) .setValueAndOriginalValue("\n") .setGeneratedCode(true) .build()); consumeEOL(code); return false; }
private void updateTokenPositionAndValue(ParseNode node) { TextLocation location = input.getLocation(node.getStartIndex()); if (location == null) { tokenBuilder.setGeneratedCode(true); // Godin: line, column and uri has no value for generated code, but we should bypass checks in TokenBuilder tokenBuilder.setLine(1); tokenBuilder.setColumn(0); tokenBuilder.setURI(FAKE_URI); } else { tokenBuilder.setGeneratedCode(false); tokenBuilder.setLine(location.getLine()); tokenBuilder.setColumn(location.getColumn() - 1); tokenBuilder.setURI(location.getFileURI() == null ? FAKE_URI : location.getFileURI()); TextLocation copyLocation = input instanceof CompositeTextCharSequence ? ((CompositeTextCharSequence) input).getCopyLocation(node.getStartIndex()) : null; if (copyLocation == null) { tokenBuilder.notCopyBook(); } else { tokenBuilder.setCopyBook(copyLocation.getFile().getAbsolutePath(), copyLocation.getLine()); } } String value = getValue(node); tokenBuilder.setValueAndOriginalValue(value); }
private boolean processNewLine(CodeReader code, Lexer output) { if (isImplicitLineJoining()) { // Implicit line joining joinLines(code); return true; } if (output.getTokens().isEmpty() || (output.getTokens().get(output.getTokens().size() - 1).getType().equals(PythonTokenType.NEWLINE))) { // Blank line consumeEOL(code); return true; } // NEWLINE token output.addToken(Token.builder() .setLine(code.getLinePosition()) .setColumn(code.getColumnPosition()) .setURI(output.getURI()) .setType(PythonTokenType.NEWLINE) .setValueAndOriginalValue("\n") .setGeneratedCode(true) .build()); consumeEOL(code); return false; }
private void updateTokenPositionAndValue(ParseNode node) { TextLocation location = input.getLocation(node.getStartIndex()); if (location == null) { tokenBuilder.setGeneratedCode(true); // Godin: line, column and uri has no value for generated code, but we should bypass checks in TokenBuilder tokenBuilder.setLine(1); tokenBuilder.setColumn(0); tokenBuilder.setURI(FAKE_URI); } else { tokenBuilder.setGeneratedCode(false); tokenBuilder.setLine(location.getLine()); tokenBuilder.setColumn(location.getColumn() - 1); tokenBuilder.setURI(location.getFileURI() == null ? FAKE_URI : location.getFileURI()); tokenBuilder.notCopyBook(); } String value = getValue(node); tokenBuilder.setValueAndOriginalValue(value); }
private void updateTokenPositionAndValue(ParseNode node) { TextLocation location = input.getLocation(node.getStartIndex()); if (location == null) { tokenBuilder.setGeneratedCode(true); // Godin: line, column and uri has no value for generated code, but we should bypass checks in TokenBuilder tokenBuilder.setLine(1); tokenBuilder.setColumn(0); tokenBuilder.setURI(FAKE_URI); } else { tokenBuilder.setGeneratedCode(false); tokenBuilder.setLine(location.getLine()); tokenBuilder.setColumn(location.getColumn() - 1); tokenBuilder.setURI(location.getFileURI() == null ? FAKE_URI : location.getFileURI()); tokenBuilder.notCopyBook(); } String value = getValue(node); tokenBuilder.setValueAndOriginalValue(value); }
@Override public AstNode createTerminal(Input input, int startIndex, int endIndex, List<Trivia> trivias, TokenType type) { int[] lineAndColumn = input.lineAndColumnAt(startIndex); Token token = Token.builder() .setType(type == null ? UNDEFINED_TOKEN_TYPE : type) .setLine(lineAndColumn[0]) .setColumn(lineAndColumn[1] - 1) .setValueAndOriginalValue(input.substring(startIndex, endIndex)) .setURI(input.uri()) .setGeneratedCode(false) .setTrivia(trivias) .build(); AstNode astNode = new AstNode(token); astNode.setFromIndex(startIndex); astNode.setToIndex(endIndex); return astNode; }
@Override public AstNode createTerminal(Input input, int startIndex, int endIndex, List<Trivia> trivias, TokenType type) { int[] lineAndColumn = input.lineAndColumnAt(startIndex); Token token = Token.builder() .setType(type == null ? UNDEFINED_TOKEN_TYPE : type) .setLine(lineAndColumn[0]) .setColumn(lineAndColumn[1] - 1) .setValueAndOriginalValue(input.substring(startIndex, endIndex)) .setURI(input.uri()) .setGeneratedCode(false) .setTrivia(trivias) .build(); AstNode astNode = new AstNode(token); astNode.setFromIndex(startIndex); astNode.setToIndex(endIndex); return astNode; }
@Override public boolean consume(CodeReader code, Lexer output) { int line = code.getLinePosition(); int column = code.getColumnPosition(); sb = new StringBuilder(); if (column == 0 && code.popTo(POD_START, sb) > 0) { String pod = consumePod(code); // POD token output.addToken(Token.builder() .setLine(line) .setColumn(column) .setURI(output.getURI()) .setType(GenericTokenType.COMMENT) .setValueAndOriginalValue(pod) .setGeneratedCode(true) .build()); return true; } return false; }
private void addComment(ParseNode node) { tokenBuilder.setGeneratedCode(false); int[] lineAndColumn = input.lineAndColumnAt(node.getStartIndex()); tokenBuilder.setLine(lineAndColumn[0]); tokenBuilder.setColumn(lineAndColumn[1] - 1); tokenBuilder.setURI(input.uri()); String value = input.substring(node.getStartIndex(), node.getEndIndex()); tokenBuilder.setValueAndOriginalValue(value); tokenBuilder.setTrivia(Collections.<Trivia>emptyList()); tokenBuilder.setType(GenericTokenType.COMMENT); trivias.add(Trivia.createComment(tokenBuilder.build())); }
private void updateTokenPositionAndValue(ParseNode node) { tokenBuilder.setGeneratedCode(false); int[] lineAndColumn = input.lineAndColumnAt(node.getStartIndex()); tokenBuilder.setLine(lineAndColumn[0]); tokenBuilder.setColumn(lineAndColumn[1] - 1); tokenBuilder.setURI(input.uri()); String value = input.substring(node.getStartIndex(), node.getEndIndex()); tokenBuilder.setValueAndOriginalValue(value); }
private void addComment(ParseNode node) { tokenBuilder.setGeneratedCode(false); int[] lineAndColumn = input.lineAndColumnAt(node.getStartIndex()); tokenBuilder.setLine(lineAndColumn[0]); tokenBuilder.setColumn(lineAndColumn[1] - 1); tokenBuilder.setURI(input.uri()); String value = input.substring(node.getStartIndex(), node.getEndIndex()); tokenBuilder.setValueAndOriginalValue(value); tokenBuilder.setTrivia(Collections.<Trivia>emptyList()); tokenBuilder.setType(GenericTokenType.COMMENT); trivias.add(Trivia.createComment(tokenBuilder.build())); }
private void updateTokenPositionAndValue(ParseNode node) { tokenBuilder.setGeneratedCode(false); int[] lineAndColumn = input.lineAndColumnAt(node.getStartIndex()); tokenBuilder.setLine(lineAndColumn[0]); tokenBuilder.setColumn(lineAndColumn[1] - 1); tokenBuilder.setURI(input.uri()); String value = input.substring(node.getStartIndex(), node.getEndIndex()); tokenBuilder.setValueAndOriginalValue(value); }