private AstNode visitTerminal(ParseNode node) { Token token = tokens.get(node.getStartIndex()); // For compatibility with SSLR < 1.19, TokenType should be checked only for TokenTypeExpression: if ((node.getMatcher() instanceof TokenTypeExpression) && token.getType().hasToBeSkippedFromAst(null)) { return null; } AstNode astNode = new AstNode(token); astNode.setFromIndex(node.getStartIndex()); astNode.setToIndex(node.getEndIndex()); return astNode; }
public AstNode(Token token) { this(token.getType(), token.getType().getName(), token); }
private AstNode visitTerminal(ParseNode node) { Token token = tokens.get(node.getStartIndex()); // For compatibility with SSLR < 1.19, TokenType should be checked only for TokenTypeExpression: if ((node.getMatcher() instanceof TokenTypeExpression) && token.getType().hasToBeSkippedFromAst(null)) { return null; } AstNode astNode = new AstNode(token); astNode.setFromIndex(node.getStartIndex()); astNode.setToIndex(node.getEndIndex()); return astNode; }
public AstNode(Token token) { this(token.getType(), token.getType().getName(), token); }
private AstNode visitTerminal(ParseNode node) { Token token = tokens.get(node.getStartIndex()); // For compatibility with SSLR < 1.19, TokenType should be checked only for TokenTypeExpression: if ((node.getMatcher() instanceof TokenTypeExpression) && token.getType().hasToBeSkippedFromAst(null)) { return null; } AstNode astNode = new AstNode(token); astNode.setFromIndex(node.getStartIndex()); astNode.setToIndex(node.getEndIndex()); return astNode; }
public AstNode(Token token) { this(token.getType(), token.getType().getName(), token); }
/** * @throws java.util.regex.PatternSyntaxException if the expression's syntax is invalid */ public IdentifierAndKeywordChannel(String regexp, boolean caseSensitive, TokenType[]... keywordSets) { for (TokenType[] keywords : keywordSets) { for (TokenType keyword : keywords) { String keywordValue = caseSensitive ? keyword.getValue() : keyword.getValue().toUpperCase(); keywordsMap.put(keywordValue, keyword); } } this.caseSensitive = caseSensitive; matcher = Pattern.compile(regexp).matcher(""); }
/** * @throws java.util.regex.PatternSyntaxException if the expression's syntax is invalid */ public IdentifierAndKeywordChannel(String regexp, boolean caseSensitive, TokenType[]... keywordSets) { for (TokenType[] keywords : keywordSets) { for (TokenType keyword : keywords) { String keywordValue = caseSensitive ? keyword.getValue() : keyword.getValue().toUpperCase(); keywordsMap.put(keywordValue, keyword); } } this.caseSensitive = caseSensitive; matcher = Pattern.compile(regexp).matcher(""); }
/** * @throws java.util.regex.PatternSyntaxException if the expression's syntax is invalid */ public IdentifierAndKeywordChannel(String regexp, boolean caseSensitive, TokenType[]... keywordSets) { ImmutableMap.Builder<String, TokenType> keywordsMapBuilder = ImmutableMap.builder(); for (TokenType[] keywords : keywordSets) { for (TokenType keyword : keywords) { String keywordValue = caseSensitive ? keyword.getValue() : keyword.getValue().toUpperCase(); keywordsMapBuilder.put(keywordValue, keyword); } } this.keywordsMap = keywordsMapBuilder.build(); this.caseSensitive = caseSensitive; matcher = Pattern.compile(regexp).matcher(""); }
public PunctuatorChannel(TokenType... punctuators) { sortedPunctuators = punctuators; Arrays.sort(sortedPunctuators, new PunctuatorComparator()); sortedPunctuatorsChars = new char[sortedPunctuators.length][]; int maxLength = 0; for (int i = 0; i < sortedPunctuators.length; i++) { sortedPunctuatorsChars[i] = sortedPunctuators[i].getValue().toCharArray(); maxLength = Math.max(maxLength, sortedPunctuatorsChars[i].length); } this.lookahead = maxLength; }
public PunctuatorChannel(TokenType... punctuators) { sortedPunctuators = punctuators; Arrays.sort(sortedPunctuators, new PunctuatorComparator()); sortedPunctuatorsChars = new char[sortedPunctuators.length][]; int maxLength = 0; for (int i = 0; i < sortedPunctuators.length; i++) { sortedPunctuatorsChars[i] = sortedPunctuators[i].getValue().toCharArray(); maxLength = Math.max(maxLength, sortedPunctuatorsChars[i].length); } this.lookahead = maxLength; }
public PunctuatorChannel(TokenType... punctuators) { sortedPunctuators = punctuators; Arrays.sort(sortedPunctuators, new PunctuatorComparator()); sortedPunctuatorsChars = new char[sortedPunctuators.length][]; int maxLength = 0; for (int i = 0; i < sortedPunctuators.length; i++) { sortedPunctuatorsChars[i] = sortedPunctuators[i].getValue().toCharArray(); maxLength = Math.max(maxLength, sortedPunctuatorsChars[i].length); } this.lookahead = maxLength; }
public RegexPunctuatorChannel(TokenType... punctuators) { Arrays.sort(punctuators, new PunctuatorComparator()); for (TokenType punctuator : punctuators) { tokenMatchers.put(punctuator, Pattern.compile(punctuator.getValue()).matcher("")); } }
public boolean is(TokenType type) { return this.text().equals(type.getValue()); }
public boolean is(TokenType type) { return this.text().equals(type.getValue()); }
@Override public boolean consume(CodeReader code, Lexer lexer) { for (Map.Entry<TokenType, Matcher> tokenMatcher : tokenMatchers.entrySet()) { TokenType punctuator = tokenMatcher.getKey(); Matcher matcher = tokenMatcher.getValue(); try { if (code.popTo(matcher, tmpBuilder) > 0) { String value = tmpBuilder.toString(); Token token = tokenBuilder .setType(punctuator) .setValueAndOriginalValue(value) .setURI(lexer.getURI()) .setLine(code.getPreviousCursor().getLine()) .setColumn(code.getPreviousCursor().getColumn()) .build(); lexer.addToken(token); tmpBuilder.delete(0, tmpBuilder.length()); return true; } } catch (StackOverflowError e) { throw new LexerException("The regular expression " + punctuator.getValue() + " has led to a stack overflow error.", e); } } return false; } }