/** * @throws java.util.regex.PatternSyntaxException if the expression's syntax is invalid */ public IdentifierAndKeywordChannel(String regexp, boolean caseSensitive, TokenType[]... keywordSets) { for (TokenType[] keywords : keywordSets) { for (TokenType keyword : keywords) { String keywordValue = caseSensitive ? keyword.getValue() : keyword.getValue().toUpperCase(); keywordsMap.put(keywordValue, keyword); } } this.caseSensitive = caseSensitive; matcher = Pattern.compile(regexp).matcher(""); }
/** * @throws java.util.regex.PatternSyntaxException if the expression's syntax is invalid */ public IdentifierAndKeywordChannel(String regexp, boolean caseSensitive, TokenType[]... keywordSets) { for (TokenType[] keywords : keywordSets) { for (TokenType keyword : keywords) { String keywordValue = caseSensitive ? keyword.getValue() : keyword.getValue().toUpperCase(); keywordsMap.put(keywordValue, keyword); } } this.caseSensitive = caseSensitive; matcher = Pattern.compile(regexp).matcher(""); }
/** * @throws java.util.regex.PatternSyntaxException if the expression's syntax is invalid */ public IdentifierAndKeywordChannel(String regexp, boolean caseSensitive, TokenType[]... keywordSets) { ImmutableMap.Builder<String, TokenType> keywordsMapBuilder = ImmutableMap.builder(); for (TokenType[] keywords : keywordSets) { for (TokenType keyword : keywords) { String keywordValue = caseSensitive ? keyword.getValue() : keyword.getValue().toUpperCase(); keywordsMapBuilder.put(keywordValue, keyword); } } this.keywordsMap = keywordsMapBuilder.build(); this.caseSensitive = caseSensitive; matcher = Pattern.compile(regexp).matcher(""); }
public PunctuatorChannel(TokenType... punctuators) { sortedPunctuators = punctuators; Arrays.sort(sortedPunctuators, new PunctuatorComparator()); sortedPunctuatorsChars = new char[sortedPunctuators.length][]; int maxLength = 0; for (int i = 0; i < sortedPunctuators.length; i++) { sortedPunctuatorsChars[i] = sortedPunctuators[i].getValue().toCharArray(); maxLength = Math.max(maxLength, sortedPunctuatorsChars[i].length); } this.lookahead = maxLength; }
public PunctuatorChannel(TokenType... punctuators) { sortedPunctuators = punctuators; Arrays.sort(sortedPunctuators, new PunctuatorComparator()); sortedPunctuatorsChars = new char[sortedPunctuators.length][]; int maxLength = 0; for (int i = 0; i < sortedPunctuators.length; i++) { sortedPunctuatorsChars[i] = sortedPunctuators[i].getValue().toCharArray(); maxLength = Math.max(maxLength, sortedPunctuatorsChars[i].length); } this.lookahead = maxLength; }
public PunctuatorChannel(TokenType... punctuators) { sortedPunctuators = punctuators; Arrays.sort(sortedPunctuators, new PunctuatorComparator()); sortedPunctuatorsChars = new char[sortedPunctuators.length][]; int maxLength = 0; for (int i = 0; i < sortedPunctuators.length; i++) { sortedPunctuatorsChars[i] = sortedPunctuators[i].getValue().toCharArray(); maxLength = Math.max(maxLength, sortedPunctuatorsChars[i].length); } this.lookahead = maxLength; }
public RegexPunctuatorChannel(TokenType... punctuators) { Arrays.sort(punctuators, new PunctuatorComparator()); for (TokenType punctuator : punctuators) { tokenMatchers.put(punctuator, Pattern.compile(punctuator.getValue()).matcher("")); } }
public boolean is(TokenType type) { return this.text().equals(type.getValue()); }
public boolean is(TokenType type) { return this.text().equals(type.getValue()); }
@Override public boolean consume(CodeReader code, Lexer lexer) { for (Map.Entry<TokenType, Matcher> tokenMatcher : tokenMatchers.entrySet()) { TokenType punctuator = tokenMatcher.getKey(); Matcher matcher = tokenMatcher.getValue(); try { if (code.popTo(matcher, tmpBuilder) > 0) { String value = tmpBuilder.toString(); Token token = tokenBuilder .setType(punctuator) .setValueAndOriginalValue(value) .setURI(lexer.getURI()) .setLine(code.getPreviousCursor().getLine()) .setColumn(code.getPreviousCursor().getColumn()) .build(); lexer.addToken(token); tmpBuilder.delete(0, tmpBuilder.length()); return true; } } catch (StackOverflowError e) { throw new LexerException("The regular expression " + punctuator.getValue() + " has led to a stack overflow error.", e); } } return false; } }
@Override public boolean consume(CodeReader code, Lexer lexer) { char[] next = code.peek(lookahead); for (int i = 0; i < sortedPunctuators.length; i++) { if (arraysEquals(next, sortedPunctuatorsChars[i])) { Token token = tokenBuilder .setType(sortedPunctuators[i]) .setValueAndOriginalValue(sortedPunctuators[i].getValue()) .setURI(lexer.getURI()) .setLine(code.getLinePosition()) .setColumn(code.getColumnPosition()) .build(); lexer.addToken(token); /* Advance the CodeReader stream by the length of the punctuator */ for (int j = 0; j < sortedPunctuatorsChars[i].length; j++) { code.pop(); } return true; } } return false; }
@Override public boolean consume(CodeReader code, Lexer lexer) { char[] next = code.peek(lookahead); for (int i = 0; i < sortedPunctuators.length; i++) { if (arraysEquals(next, sortedPunctuatorsChars[i])) { Token token = tokenBuilder .setType(sortedPunctuators[i]) .setValueAndOriginalValue(sortedPunctuators[i].getValue()) .setURI(lexer.getURI()) .setLine(code.getLinePosition()) .setColumn(code.getColumnPosition()) .build(); lexer.addToken(token); /* Advance the CodeReader stream by the length of the punctuator */ for (int j = 0; j < sortedPunctuatorsChars[i].length; j++) { code.pop(); } return true; } } return false; }
@Override public boolean consume(CodeReader code, Lexer lexer) { char[] next = code.peek(lookahead); for (int i = 0; i < sortedPunctuators.length; i++) { if (arraysEquals(next, sortedPunctuatorsChars[i])) { Token token = tokenBuilder .setType(sortedPunctuators[i]) .setValueAndOriginalValue(sortedPunctuators[i].getValue()) .setURI(lexer.getURI()) .setLine(code.getLinePosition()) .setColumn(code.getColumnPosition()) .build(); lexer.addToken(token); /* Advance the CodeReader stream by the length of the punctuator */ for (int j = 0; j < sortedPunctuatorsChars[i].length; j++) { code.pop(); } return true; } } return false; }