private static Object word(LexerlessGrammarBuilder b, String value) { return b.sequence(SPACING, b.token(GenericTokenType.IDENTIFIER, value), b.nextNot(LETTER_OR_DIGIT)); }
private static void embeddedCss(LexerlessGrammarBuilder b) { String openingHtmlStyleTagRegex = "<style[^>]+type[\\s]*=[\\s]*\"text/css\"[^>]*>"; b.rule(OPENING_HTML_STYLE_TAG).is( SPACING, b.token( GenericTokenType.LITERAL, b.regexp(openingHtmlStyleTagRegex))); b.rule(CLOSING_HTML_STYLE_TAG).is( SPACING, b.token( GenericTokenType.LITERAL, b.regexp("</style[\\s]*>"))); b.rule(NON_CSS_TOKEN).is( SPACING, b.token( GenericTokenType.LITERAL, b.firstOf( b.regexp(".+?(?=" + openingHtmlStyleTagRegex + ")"), b.regexp(".+")))); }
private static void embeddedCss(LexerlessGrammarBuilder b) { String openingHtmlStyleTagRegex = "<style[^>]+type[\\s]*=[\\s]*\"text/css\"[^>]*>"; b.rule(OPENING_HTML_STYLE_TAG).is( SPACING, b.token( GenericTokenType.LITERAL, b.regexp(openingHtmlStyleTagRegex))); b.rule(CLOSING_HTML_STYLE_TAG).is( SPACING, b.token( GenericTokenType.LITERAL, b.regexp("</style[\\s]*>"))); b.rule(NON_CSS_TOKEN).is( SPACING, b.token( GenericTokenType.LITERAL, b.firstOf( b.regexp(".+?(?=" + openingHtmlStyleTagRegex + ")"), b.regexp(".+")))); }
private static void scss(LexerlessGrammarBuilder b) { b.rule(SCSS_DOLLAR).is(b.token(GenericTokenType.LITERAL, "$")); b.rule(SCSS_ELLIPSIS).is(b.token(GenericTokenType.LITERAL, "...")); b.rule(SCSS_PARENT_SELECTOR_KEYWORD).is("&"); b.rule(SCSS_DEFAULT_KEYWORD).is(SPACING, b.token(GenericTokenType.LITERAL, "!default")); b.rule(SCSS_GLOBAL_KEYWORD).is(SPACING, b.token(GenericTokenType.LITERAL, "!global")); b.rule(SCSS_OPTIONAL_KEYWORD).is(SPACING, b.token(GenericTokenType.LITERAL, "!optional")); b.rule(SCSS_MIXIN_DIRECTIVE_LITERAL).is(SPACING, b.token(GenericTokenType.LITERAL, "mixin")); b.rule(SCSS_INCLUDE_DIRECTIVE_LITERAL).is(SPACING, b.token(GenericTokenType.LITERAL, "include")); b.rule(SCSS_EXTEND_DIRECTIVE_LITERAL).is(SPACING, b.token(GenericTokenType.LITERAL, "extend"), SPACING); b.rule(SCSS_CONTENT_DIRECTIVE_LITERAL).is(SPACING, b.token(GenericTokenType.LITERAL, "content")); b.rule(SCSS_DEBUG_DIRECTIVE_LITERAL).is(SPACING, b.token(GenericTokenType.LITERAL, "debug")); b.rule(SCSS_WARN_DIRECTIVE_LITERAL).is(SPACING, b.token(GenericTokenType.LITERAL, "warn")); b.rule(SCSS_ERROR_DIRECTIVE_LITERAL).is(SPACING, b.token(GenericTokenType.LITERAL, "error")); b.rule(SCSS_IF_DIRECTIVE_LITERAL).is(SPACING, b.token(GenericTokenType.LITERAL, "if")); b.rule(SCSS_ELSE_DIRECTIVE_LITERAL).is(SPACING, b.token(GenericTokenType.LITERAL, "else")); b.rule(SCSS_ELSE_IF_DIRECTIVE_LITERAL).is(SPACING, b.token(GenericTokenType.LITERAL, b.sequence("else", SPACING, "if"))); b.rule(SCSS_FOR_DIRECTIVE_LITERAL).is(SPACING, b.token(GenericTokenType.LITERAL, "for")); b.rule(SCSS_EACH_DIRECTIVE_LITERAL).is(SPACING, b.token(GenericTokenType.LITERAL, "each")); b.rule(SCSS_WHILE_DIRECTIVE_LITERAL).is(SPACING, b.token(GenericTokenType.LITERAL, "while")); b.rule(SCSS_FUNCTION_DIRECTIVE_LITERAL).is(SPACING, b.token(GenericTokenType.LITERAL, "function")); b.rule(SCSS_RETURN_DIRECTIVE_LITERAL).is(SPACING, b.token(GenericTokenType.LITERAL, "return"));
private static void scss(LexerlessGrammarBuilder b) { b.rule(SCSS_DOLLAR).is(b.token(GenericTokenType.LITERAL, "$")); b.rule(SCSS_ELLIPSIS).is(b.token(GenericTokenType.LITERAL, "...")); b.rule(SCSS_PARENT_SELECTOR_KEYWORD).is("&"); b.rule(SCSS_DEFAULT_KEYWORD).is(SPACING, b.token(GenericTokenType.LITERAL, "!default")); b.rule(SCSS_GLOBAL_KEYWORD).is(SPACING, b.token(GenericTokenType.LITERAL, "!global")); b.rule(SCSS_OPTIONAL_KEYWORD).is(SPACING, b.token(GenericTokenType.LITERAL, "!optional")); b.rule(SCSS_MIXIN_DIRECTIVE_LITERAL).is(SPACING, b.token(GenericTokenType.LITERAL, "mixin")); b.rule(SCSS_INCLUDE_DIRECTIVE_LITERAL).is(SPACING, b.token(GenericTokenType.LITERAL, "include")); b.rule(SCSS_EXTEND_DIRECTIVE_LITERAL).is(SPACING, b.token(GenericTokenType.LITERAL, "extend"), SPACING); b.rule(SCSS_CONTENT_DIRECTIVE_LITERAL).is(SPACING, b.token(GenericTokenType.LITERAL, "content")); b.rule(SCSS_DEBUG_DIRECTIVE_LITERAL).is(SPACING, b.token(GenericTokenType.LITERAL, "debug")); b.rule(SCSS_WARN_DIRECTIVE_LITERAL).is(SPACING, b.token(GenericTokenType.LITERAL, "warn")); b.rule(SCSS_ERROR_DIRECTIVE_LITERAL).is(SPACING, b.token(GenericTokenType.LITERAL, "error")); b.rule(SCSS_IF_DIRECTIVE_LITERAL).is(SPACING, b.token(GenericTokenType.LITERAL, "if")); b.rule(SCSS_ELSE_DIRECTIVE_LITERAL).is(SPACING, b.token(GenericTokenType.LITERAL, "else")); b.rule(SCSS_ELSE_IF_DIRECTIVE_LITERAL).is(SPACING, b.token(GenericTokenType.LITERAL, b.sequence("else", SPACING, "if"))); b.rule(SCSS_FOR_DIRECTIVE_LITERAL).is(SPACING, b.token(GenericTokenType.LITERAL, "for")); b.rule(SCSS_EACH_DIRECTIVE_LITERAL).is(SPACING, b.token(GenericTokenType.LITERAL, "each")); b.rule(SCSS_WHILE_DIRECTIVE_LITERAL).is(SPACING, b.token(GenericTokenType.LITERAL, "while")); b.rule(SCSS_FUNCTION_DIRECTIVE_LITERAL).is(SPACING, b.token(GenericTokenType.LITERAL, "function")); b.rule(SCSS_RETURN_DIRECTIVE_LITERAL).is(SPACING, b.token(GenericTokenType.LITERAL, "return"));
private static void less(LexerlessGrammarBuilder b) { b.rule(LESS_VARIABLE_PREFIX).is(SPACING, b.token(GenericTokenType.LITERAL, "@")); b.token(GenericTokenType.LITERAL, b.sequence( b.optional(IDENT_IDENTIFIER_NO_WS), _NMCHAR))))); b.rule(LESS_IDENT_IDENTIFIER_NO_WS_NOR_WHEN).is(b.token(GenericTokenType.LITERAL, b.sequence( b.nextNot(b.regexp("when[\\s]+|when\\(")), b.rule(LESS_EXTEND_PREFIX).is(SPACING, b.token(GenericTokenType.LITERAL, b.sequence(":", LESS_EXTEND_KEYWORD))); b.rule(LESS_MIXIN_GUARD_WHEN).is(SPACING, b.token(GenericTokenType.LITERAL, "when")); b.rule(LESS_MIXIN_GUARD_NOT).is(SPACING, b.token(GenericTokenType.LITERAL, "not")); b.rule(LESS_MIXIN_GUARD_AND).is(SPACING, b.token(GenericTokenType.LITERAL, "and")); b.rule(LESS_MIXIN_GUARD_OR).is(SPACING, b.token(GenericTokenType.LITERAL, ",")); b.rule(LESS_MERGE).is(b.token(GenericTokenType.LITERAL, b.firstOf("+_", "+"))); b.rule(LESS_ESCAPING_SYMBOL).is(SPACING, b.token(GenericTokenType.LITERAL, "~")); b.token(GenericTokenType.LITERAL, b.firstOf( "+",
private static void less(LexerlessGrammarBuilder b) { b.rule(LESS_VARIABLE_PREFIX).is(SPACING, b.token(GenericTokenType.LITERAL, "@")); b.token(GenericTokenType.LITERAL, b.sequence( b.optional(IDENT_IDENTIFIER_NO_WS), _NMCHAR))))); b.rule(LESS_IDENT_IDENTIFIER_NO_WS_NOR_WHEN).is(b.token(GenericTokenType.LITERAL, b.sequence( b.nextNot(b.regexp("when[\\s]+|when\\(")), b.rule(LESS_EXTEND_PREFIX).is(SPACING, b.token(GenericTokenType.LITERAL, b.sequence(":", LESS_EXTEND_KEYWORD))); b.rule(LESS_MIXIN_GUARD_WHEN).is(SPACING, b.token(GenericTokenType.LITERAL, "when")); b.rule(LESS_MIXIN_GUARD_NOT).is(SPACING, b.token(GenericTokenType.LITERAL, "not")); b.rule(LESS_MIXIN_GUARD_AND).is(SPACING, b.token(GenericTokenType.LITERAL, "and")); b.rule(LESS_MIXIN_GUARD_OR).is(SPACING, b.token(GenericTokenType.LITERAL, ",")); b.rule(LESS_MERGE).is(b.token(GenericTokenType.LITERAL, b.firstOf("+_", "+"))); b.rule(LESS_ESCAPING_SYMBOL).is(SPACING, b.token(GenericTokenType.LITERAL, "~")); b.token(GenericTokenType.LITERAL, b.firstOf( "+",
private static void tokens(LexerlessGrammarBuilder b) { b.rule(IDENT_IDENTIFIER).is(SPACING, b.token(GenericTokenType.LITERAL, _IDENT)); b.rule(IDENT_IDENTIFIER_NO_WS).is(b.token(GenericTokenType.LITERAL, _IDENT)); b.rule(NUMBER_LITERAL).is(SPACING, b.token(GenericTokenType.LITERAL, _NUM)); b.rule(STRING_LITERAL_NO_WS).is(b.token(GenericTokenType.LITERAL, b.firstOf(_STRING1, _STRING2))); b.rule(UNICODE_RANGE_LITERAL).is(SPACING, b.token(GenericTokenType.LITERAL, b.regexp("u\\+[0-9a-f?]{1,6}(-[0-9a-f]{1,6})?"))); b.rule(VARIABLE_PREFIX).is(SPACING, b.token(GenericTokenType.LITERAL, "--")); b.rule(NAME).is(b.token(GenericTokenType.LITERAL, b.oneOrMore(_NMCHAR))); b.rule(PSEUDO_PREFIX).is(b.token(GenericTokenType.LITERAL, b.sequence(b.firstOf("::", ":"), b.nextNot("extend")))); b.rule(URI_CONTENT_LITERAL).is(b.token(GenericTokenType.LITERAL, b.sequence( _W, b.rule(UNIT_LITERAL).is(b.token(GenericTokenType.LITERAL, b.firstOf( matchCaseInsensitive(b, "em"), matchCaseInsensitive(b, "ex"), b.token(GenericTokenType.LITERAL, b.sequence( SPACING, b.rule(EOF).is(SPACING, b.token(GenericTokenType.EOF, b.endOfInput()));
public static LexerlessGrammarBuilder createGrammarBuilder() { LexerlessGrammarBuilder b = LexerlessGrammarBuilder.create(); punctuator(b, LCURLYBRACE, "{"); punctuator(b, RCURLYBRACE, "}"); punctuator(b, LBRACKET, "["); punctuator(b, RBRACKET, "]"); punctuator(b, COMMA, ","); punctuator(b, COLON, ":"); b.rule(TRUE).is("true", WHITESPACE); b.rule(FALSE).is("false", WHITESPACE); b.rule(NULL).is("null", WHITESPACE); b.rule(WHITESPACE).is(b.regexp("[ \n\r\t\f]*+")); b.rule(NUMBER).is(b.regexp("-?+(0|[1-9][0-9]*+)(\\.[0-9]++)?+([eE][+-]?+[0-9]++)?+"), WHITESPACE); b.rule(STRING).is(b.regexp("\"([^\"\\\\]|\\\\([\"\\\\/bfnrt]|u[0-9a-fA-F]{4}))*+\""), WHITESPACE); b.rule(EOF).is(b.token(GenericTokenType.EOF, b.endOfInput())); b.setRootRule(JSON); return b; }
private static void macros(LexerlessGrammarBuilder b) { b.rule(_IDENT).is(b.token(GenericTokenType.IDENTIFIER, b.sequence(_NMSTART, b.zeroOrMore(_NMCHAR)))).skip(); b.rule(_NAME).is(b.token(GenericTokenType.LITERAL, b.oneOrMore(_NMCHAR))).skip(); b.rule(_NMSTART).is( b.firstOf(b.regexp(NMSTART_REGEX), "-", "*", _NONASCII, _ESCAPE)).skip(); b.rule(_NMCHAR).is( b.firstOf(b.regexp(NMCHAR_REGEX), _NONASCII, _ESCAPE)).skip(); b.rule(_NUM).is(b.token(GenericTokenType.LITERAL, b.sequence(b.optional("-"), // '-' IS NOT DEFINED IN THE W3 spec b.firstOf(b.regexp("[0-9]*\\.[0-9]+"), b.regexp("[0-9]+"))))).skip(); b.rule(_STRING).is(b.token(GenericTokenType.LITERAL, b.firstOf(_STRING1, _STRING2))).skip(); b.rule(_STRING1).is( "\"",
/** * 3.10. Literals */ private static void literals(LexerlessGrammarBuilder b) { b.rule(SPACING).is( b.skippedTrivia(whitespace(b)), b.zeroOrMore( b.commentTrivia(b.firstOf(inlineComment(b), multilineComment(b))), b.skippedTrivia(whitespace(b)))); b.rule(EOF).is(b.token(GenericTokenType.EOF, b.endOfInput())); b.rule(CHARACTER_LITERAL).is(characterLiteral(b), SPACING); b.rule(JavaTokenType.LITERAL).is(stringLiteral(b), SPACING); b.rule(FLOAT_LITERAL).is(b.regexp(FLOATING_LITERAL_WITHOUT_SUFFIX_REGEXP + "[fF]|[0-9][0-9_]*+[fF]"), SPACING); b.rule(DOUBLE_LITERAL).is(b.regexp(FLOATING_LITERAL_WITHOUT_SUFFIX_REGEXP + "[dD]?+|[0-9][0-9_]*+[dD]"), SPACING); b.rule(LONG_LITERAL).is(b.regexp(INTEGER_LITERAL_REGEXP + "[lL]"), SPACING); b.rule(INTEGER_LITERAL).is(b.regexp(INTEGER_LITERAL_REGEXP), SPACING); b.rule(JavaTokenType.IDENTIFIER).is( b.firstOf( b.next(ENUM), b.nextNot(KEYWORD)), javaIdentifier(b), SPACING); }
/** * 3.10. Literals */ private static void literals(LexerlessGrammarBuilder b) { b.rule(SPACING).is( b.skippedTrivia(whitespace(b)), b.zeroOrMore( b.commentTrivia(b.firstOf(inlineComment(b), multilineComment(b))), b.skippedTrivia(whitespace(b)))); b.rule(EOF).is(b.token(GenericTokenType.EOF, b.endOfInput())); b.rule(CHARACTER_LITERAL).is(characterLiteral(b), SPACING); b.rule(JavaTokenType.STRING_LITERAL).is(stringLiteral(b), SPACING); b.rule(FLOAT_LITERAL).is(b.regexp(FLOATING_LITERAL_WITHOUT_SUFFIX_REGEXP + "[fF]|[0-9][0-9_]*+[fF]"), SPACING); b.rule(DOUBLE_LITERAL).is(b.regexp(FLOATING_LITERAL_WITHOUT_SUFFIX_REGEXP + "[dD]?+|[0-9][0-9_]*+[dD]"), SPACING); b.rule(LONG_LITERAL).is(b.regexp(INTEGER_LITERAL_REGEXP + "[lL]"), SPACING); b.rule(INTEGER_LITERAL).is(b.regexp(INTEGER_LITERAL_REGEXP), SPACING); b.rule(JavaTokenType.IDENTIFIER).is( b.firstOf( b.next(ENUM), b.nextNot(KEYWORD)), javaIdentifier(b), SPACING); }
/** * 3.10. Literals */ private static void literals(LexerlessGrammarBuilder b) { b.rule(SPACING).is( b.skippedTrivia(whitespace(b)), b.zeroOrMore( b.commentTrivia(b.firstOf(inlineComment(b), multilineComment(b))), b.skippedTrivia(whitespace(b)))); b.rule(EOF).is(b.token(GenericTokenType.EOF, b.endOfInput())); b.rule(CHARACTER_LITERAL).is(characterLiteral(b), SPACING); b.rule(JavaTokenType.LITERAL).is(stringLiteral(b), SPACING); b.rule(FLOAT_LITERAL).is(b.regexp(FLOATING_LITERAL_WITHOUT_SUFFIX_REGEXP + "[fF]|[0-9][0-9_]*+[fF]"), SPACING); b.rule(DOUBLE_LITERAL).is(b.regexp(FLOATING_LITERAL_WITHOUT_SUFFIX_REGEXP + "[dD]?+|[0-9][0-9_]*+[dD]"), SPACING); b.rule(LONG_LITERAL).is(b.regexp(INTEGER_LITERAL_REGEXP + "[lL]"), SPACING); b.rule(INTEGER_LITERAL).is(b.regexp(INTEGER_LITERAL_REGEXP), SPACING); b.rule(JavaTokenType.IDENTIFIER).is( b.firstOf( b.next(ENUM), b.nextNot(KEYWORD)), javaIdentifier(b), SPACING); }
/** * 3.10. Literals */ private static void literals(LexerlessGrammarBuilder b) { b.rule(SPACING).is( b.skippedTrivia(whitespace(b)), b.zeroOrMore( b.commentTrivia(b.firstOf(inlineComment(b), multilineComment(b))), b.skippedTrivia(whitespace(b)))); b.rule(EOF).is(b.token(GenericTokenType.EOF, b.endOfInput())); b.rule(CHARACTER_LITERAL).is(characterLiteral(b), SPACING); b.rule(JavaTokenType.STRING_LITERAL).is(stringLiteral(b), SPACING); b.rule(FLOAT_LITERAL).is(b.regexp(FLOATING_LITERAL_WITHOUT_SUFFIX_REGEXP + "[fF]|[0-9][0-9_]*+[fF]"), SPACING); b.rule(DOUBLE_LITERAL).is(b.regexp(FLOATING_LITERAL_WITHOUT_SUFFIX_REGEXP + "[dD]?+|[0-9][0-9_]*+[dD]"), SPACING); b.rule(LONG_LITERAL).is(b.regexp(INTEGER_LITERAL_REGEXP + "[lL]"), SPACING); b.rule(INTEGER_LITERAL).is(b.regexp(INTEGER_LITERAL_REGEXP), SPACING); b.rule(JavaTokenType.IDENTIFIER).is( b.firstOf( b.next(ENUM), b.nextNot(KEYWORD)), javaIdentifier(b), SPACING); }
b.rule(eof).is(b.token(GenericTokenType.EOF, b.endOfInput())).skip();
b.sequence(SPACING_NO_LB, b.endOfInput()))); b.rule(EOF).is(b.token(GenericTokenType.EOF, b.endOfInput())).skip(); b.rule(IDENTIFIER).is( SPACING, b.rule(STRING_LITERAL).is( SPACING, b.token(GenericTokenType.LITERAL, b.regexp(EcmaScriptLexer.LITERAL))); b.rule(REGULAR_EXPRESSION_LITERAL).is( SPACING,
DIRECTIVES), SPACING, b.token(GenericTokenType.EOF, b.endOfInput()));
IMPORTANT, addSpacing(DELIM, b))).skipIfOneChild(); b.rule(eof).is(b.token(GenericTokenType.EOF, b.endOfInput())).skip();
b.rule(ANYTHING_BUT_START_TAG).is(SPACING, b.regexp(LexicalConstant.ANYTHING_BUT_START_TAG)).skip(); b.rule(EOF).is(b.token(GenericTokenType.EOF, b.endOfInput())).skip();
b.rule(ANYTHING_BUT_START_TAG).is(SPACING, b.regexp(LexicalConstant.ANYTHING_BUT_START_TAG)).skip(); b.rule(EOF).is(b.token(GenericTokenType.EOF, b.endOfInput())).skip();