public static Lexer create(Charset charset) { return Lexer.builder() .withCharset(charset) .withFailIfNoChannelToConsumeOneCharacter(true) .withChannel(new BomCharacterChannel()) .withChannel(new BlackHoleChannel("\\s++")) .withChannel(regexp(HtmlTokenType.DOCTYPE, "<!DOCTYPE.*>")) .withChannel(regexp(HtmlTokenType.TAG, "</?[:\\w]+>?")) .withChannel(regexp(HtmlTokenType.TAG, "/?>")) // JSP comment .withChannel(commentRegexp("<%--[\\w\\W]*?%>")) // HTML comment .withChannel(commentRegexp("<!--[\\w\\W]*?-->")) // C comment .withChannel(commentRegexp("/\\*[\\w\\W]*?\\*/")) // CPP comment .withChannel(commentRegexp("//[^\n\r]*")) .withChannel(regexp(HtmlTokenType.EXPRESSION, "<%[\\w\\W]*?%>")) .withChannel(regexp(HtmlTokenType.ATTRIBUTE, "=[\"']{1}[\\w\\W]*?[\"']{1}")) .withChannel(regexp(HtmlTokenType.ATTRIBUTE, "=[^\\s'\"=<>`]++")) .withChannel(new IdentifierAndKeywordChannel("\\w++", true, new TokenType[]{})) .withChannel(new UnknownCharacterChannel()) .build(); }
public static Lexer create(FlexConfiguration conf) { return Lexer.builder() .withCharset(conf.getCharset()) .withFailIfNoChannelToConsumeOneCharacter(true) .withChannel(new BomCharacterChannel()) .withChannel(new BlackHoleChannel("\\s++")) // Comments .withChannel(commentRegexp("//[^\\n\\r]*+")) .withChannel(commentRegexp("/\\*[\\s\\S]*?\\*/")) // String Literals .withChannel(regexp(GenericTokenType.LITERAL, "\"([^\"\\\\]*+(\\\\[\\s\\S])?+)*+\"")) .withChannel(regexp(GenericTokenType.LITERAL, "\'([^\'\\\\]*+(\\\\[\\s\\S])?+)*+\'")) // Regular Expression Literal .withChannel(new FlexRegularExpressionLiteralChannel()) // Numbers .withChannel(regexp(FlexTokenType.NUMERIC_LITERAL, "0[xX][0-9a-fA-F]++")) .withChannel(regexp(FlexTokenType.NUMERIC_LITERAL, "[0-9]++\\.([0-9]++)?+" + EXP + "?+")) .withChannel(regexp(FlexTokenType.NUMERIC_LITERAL, "\\.[0-9]++" + EXP + "?+")) .withChannel(regexp(FlexTokenType.NUMERIC_LITERAL, "[0-9]++" + EXP + "?+")) .withChannel(new IdentifierAndKeywordChannel("\\p{javaJavaIdentifierStart}++\\p{javaJavaIdentifierPart}*+", true, FlexKeyword.values())) .withChannel(new PunctuatorChannel(FlexPunctuator.values())) .withChannel(new UnknownCharacterChannel()) .build(); }
.withChannel(new BomCharacterChannel()) .withChannel(new UnknownCharacterChannel());