@Override public final void tokenize(SourceCode source, Tokens cpdTokens) { String fileName = source.getFileName(); ChannelDispatcher.Builder lexerBuilder = ChannelDispatcher.builder(); lexerBuilder.addChannel(CommentChannel.JSP_COMMENT); lexerBuilder.addChannel(CommentChannel.HTML_COMMENT); lexerBuilder.addChannel(CommentChannel.C_COMMENT); lexerBuilder.addChannel(CommentChannel.CPP_COMMENT); lexerBuilder.addChannel(new WordChannel(fileName)); lexerBuilder.addChannel(new LiteralChannel(fileName)); lexerBuilder.addChannel(new BlackHoleChannel()); ChannelDispatcher<Tokens> lexer = lexerBuilder.build(); try { lexer.consume(new CodeReader(new FileReader(new File(fileName))), cpdTokens); cpdTokens.add(TokenEntry.getEOF()); } catch (FileNotFoundException e) { LOG.error("Unable to open file : " + fileName, e); } }
/** * Defines that sequence of characters, which is matched specified regular expression, is a token with specified value. */ public Builder token(String regularExpression, String normalizationValue) { channelDispatcherBuilder.addChannel(new TokenChannel(regularExpression, normalizationValue)); return this; }
public Builder addChannels(Channel... c) { for (Channel channel : c) { addChannel(channel); } return this; }
/** * Defines that sequence of characters, which is matched specified regular expression, is a token. */ public Builder token(String regularExpression) { channelDispatcherBuilder.addChannel(new TokenChannel(regularExpression)); return this; }
/** * Defines that sequence of characters must be ignored, if it matches specified regular expression. */ public Builder ignore(String regularExpression) { channelDispatcherBuilder.addChannel(new BlackHoleTokenChannel(regularExpression)); return this; }
private Markdown() { dispatcher = ChannelDispatcher.builder() .addChannel(new HtmlLinkChannel()) .addChannel(new HtmlUrlChannel()) .addChannel(new HtmlEndOfLineChannel()) .addChannel(new HtmlEmphasisChannel()) .addChannel(new HtmlListChannel()) .addChannel(new HtmlBlockquoteChannel()) .addChannel(new HtmlHeadingChannel()) .addChannel(new HtmlCodeChannel()) .addChannel(new HtmlMultilineCodeChannel()) .addChannel(new IdentifierAndNumberChannel()) .addChannel(new BlackholeChannel()) .build(); }
/** * Defines that sequence of characters, which is matched specified regular expression, is a token. */ public Builder token(String regularExpression) { channelDispatcherBuilder.addChannel(new TokenChannel(regularExpression)); return this; }
/** * Defines that sequence of characters, which is matched specified regular expression, is a token with specified value. */ public Builder token(String regularExpression, String normalizationValue) { channelDispatcherBuilder.addChannel(new TokenChannel(regularExpression, normalizationValue)); return this; }
/** * Defines that sequence of characters must be ignored, if it matches specified regular expression. */ public Builder ignore(String regularExpression) { channelDispatcherBuilder.addChannel(new BlackHoleTokenChannel(regularExpression)); return this; }