private static TokenStream loadTokenizer(TokenizerFactory tokFactory, Reader reader){ return tokFactory.create( reader ); }
@Override public TokenStreamInfo getStream(String fieldName, Reader reader) { Tokenizer tk = (Tokenizer)tokenizer.create(charStream(reader)); TokenStream ts = tk; for (int i=0; i<filters.length; i++) { ts = filters[i].create(ts); } return new TokenStreamInfo(tk,ts); }
TokenStream tokenStream = tfac.create(tokenizerChain.charStream(new StringReader(value))); List<Token> tokens = analyzeTokenStream(tokenStream);