@Override public ThaiWordFilter create(TokenStream input) { return new ThaiWordFilter(input); } }
public ThaiWordFilter create(TokenStream input) { return new ThaiWordFilter(input); } }
@Override public TokenStream tokenStream(String fieldName, Reader reader) { TokenStream ts = new StandardTokenizer(matchVersion, reader); ts = new ThaiWordFilter(ts); if (doStopWords && stopSet != null) { ts = new StopFilter(false, ts, stopSet); } return ts; }
/** * Creates a TokenStream which tokenizes all the text in the provided Reader. * * @return A TokenStream build from a ClassicTokenizer and appropriate filters for the * language. See */ @Override public final TokenStream tokenStream(final String fieldName, final Reader reader) { TokenStream result = new ClassicTokenizer(matchVersion, reader); result = new StandardFilter(matchVersion, result); result = wrapStreamForIndexing(result); result = new LowerCaseFilter(matchVersion, result); result = stopWordFilter.apply(result); result = new ThaiWordFilter(matchVersion, result); return result; } }
/** * Creates * {@link org.apache.lucene.analysis.ReusableAnalyzerBase.TokenStreamComponents} * used to tokenize all the text in the provided {@link Reader}. * * @return {@link org.apache.lucene.analysis.ReusableAnalyzerBase.TokenStreamComponents} * built from a {@link StandardTokenizer} filtered with * {@link StandardFilter}, {@link LowerCaseFilter}, {@link ThaiWordFilter}, and * {@link StopFilter} */ @Override protected TokenStreamComponents createComponents(String fieldName, Reader reader) { final Tokenizer source = new StandardTokenizer(matchVersion, reader); TokenStream result = new StandardFilter(matchVersion, source); if (matchVersion.onOrAfter(Version.LUCENE_31)) result = new LowerCaseFilter(matchVersion, result); result = new ThaiWordFilter(matchVersion, result); return new TokenStreamComponents(source, new StopFilter(matchVersion, result, stopwords)); } }
@Override public TokenStream reusableTokenStream(String fieldName, Reader reader) throws IOException { SavedStreams streams = (SavedStreams) getPreviousTokenStream(); if (streams == null) { streams = new SavedStreams(new StandardTokenizer(matchVersion, reader)); streams.setResult(new ThaiWordFilter(streams.getResult())); if (doStopWords && stopSet != null) { streams.setResult(new StopFilter(StopFilter.getEnablePositionIncrementsVersionDefault(matchVersion), streams.getResult(), stopSet)); } setPreviousTokenStream(streams); } else { streams.getSource().reset(reader); streams.getResult().reset(); // reset the ThaiWordFilter's state } return streams.getResult(); }
result = new ThaiWordFilter(result); return new TokenStreamComponents(source, new StopFilter(result, stopwords));