public TokenStream reusableTokenStream(String fieldName, Reader reader) throws IOException { SavedStreams streams = (SavedStreams) getPreviousTokenStream(); if (streams == null) { streams = new SavedStreams(); streams.source = new LowerCaseTokenizer(reader); streams.result = new StopFilter(streams.source, stopWords); setPreviousTokenStream(streams); } else streams.source.reset(reader); return streams.result; } }
public Analyzer buildAnalyzer(String analyzerName, CompassSettings settings, DefaultLuceneAnalyzerFactory analyzerFactory) throws SearchEngineException { String analyzerSetting = settings.getSetting(LuceneEnvironment.Analyzer.TYPE, LuceneEnvironment.Analyzer.CoreTypes.STANDARD); Analyzer analyzer = null; if (LuceneEnvironment.Analyzer.CoreTypes.WHITESPACE.equalsIgnoreCase(analyzerSetting)) { analyzer = new WhitespaceAnalyzer(); } else if (LuceneEnvironment.Analyzer.CoreTypes.STANDARD.equalsIgnoreCase(analyzerSetting)) { analyzer = new StandardAnalyzer(analyzerFactory.parseStopWords(analyzerName, settings, StandardAnalyzer.STOP_WORDS)); } else if (LuceneEnvironment.Analyzer.CoreTypes.SIMPLE.equalsIgnoreCase(analyzerSetting)) { analyzer = new SimpleAnalyzer(); } else if (LuceneEnvironment.Analyzer.CoreTypes.STOP.equalsIgnoreCase(analyzerSetting)) { analyzer = new StopAnalyzer(analyzerFactory.parseStopWords(analyzerName, settings, StopAnalyzer.ENGLISH_STOP_WORDS)); } else if (LuceneEnvironment.Analyzer.CoreTypes.KEYWORD.equalsIgnoreCase(analyzerSetting)) { analyzer = new KeywordAnalyzer(); } return analyzer; } }
public TokenStream reusableTokenStream(String fieldName, Reader reader) throws IOException { SavedStreams streams = (SavedStreams) getPreviousTokenStream(); if (streams == null) { streams = new SavedStreams(); streams.source = new LowerCaseTokenizer(reader); streams.result = new StopFilter(streams.source, stopWords); setPreviousTokenStream(streams); } else streams.source.reset(reader); return streams.result; } }