@Override protected TokenStreamComponents createComponents( String fieldName ) { KeywordTokenizer source = new KeywordTokenizer(); return new TokenStreamComponents( source, new LowerCaseFilter( source ) ); }
@Override protected TokenStreamComponents createComponents( String fieldName ) { Tokenizer source = new WhitespaceTokenizer(); TokenStream filter = new LowerCaseFilter( source ); return new TokenStreamComponents( source, filter ); }
@Override protected TokenStreamComponents createComponents( String fieldName ) { called = true; Tokenizer source = new WhitespaceTokenizer(); return new TokenStreamComponents( source, new LowerCaseFilter( source ) ); } }
/** * Creates a the TokenStreamComponents used to analyze the stream. * * @param fieldName the field that this lucene analyzer will process * @return the token stream filter chain */ @Override protected TokenStreamComponents createComponents(String fieldName) { //final Tokenizer source = new AlphaNumericTokenizer(); final Tokenizer source = new WhitespaceTokenizer(); TokenStream stream = source; stream = new UrlTokenizingFilter(stream); stream = new AlphaNumericFilter(stream); stream = new WordDelimiterGraphFilter(stream, WordDelimiterGraphFilter.GENERATE_WORD_PARTS | WordDelimiterGraphFilter.GENERATE_NUMBER_PARTS | WordDelimiterGraphFilter.PRESERVE_ORIGINAL | WordDelimiterGraphFilter.SPLIT_ON_CASE_CHANGE | WordDelimiterGraphFilter.SPLIT_ON_NUMERICS | WordDelimiterGraphFilter.STEM_ENGLISH_POSSESSIVE, null); stream = new LowerCaseFilter(stream); stream = new StopFilter(stream, stopWords); concatenatingFilter = new TokenPairConcatenatingFilter(stream); return new TokenStreamComponents(source, concatenatingFilter); }
@Override public LowerCaseFilter create(TokenStream input) { return new LowerCaseFilter(input); }
@Override protected TokenStreamComponents createComponents( String fieldName ) { Tokenizer source = new WhitespaceTokenizer(); TokenStream filter = new LowerCaseFilter( source ); return new TokenStreamComponents( source, filter ); }
@Override protected Analyzer.TokenStreamComponents createComponents(final String fieldName) { Tokenizer source = new WhitespaceTokenizer(); TokenStream filter = new LowerCaseFilter( source ); return new TokenStreamComponents( source, filter ); }
@Override protected TokenStreamComponents createComponents(String fieldName) { Tokenizer tokenizer = factory == null ? new WhitespaceTokenizer() : factory.create(); TokenStream stream = ignoreCase ? new LowerCaseFilter(tokenizer) : tokenizer; return new TokenStreamComponents(tokenizer, stream); } };
@Override protected TokenStreamComponents createComponents(String fieldName) { KeywordTokenizer source = new KeywordTokenizer(); LowerCaseFilter filter = new LowerCaseFilter(source); return new TokenStreamComponents(source, filter); }
@Override protected TokenStreamComponents createComponents(final String field) { //Use default grouping final Tokenizer tokenizer = new PatternTokenizer(pattern,-1); final TokenStream filter = new LowerCaseFilter(tokenizer); return new TokenStreamComponents(tokenizer, filter); } }
@Override protected TokenStreamComponents createComponents(String fieldName) { WhitespaceTokenizer src = new WhitespaceTokenizer(); TokenStream tok = new LowerCaseFilter(src); return new TokenStreamComponents(src, tok); } }
@Override protected TokenStreamComponents createComponents(String fieldName) { Tokenizer tokenizer = factory == null ? new WhitespaceTokenizer() : factory.create(); TokenStream stream = ignoreCase ? new LowerCaseFilter(tokenizer) : tokenizer; return new TokenStreamComponents(tokenizer, stream); } };
/** {@inheritDoc} */ @Override protected Analyzer.TokenStreamComponents createComponents(String fieldName) { final Tokenizer source = new StandardTokenizer(); TokenStream result = new StandardFilter(source); result = new LowerCaseFilter(result); result = new StopFilter(result, stopwords); result = new SnowballFilter(result, language); return new TokenStreamComponents(source, result); } }
@Override protected TokenStreamComponents createComponents(final String fieldName) { final Tokenizer source = new KeywordTokenizer(); TokenStream result = new StandardFilter(source); result = new CharacterFilter(result); result = new ASCIIFoldingFilter(result); result = new LowerCaseFilter(result); // result = new WordDelimiterFilter(result, WordDelimiterFilter.DIGIT, null); return new TokenStreamComponents(source, result); }
@Override protected TokenStreamComponents createComponents(String fieldName, Reader reader) { final Tokenizer source = new CharacterShingleTokenizer(reader); TokenStream result = new StandardFilter(matchVersion, source); result = new ASCIIFoldingFilter(result); result = new LowerCaseFilter(matchVersion, result); result = new ShingleFilter(result, 3); // result = new WordDelimiterFilter(result, WordDelimiterFilter.DIGIT, null); return new TokenStreamComponents(source, result); }
@Override protected TokenStreamComponents createComponents(String fieldName) { final Tokenizer source = new CharacterShingleTokenizer(); TokenStream result = new StandardFilter(source); result = new ASCIIFoldingFilter(result); result = new LowerCaseFilter(result); result = new ShingleFilter(result, 3); // result = new WordDelimiterFilter(result, WordDelimiterFilter.DIGIT, null); return new TokenStreamComponents(source, result); }
@Override protected TokenStreamComponents createComponents(final String fieldName) { final Tokenizer source = new KeywordTokenizer(); TokenStream result = new StandardFilter(source); result = new CharacterFilter(result); result = new ASCIIFoldingFilter(result); result = new LowerCaseFilter(result); // result = new WordDelimiterFilter(result, WordDelimiterFilter.DIGIT, null); return new TokenStreamComponents(source, result); }
@Override protected TokenStreamComponents createComponents(final String fieldName, final Reader reader) { final WhitespaceTokenizer source = new WhitespaceTokenizer(matchVersion, reader); TokenStream sink = new URIDecodingFilter(source, "UTF-8"); sink = this.applyURINormalisation(sink); sink = new MailtoFilter(sink); sink = new LowerCaseFilter(matchVersion, sink ); sink = new StopFilter(matchVersion, sink, stopSet); sink = new LengthFilter(true, sink, 2, 256); return new TokenStreamComponents(source, sink); }
@Override protected TokenStreamComponents createComponents(String fieldName) { final Tokenizer source = new StandardTokenizer(); TokenStream result = new StandardFilter(source); result = new ASCIIFoldingFilter(result); result = new EnglishPossessiveFilter(result); result = new WordDelimiterFilter(result,WordDelimiterFilter.ALPHA,null); result = new LowerCaseFilter(result); result = new StopFilter(result, EnglishAnalyzer.getDefaultStopSet()); result = new PorterStemFilter(result); return new TokenStreamComponents(source, result); }
@Override protected TokenStreamComponents createComponents(String fieldName) { final Tokenizer source = new StandardTokenizer(); TokenStream result = new StandardFilter(source); result = new ASCIIFoldingFilter(result); result = new EnglishPossessiveFilter(result); result = new WordDelimiterFilter(result,WordDelimiterFilter.ALPHA,null); result = new LowerCaseFilter(result); result = new StopFilter(result, EnglishAnalyzer.getDefaultStopSet()); result = new PorterStemFilter(result); return new TokenStreamComponents(source, result); }