@Override public DecimalDigitFilter create(TokenStream input) { return new DecimalDigitFilter(input); }
@Override protected TokenStream normalize(String fieldName, TokenStream in) { TokenStream result = new LowerCaseFilter(in); result = new DecimalDigitFilter(result); return result; } }
@Override protected TokenStream normalize(String fieldName, TokenStream in) { TokenStream result = new SoraniNormalizationFilter(in); result = new LowerCaseFilter(result); result = new DecimalDigitFilter(result); return result; } }
@Override protected TokenStream normalize(String fieldName, TokenStream in) { TokenStream result = new LowerCaseFilter(in); result = new DecimalDigitFilter(result); result = new ArabicNormalizationFilter(result); return result; } }
@Override protected TokenStream normalize(String fieldName, TokenStream in) { TokenStream result = new LowerCaseFilter(in); result = new DecimalDigitFilter(result); result = new IndicNormalizationFilter(result); result = new HindiNormalizationFilter(result); return result; } }
@Override protected TokenStream normalize(String fieldName, TokenStream in) { TokenStream result = new LowerCaseFilter(in); result = new DecimalDigitFilter(result); result = new ArabicNormalizationFilter(result); /* additional persian-specific normalization */ result = new PersianNormalizationFilter(result); return result; }
@Override protected TokenStream normalize(String fieldName, TokenStream in) { TokenStream result = new LowerCaseFilter(in); result = new DecimalDigitFilter(result); result = new IndicNormalizationFilter(result); result = new BengaliNormalizationFilter(result); return result; } }
/** * Creates * {@link org.apache.lucene.analysis.Analyzer.TokenStreamComponents} * used to tokenize all the text in the provided {@link Reader}. * * @return {@link org.apache.lucene.analysis.Analyzer.TokenStreamComponents} * built from a {@link ThaiTokenizer} filtered with * {@link LowerCaseFilter}, {@link DecimalDigitFilter} and {@link StopFilter} */ @Override protected TokenStreamComponents createComponents(String fieldName) { final Tokenizer source = new ThaiTokenizer(); TokenStream result = new LowerCaseFilter(source); result = new DecimalDigitFilter(result); result = new StopFilter(result, stopwords); return new TokenStreamComponents(source, result); }
@Override public TokenStream create(TokenStream tokenStream) { return new DecimalDigitFilter(tokenStream); }
@Override public TokenStream create(TokenStream tokenStream, Version version) { return new DecimalDigitFilter(tokenStream); } @Override
/** * Creates * {@link org.apache.lucene.analysis.Analyzer.TokenStreamComponents} * used to tokenize all the text in the provided {@link Reader}. * * @return {@link org.apache.lucene.analysis.Analyzer.TokenStreamComponents} * built from a {@link StandardTokenizer} filtered with * {@link LowerCaseFilter}, {@link DecimalDigitFilter}, {@link ArabicNormalizationFilter}, * {@link PersianNormalizationFilter} and Persian Stop words */ @Override protected TokenStreamComponents createComponents(String fieldName) { final Tokenizer source = new StandardTokenizer(); TokenStream result = new LowerCaseFilter(source); result = new DecimalDigitFilter(result); result = new ArabicNormalizationFilter(result); /* additional persian-specific normalization */ result = new PersianNormalizationFilter(result); /* * the order here is important: the stopword list is normalized with the * above! */ return new TokenStreamComponents(source, new StopFilter(result, stopwords)); }
@Override public TokenStream create(TokenStream tokenStream) { return new DecimalDigitFilter(tokenStream); }
@Override public DecimalDigitFilter create(TokenStream input) { return new DecimalDigitFilter(input); }
/** * Creates a * {@link org.apache.lucene.analysis.Analyzer.TokenStreamComponents} * which tokenizes all the text in the provided {@link Reader}. * * @return A * {@link org.apache.lucene.analysis.Analyzer.TokenStreamComponents} * built from an {@link StandardTokenizer} filtered with * {@link SoraniNormalizationFilter}, * {@link LowerCaseFilter}, {@link DecimalDigitFilter}, {@link StopFilter} * , {@link SetKeywordMarkerFilter} if a stem exclusion set is * provided and {@link SoraniStemFilter}. */ @Override protected TokenStreamComponents createComponents(String fieldName) { final Tokenizer source = new StandardTokenizer(); TokenStream result = new SoraniNormalizationFilter(source); result = new LowerCaseFilter(result); result = new DecimalDigitFilter(result); result = new StopFilter(result, stopwords); if(!stemExclusionSet.isEmpty()) result = new SetKeywordMarkerFilter(result, stemExclusionSet); result = new SoraniStemFilter(result); return new TokenStreamComponents(source, result); }
/** * Creates * {@link org.apache.lucene.analysis.Analyzer.TokenStreamComponents} * used to tokenize all the text in the provided {@link Reader}. * * @return {@link org.apache.lucene.analysis.Analyzer.TokenStreamComponents} * built from a {@link StandardTokenizer} filtered with * {@link LowerCaseFilter}, {@link DecimalDigitFilter}, {@link IndicNormalizationFilter}, * {@link BengaliNormalizationFilter}, {@link SetKeywordMarkerFilter} * if a stem exclusion set is provided, {@link BengaliStemFilter}, and * Bengali Stop words */ @Override protected TokenStreamComponents createComponents(String fieldName) { final Tokenizer source = new StandardTokenizer(); TokenStream result = new LowerCaseFilter(source); result = new DecimalDigitFilter(result); if (!stemExclusionSet.isEmpty()) result = new SetKeywordMarkerFilter(result, stemExclusionSet); result = new IndicNormalizationFilter(result); result = new BengaliNormalizationFilter(result); result = new StopFilter(result, stopwords); result = new BengaliStemFilter(result); return new TokenStreamComponents(source, result); }
/** * Creates * {@link org.apache.lucene.analysis.Analyzer.TokenStreamComponents} * used to tokenize all the text in the provided {@link Reader}. * * @return {@link org.apache.lucene.analysis.Analyzer.TokenStreamComponents} * built from a {@link StandardTokenizer} filtered with * {@link LowerCaseFilter}, {@link DecimalDigitFilter}, {@link IndicNormalizationFilter}, * {@link HindiNormalizationFilter}, {@link SetKeywordMarkerFilter} * if a stem exclusion set is provided, {@link HindiStemFilter}, and * Hindi Stop words */ @Override protected TokenStreamComponents createComponents(String fieldName) { final Tokenizer source = new StandardTokenizer(); TokenStream result = new LowerCaseFilter(source); result = new DecimalDigitFilter(result); if (!stemExclusionSet.isEmpty()) result = new SetKeywordMarkerFilter(result, stemExclusionSet); result = new IndicNormalizationFilter(result); result = new HindiNormalizationFilter(result); result = new StopFilter(result, stopwords); result = new HindiStemFilter(result); return new TokenStreamComponents(source, result); }
/** * Creates * {@link org.apache.lucene.analysis.Analyzer.TokenStreamComponents} * used to tokenize all the text in the provided {@link Reader}. * * @return {@link org.apache.lucene.analysis.Analyzer.TokenStreamComponents} * built from an {@link StandardTokenizer} filtered with * {@link LowerCaseFilter}, {@link DecimalDigitFilter}, {@link StopFilter}, * {@link ArabicNormalizationFilter}, {@link SetKeywordMarkerFilter} * if a stem exclusion set is provided and {@link ArabicStemFilter}. */ @Override protected TokenStreamComponents createComponents(String fieldName) { final Tokenizer source = new StandardTokenizer(); TokenStream result = new LowerCaseFilter(source); result = new DecimalDigitFilter(result); // the order here is important: the stopword list is not normalized! result = new StopFilter(result, stopwords); // TODO maybe we should make ArabicNormalization filter also KeywordAttribute aware?! result = new ArabicNormalizationFilter(result); if(!stemExclusionSet.isEmpty()) { result = new SetKeywordMarkerFilter(result, stemExclusionSet); } return new TokenStreamComponents(source, new ArabicStemFilter(result)); }
static public TokenStream arabic(TokenStream result) { result = new LowerCaseFilter(result); result = new DecimalDigitFilter(result); result = new ArabicNormalizationFilter(result); return result; }
static public TokenStream hindi(TokenStream result) { result = new LowerCaseFilter(result); result = new DecimalDigitFilter(result); result = new IndicNormalizationFilter(result); result = new HindiNormalizationFilter(result); result = new HindiStemFilter(result); return result; }
@Override protected TokenStreamComponents createComponents(String s) { final Tokenizer source = new StandardTokenizer(); TokenStream result = new PersianStopFilter(source, PERSIAN_STOP_WORDS_SET); result = new DecimalDigitFilter(result); result = new PersianNormalizationFilter(result); result = new PersianStemFilter(result); return new TokenStreamComponents(source, result); }