@Override public Analyzer createAnalyzer() { return new WhitespaceAnalyzer(); }
@Override public Analyzer createAnalyzer() { return new KeywordAnalyzer(); }
@Override public Analyzer createAnalyzer() { return new SimpleAnalyzer(); }
/** * Creates a the TokenStreamComponents used to analyze the stream. * * @param fieldName the field that this lucene analyzer will process * @return the token stream filter chain */ @Override protected TokenStreamComponents createComponents(String fieldName) { //final Tokenizer source = new AlphaNumericTokenizer(); final Tokenizer source = new WhitespaceTokenizer(); TokenStream stream = source; stream = new UrlTokenizingFilter(stream); stream = new AlphaNumericFilter(stream); stream = new WordDelimiterGraphFilter(stream, WordDelimiterGraphFilter.GENERATE_WORD_PARTS | WordDelimiterGraphFilter.GENERATE_NUMBER_PARTS | WordDelimiterGraphFilter.PRESERVE_ORIGINAL | WordDelimiterGraphFilter.SPLIT_ON_CASE_CHANGE | WordDelimiterGraphFilter.SPLIT_ON_NUMERICS | WordDelimiterGraphFilter.STEM_ENGLISH_POSSESSIVE, null); stream = new LowerCaseFilter(stream); stream = new StopFilter(stream, stopWords); concatenatingFilter = new TokenPairConcatenatingFilter(stream); return new TokenStreamComponents(source, concatenatingFilter); }
@Override protected TokenStreamComponents createComponents( String fieldName ) { Tokenizer source = new WhitespaceTokenizer(); TokenStream filter = new LowerCaseFilter( source ); return new TokenStreamComponents( source, filter ); }
@Override protected TokenStreamComponents createComponents( String fieldName ) { KeywordTokenizer source = new KeywordTokenizer(); return new TokenStreamComponents( source, new LowerCaseFilter( source ) ); }
@Override public Analyzer createAnalyzer() { return new StopAnalyzer(); }
/** * Returns the set of stop words being used. * * @return the set of stop words being used */ public static CharArraySet getStopWords() { final CharArraySet words = StopFilter.makeStopSet(ADDITIONAL_STOP_WORDS, true); words.addAll(StopAnalyzer.ENGLISH_STOP_WORDS_SET); return words; }
@Override public Analyzer createAnalyzer() { return new UnicodeWhitespaceAnalyzer(); }
@Override protected TokenStreamComponents createComponents( String fieldName ) { called = true; Tokenizer source = new WhitespaceTokenizer(); return new TokenStreamComponents( source, new LowerCaseFilter( source ) ); } }
/** * Creates an Analyzer for searching the CPE Index. * * @return the CPE Analyzer. */ private Analyzer createSearchingAnalyzer() { final Map<String, Analyzer> fieldAnalyzers = new HashMap<>(); fieldAnalyzers.put(Fields.DOCUMENT_KEY, new KeywordAnalyzer()); productFieldAnalyzer = new SearchFieldAnalyzer(); vendorFieldAnalyzer = new SearchFieldAnalyzer(); fieldAnalyzers.put(Fields.PRODUCT, productFieldAnalyzer); fieldAnalyzers.put(Fields.VENDOR, vendorFieldAnalyzer); return new PerFieldAnalyzerWrapper(new KeywordAnalyzer(), fieldAnalyzers); }
private LuceneIndexCreationProfile getTwoAnalyzersLuceneIndexCreationProfile() { Map<String, Analyzer> fieldAnalyzers = new HashMap<>(); fieldAnalyzers.put("field1", new KeywordAnalyzer()); fieldAnalyzers.put("field2", new KeywordAnalyzer()); return new LuceneIndexCreationProfile(INDEX_NAME, REGION_NAME, new String[] {"field1", "field2"}, getPerFieldAnalyzerWrapper(fieldAnalyzers), fieldAnalyzers, null); }
private LuceneIndexCreationProfile getNullField2AnalyzerLuceneIndexCreationProfile() { Map<String, Analyzer> fieldAnalyzers = new HashMap<>(); fieldAnalyzers.put("field1", new KeywordAnalyzer()); fieldAnalyzers.put("field2", null); return new LuceneIndexCreationProfile(INDEX_NAME, REGION_NAME, new String[] {"field1", "field2"}, getPerFieldAnalyzerWrapper(fieldAnalyzers), fieldAnalyzers, null); }
private LuceneIndexCreationProfile getNullField1AnalyzerLuceneIndexCreationProfile() { Map<String, Analyzer> fieldAnalyzers = new HashMap<>(); fieldAnalyzers.put("field1", null); fieldAnalyzers.put("field2", new KeywordAnalyzer()); return new LuceneIndexCreationProfile(INDEX_NAME, REGION_NAME, new String[] {"field1", "field2"}, getPerFieldAnalyzerWrapper(fieldAnalyzers), fieldAnalyzers, null); }
private LuceneIndexForPartitionedRegion getMockLuceneIndex(final String indexName) { LuceneIndexForPartitionedRegion index = mock(LuceneIndexForPartitionedRegion.class); String[] searchableFields = {"field1", "field2"}; Map<String, Analyzer> fieldAnalyzers = new HashMap<>(); fieldAnalyzers.put("field1", new StandardAnalyzer()); fieldAnalyzers.put("field2", new KeywordAnalyzer()); when(index.getName()).thenReturn(indexName); when(index.getRegionPath()).thenReturn("/region"); when(index.getFieldNames()).thenReturn(searchableFields); when(index.getFieldAnalyzers()).thenReturn(fieldAnalyzers); return index; }
private LuceneIndexImpl getMockLuceneIndex(final String indexName) { String[] searchableFields = {"field1", "field2"}; Map<String, Analyzer> fieldAnalyzers = new HashMap<>(); fieldAnalyzers.put("field1", new StandardAnalyzer()); fieldAnalyzers.put("field2", new KeywordAnalyzer()); LuceneIndexImpl index = mock(LuceneIndexImpl.class); when(index.getName()).thenReturn(indexName); when(index.getRegionPath()).thenReturn("/region"); when(index.getFieldNames()).thenReturn(searchableFields); when(index.getFieldAnalyzers()).thenReturn(fieldAnalyzers); return index; }
private Object[] getCheckCompatibilityProfiles() { return $( new Object[] {getOneFieldLuceneIndexCreationProfile(), getTwoFieldLuceneIndexCreationProfile(), CANNOT_CREATE_LUCENE_INDEX_DIFFERENT_FIELDS}, new Object[] {getTwoFieldLuceneIndexCreationProfile(), getReverseFieldsLuceneIndexCreationProfile(), null}, new Object[] {getTwoAnalyzersLuceneIndexCreationProfile(), getOneAnalyzerLuceneIndexCreationProfile(new KeywordAnalyzer()), CANNOT_CREATE_LUCENE_INDEX_DIFFERENT_ANALYZERS}, new Object[] {getOneAnalyzerLuceneIndexCreationProfile(new KeywordAnalyzer()), getTwoAnalyzersLuceneIndexCreationProfile(), CANNOT_CREATE_LUCENE_INDEX_DIFFERENT_ANALYZERS_3}, new Object[] {getOneAnalyzerLuceneIndexCreationProfile(new StandardAnalyzer()), getOneAnalyzerLuceneIndexCreationProfile(new KeywordAnalyzer()), CANNOT_CREATE_LUCENE_INDEX_DIFFERENT_ANALYZERS_2}, new Object[] {getNullField2AnalyzerLuceneIndexCreationProfile(), getNullField1AnalyzerLuceneIndexCreationProfile(), CANNOT_CREATE_LUCENE_INDEX_DIFFERENT_ANALYZERS_1}, new Object[] {getDefaultSerializerCreationProfile(), getDummySerializerCreationProfile(), CANNOT_CREATE_LUCENE_INDEX_DIFFERENT_SERIALIZER}, new Object[] {getDefaultSerializerCreationProfile(), getHeterogeneousLuceneSerializerCreationProfile(), null}, new Object[] {getNullField1AnalyzerLuceneIndexCreationProfile(), getNullField2AnalyzerLuceneIndexCreationProfile(), LuceneTestUtilities.CANNOT_CREATE_LUCENE_INDEX_DIFFERENT_ANALYZERS_2}); }
Map<String, Analyzer> fieldAnalyzers = new HashMap<>(); fieldAnalyzers.put("field1", new StandardAnalyzer()); fieldAnalyzers.put("field2", new KeywordAnalyzer()); fieldAnalyzers.put("field3", null); final LuceneIndexStats mockIndexStats = getMockIndexStats(1, 10, 5, 1);
Map<String, Analyzer> fieldAnalyzers = new HashMap<>(); fieldAnalyzers.put("field1", new StandardAnalyzer()); fieldAnalyzers.put("field2", new KeywordAnalyzer()); fieldAnalyzers.put("field3", null); LuceneSerializer serializer = new HeterogeneousLuceneSerializer();
Map<String, Analyzer> fieldAnalyzers = new HashMap<>(); fieldAnalyzers.put("field1", new StandardAnalyzer()); fieldAnalyzers.put("field2", new KeywordAnalyzer()); fieldAnalyzers.put("field3", null); LuceneSerializer serializer = new HeterogeneousLuceneSerializer();