@Override public Analyzer createAnalyzer() { return new KeywordAnalyzer(); }
String foo = "foo:doc1.txt"; StandardAnalyzer sa = new StandardAnalyzer(Version.LUCENE_34); TokenStream tokenStream = sa.tokenStream("foo", new StringReader(foo)); while (tokenStream.incrementToken()) { System.out.println(tokenStream.getAttribute(TermAttribute.class).term()); } System.out.println("-------------"); KeywordAnalyzer ka = new KeywordAnalyzer(); TokenStream tokenStream2 = ka.tokenStream("foo", new StringReader(foo)); while (tokenStream2.incrementToken()) { System.out.println(tokenStream2.getAttribute(TermAttribute.class).term()); }
/** * Creates an Analyzer for searching the CPE Index. * * @return the CPE Analyzer. */ private Analyzer createSearchingAnalyzer() { final Map<String, Analyzer> fieldAnalyzers = new HashMap<>(); fieldAnalyzers.put(Fields.DOCUMENT_KEY, new KeywordAnalyzer()); productFieldAnalyzer = new SearchFieldAnalyzer(); vendorFieldAnalyzer = new SearchFieldAnalyzer(); fieldAnalyzers.put(Fields.PRODUCT, productFieldAnalyzer); fieldAnalyzers.put(Fields.VENDOR, vendorFieldAnalyzer); return new PerFieldAnalyzerWrapper(new KeywordAnalyzer(), fieldAnalyzers); }
private LuceneIndexCreationProfile getTwoAnalyzersLuceneIndexCreationProfile() { Map<String, Analyzer> fieldAnalyzers = new HashMap<>(); fieldAnalyzers.put("field1", new KeywordAnalyzer()); fieldAnalyzers.put("field2", new KeywordAnalyzer()); return new LuceneIndexCreationProfile(INDEX_NAME, REGION_NAME, new String[] {"field1", "field2"}, getPerFieldAnalyzerWrapper(fieldAnalyzers), fieldAnalyzers, null); }
private LuceneIndexForPartitionedRegion getMockLuceneIndex(final String indexName) { LuceneIndexForPartitionedRegion index = mock(LuceneIndexForPartitionedRegion.class); String[] searchableFields = {"field1", "field2"}; Map<String, Analyzer> fieldAnalyzers = new HashMap<>(); fieldAnalyzers.put("field1", new StandardAnalyzer()); fieldAnalyzers.put("field2", new KeywordAnalyzer()); when(index.getName()).thenReturn(indexName); when(index.getRegionPath()).thenReturn("/region"); when(index.getFieldNames()).thenReturn(searchableFields); when(index.getFieldAnalyzers()).thenReturn(fieldAnalyzers); return index; }
private LuceneIndexCreationProfile getNullField2AnalyzerLuceneIndexCreationProfile() { Map<String, Analyzer> fieldAnalyzers = new HashMap<>(); fieldAnalyzers.put("field1", new KeywordAnalyzer()); fieldAnalyzers.put("field2", null); return new LuceneIndexCreationProfile(INDEX_NAME, REGION_NAME, new String[] {"field1", "field2"}, getPerFieldAnalyzerWrapper(fieldAnalyzers), fieldAnalyzers, null); }
private LuceneIndexCreationProfile getNullField1AnalyzerLuceneIndexCreationProfile() { Map<String, Analyzer> fieldAnalyzers = new HashMap<>(); fieldAnalyzers.put("field1", null); fieldAnalyzers.put("field2", new KeywordAnalyzer()); return new LuceneIndexCreationProfile(INDEX_NAME, REGION_NAME, new String[] {"field1", "field2"}, getPerFieldAnalyzerWrapper(fieldAnalyzers), fieldAnalyzers, null); }
private LuceneIndexImpl getMockLuceneIndex(final String indexName) { String[] searchableFields = {"field1", "field2"}; Map<String, Analyzer> fieldAnalyzers = new HashMap<>(); fieldAnalyzers.put("field1", new StandardAnalyzer()); fieldAnalyzers.put("field2", new KeywordAnalyzer()); LuceneIndexImpl index = mock(LuceneIndexImpl.class); when(index.getName()).thenReturn(indexName); when(index.getRegionPath()).thenReturn("/region"); when(index.getFieldNames()).thenReturn(searchableFields); when(index.getFieldAnalyzers()).thenReturn(fieldAnalyzers); return index; }
@Override public Analyzer newAnalyzer(final JSONObject args) { return new KeywordAnalyzer(); } },
@Override public Analyzer newAnalyzer(final String args) { return new KeywordAnalyzer(); } @Override
Map<String, Analyzer> fieldAnalyzers = new HashMap<>(); fieldAnalyzers.put("field1", new StandardAnalyzer()); fieldAnalyzers.put("field2", new KeywordAnalyzer()); fieldAnalyzers.put("field3", null); final LuceneIndexStats mockIndexStats = getMockIndexStats(1, 10, 5, 1);
@Override protected Analyzer create(Version version) { return new KeywordAnalyzer(); } },
public KeywordAnalyzerProvider(IndexSettings indexSettings, Environment environment, String name, Settings settings) { super(indexSettings, name, settings); this.keywordAnalyzer = new KeywordAnalyzer(); }
Map<String, Analyzer> fieldAnalyzers = new HashMap<>(); fieldAnalyzers.put("field1", new StandardAnalyzer()); fieldAnalyzers.put("field2", new KeywordAnalyzer()); fieldAnalyzers.put("field3", null); LuceneSerializer serializer = new HeterogeneousLuceneSerializer();
Map<String, Analyzer> fieldAnalyzers = new HashMap<>(); fieldAnalyzers.put("field1", new StandardAnalyzer()); fieldAnalyzers.put("field2", new KeywordAnalyzer()); fieldAnalyzers.put("field3", null); LuceneSerializer serializer = new HeterogeneousLuceneSerializer();
private Object[] getCheckCompatibilityProfiles() { return $( new Object[] {getOneFieldLuceneIndexCreationProfile(), getTwoFieldLuceneIndexCreationProfile(), CANNOT_CREATE_LUCENE_INDEX_DIFFERENT_FIELDS}, new Object[] {getTwoFieldLuceneIndexCreationProfile(), getReverseFieldsLuceneIndexCreationProfile(), null}, new Object[] {getTwoAnalyzersLuceneIndexCreationProfile(), getOneAnalyzerLuceneIndexCreationProfile(new KeywordAnalyzer()), CANNOT_CREATE_LUCENE_INDEX_DIFFERENT_ANALYZERS}, new Object[] {getOneAnalyzerLuceneIndexCreationProfile(new KeywordAnalyzer()), getTwoAnalyzersLuceneIndexCreationProfile(), CANNOT_CREATE_LUCENE_INDEX_DIFFERENT_ANALYZERS_3}, new Object[] {getOneAnalyzerLuceneIndexCreationProfile(new StandardAnalyzer()), getOneAnalyzerLuceneIndexCreationProfile(new KeywordAnalyzer()), CANNOT_CREATE_LUCENE_INDEX_DIFFERENT_ANALYZERS_2}, new Object[] {getNullField2AnalyzerLuceneIndexCreationProfile(), getNullField1AnalyzerLuceneIndexCreationProfile(), CANNOT_CREATE_LUCENE_INDEX_DIFFERENT_ANALYZERS_1}, new Object[] {getDefaultSerializerCreationProfile(), getDummySerializerCreationProfile(), CANNOT_CREATE_LUCENE_INDEX_DIFFERENT_SERIALIZER}, new Object[] {getDefaultSerializerCreationProfile(), getHeterogeneousLuceneSerializerCreationProfile(), null}, new Object[] {getNullField1AnalyzerLuceneIndexCreationProfile(), getNullField2AnalyzerLuceneIndexCreationProfile(), LuceneTestUtilities.CANNOT_CREATE_LUCENE_INDEX_DIFFERENT_ANALYZERS_2}); }
/** * Parses the query. Using this instead of a QueryParser in order * to avoid thread-safety issues with Lucene's query parser. * * @param fieldName the name of the field * @param value the value of the field * @return the parsed query */ private Query parseTokens(String fieldName, String value) { BooleanQuery searchQuery = new BooleanQuery(); if (value != null) { Analyzer analyzer = new KeywordAnalyzer(); try { TokenStream tokenStream = analyzer.tokenStream(fieldName, new StringReader(value)); tokenStream.reset(); CharTermAttribute attr = tokenStream.getAttribute(CharTermAttribute.class); while (tokenStream.incrementToken()) { String term = attr.toString(); Query termQuery = new TermQuery(new Term(fieldName, term)); searchQuery.add(termQuery, Occur.SHOULD); } } catch (IOException e) { throw new DukeException("Error parsing input string '" + value + "' " + "in field " + fieldName); } } return searchQuery; }
tmpAnalyzer = new WhitespaceAnalyzer(); // WhitespaceTokenizer else if (analyzer == AnalyzerType.KeywordAnalyzer) tmpAnalyzer = new KeywordAnalyzer(); // entire string as one token. else if (analyzer == AnalyzerType.StandardAnalyzer) tmpAnalyzer = new StandardAnalyzer();
tmpAnalyzer = new WhitespaceAnalyzer(); // WhitespaceTokenizer else if (analyzer == AnalyzerType.KeywordAnalyzer) tmpAnalyzer = new KeywordAnalyzer(); // entire string as one token. else if (analyzer == AnalyzerType.StandardAnalyzer) tmpAnalyzer = new StandardAnalyzer();
@Override public Analyzer open(Assembler a, Resource root, Mode mode) { return new KeywordAnalyzer(); } }