@Override public LetterTokenizer create(AttributeFactory factory) { return new LetterTokenizer(factory, maxTokenLen); } }
/** * Instantiates a new lucandra indexer. * * @param analyzer * the analyzer */ public DocumentIndexer() { final String empty = ""; this.analyzer = new StandardAnalyzer(); tokenizer = new LetterTokenizer(); }
@Override public Tokenizer create() { return new LetterTokenizer(); } }
@Override protected Tokenizer create(Version version) { return new LetterTokenizer(); } },
@Override public Tokenizer create() { return new LetterTokenizer(); } }
@Override public LetterTokenizer create(AttributeFactory factory) { return new LetterTokenizer(factory); } }
@Override public Tokenizer create() { return new LetterTokenizer(); } }
TokenStream tokenStream = new LetterTokenizer(new StringReader(document.trim())); CharTermAttribute charTermAttribute = tokenStream .addAttribute(CharTermAttribute.class); tokenStream.reset(); while (tokenStream.incrementToken()) { String string = charTermAttribute.toString(); //Do something with your string } tokenStream.close();
@Override protected TokenStreamComponents createComponents(String fieldName, Reader reader) { LetterTokenizer tokenizer = new LetterTokenizer(BonnieConstants.LUCENE_VERSION, reader); TokenStream result = new LowerCaseFilter(BonnieConstants.LUCENE_VERSION, tokenizer); result = new StopFilter(BonnieConstants.LUCENE_VERSION, result, StopAnalyzer.ENGLISH_STOP_WORDS_SET); result = new PorterStemFilter(result); return new TokenStreamComponents(tokenizer, result); }