private String processToken(String token, Language language, StemMode stemMode, boolean removeAccents) { token = normalizer.normalize(token); token = LinguisticsCase.toLowerCase(token); if (removeAccents) token = transformer.accentDrop(token, language); if (stemMode != StemMode.NONE) token = stemmer.stem(token); return token; }
private String processToken(String token, Language language, StemMode stemMode, boolean removeAccents, Stemmer stemmer) { token = normalizer.normalize(token); token = LinguisticsCase.toLowerCase(token); if (removeAccents) token = transformer.accentDrop(token, language); if (stemMode != StemMode.NONE) { token = doStemming(token, stemmer); } return token; }
@Override protected void doExecute(ExecutionContext context) { Transformer transformer = linguistics.getTransformer(); context.setValue(new StringFieldValue(transformer.accentDrop(String.valueOf(context.getValue()), context.resolveLanguage(linguistics)))); }
private void normalizeWord(Language language, IndexFacts.Session indexFacts, TermItem term, ListIterator<Item> i) { if ( ! (term instanceof WordItem)) return; if ( ! term.isNormalizable()) return; Index index = indexFacts.getIndex(term.getIndexName()); if (index.isAttribute()) return; if ( ! index.getNormalize()) return; WordItem word = (WordItem) term; String accentDropped = linguistics.getTransformer().accentDrop(word.getWord(), language); if (accentDropped.length() == 0) i.remove(); else word.setWord(accentDropped); }
private void normalizeAlternatives(Language language, Session indexFacts, WordAlternativesItem block) { if (!block.isNormalizable()) { return; } { Index index = indexFacts.getIndex(block.getIndexName()); if (index.isAttribute()) { return; } if (!index.getNormalize()) { return; } } List<Alternative> terms = block.getAlternatives(); for (Alternative term : terms) { String accentDropped = linguistics.getTransformer().accentDrop(term.word, language); if (!term.word.equals(accentDropped) && accentDropped.length() > 0) { block.addTerm(accentDropped, term.exactness * .7d); } } }