@Override public void copyTo(AttributeImpl target) { KeywordAttribute attr = (KeywordAttribute) target; attr.setKeyword(keyword); }
@Override public final boolean incrementToken() throws IOException { if (input.incrementToken()) { if (isKeyword()) { keywordAttr.setKeyword(true); } return true; } else { return false; } }
@Override public boolean incrementToken() throws IOException { if (state != null) { restoreState(state); posIncAttr.setPositionIncrement(0); keywordAttribute.setKeyword(false); state = null; return true; } if (input.incrementToken()) { state = captureState(); keywordAttribute.setKeyword(true); return true; } return false; }
@Override public boolean incrementToken() throws IOException { if (input.incrementToken()) { if (fstReader == null) { // No overrides return true; } if (!keywordAtt.isKeyword()) { // don't muck with already-keyworded terms final BytesRef stem = stemmerOverrideMap.get(termAtt.buffer(), termAtt.length(), scratchArc, fstReader); if (stem != null) { spare = ArrayUtil.grow(termAtt.buffer(), stem.length); final int length = UnicodeUtil.UTF8toUTF16(stem, spare); if (spare != termAtt.buffer()) { termAtt.copyBuffer(spare, 0, length); } else { termAtt.setLength(length); } keywordAtt.setKeyword(true); } } return true; } else { return false; } }
@Override public void copyTo(AttributeImpl target) { KeywordAttribute attr = (KeywordAttribute) target; attr.setKeyword(keyword); }
@Override public void copyTo(AttributeImpl target) { KeywordAttribute attr = (KeywordAttribute) target; attr.setKeyword(keyword); }
@Override public void copyTo(AttributeImpl target) { KeywordAttribute attr = (KeywordAttribute) target; attr.setKeyword(keyword); }
@Override public final boolean incrementToken() throws IOException { if (input.incrementToken()) { if (isKeyword()) { keywordAttr.setKeyword(true); } return true; } else { return false; } }
@Override public boolean incrementToken() throws IOException { if (state != null) { restoreState(state); posIncAttr.setPositionIncrement(0); keywordAttribute.setKeyword(false); state = null; return true; } if (input.incrementToken()) { state = captureState(); keywordAttribute.setKeyword(true); return true; } return false; }
@Override public boolean incrementToken() throws IOException { if (input.incrementToken()) { if (!keywordAtt.isKeyword()) { // don't muck with already-keyworded terms String stem = dictionary.get(termAtt.buffer(), 0, termAtt.length()); if (stem != null) { termAtt.setEmpty().append(stem); keywordAtt.setKeyword(true); } } return true; } else { return false; } } }
keywordAtt.setKeyword(true); keywordAtt.setKeyword(true); if ((tokenType & com.code972.hebmorph.Tokenizer.TokenType.Exact) == 0) { stack.add(new HebrewToken(word, (byte) 0, DescFlag.D_EMPTY, word, PrefixType.PS_EMPTY, 1.0f)); keywordAtt.setKeyword(true); keywordAtt.setKeyword(true); keywordAtt.setKeyword(true); return true;
keywordAttr.setKeyword(true); return true; keywordAttr.setKeyword(true); return true; keywordAttr.setKeyword(true);
keywordAtt.setKeyword(false); posIncrAtt.setPositionIncrement(1); offsetAtt.setOffset( for (int i = 0; i < finders.length; i++) { if (types[i]) { keywordAtt.setKeyword(true); posIncrAtt.setPositionIncrement(0); tokenQueue.add(captureState());
keywordAttr.setKeyword(true); return true; keywordAttr.setKeyword(true); return true; keywordAttr.setKeyword(true);
@Override public boolean incrementToken() throws IOException { if (input.incrementToken()) { if (fstReader == null) { // No overrides return true; } if (!keywordAtt.isKeyword()) { // don't muck with already-keyworded terms final BytesRef stem = stemmerOverrideMap.get(termAtt.buffer(), termAtt.length(), scratchArc, fstReader); if (stem != null) { spare = ArrayUtil.grow(termAtt.buffer(), stem.length); final int length = UnicodeUtil.UTF8toUTF16(stem, spare); if (spare != termAtt.buffer()) { termAtt.copyBuffer(spare, 0, length); } else { termAtt.setLength(length); } keywordAtt.setKeyword(true); } } return true; } else { return false; } }