Refine search
@Override public final boolean incrementToken() throws IOException { if (input.incrementToken()) { if (!keywordAttr.isKeyword() && termAttribute.length() > length) termAttribute.setLength(length); return true; } else { return false; } } }
@Override public final boolean incrementToken() throws IOException { if (input.incrementToken()) { if (isKeyword()) { keywordAttr.setKeyword(true); } return true; } else { return false; } }
/** Returns the next, stemmed, input Token. * @return The stemmed form of a token. * @throws IOException If there is a low-level I/O error. */ @Override public boolean incrementToken() throws IOException { if (!input.incrementToken()) return false; char[] term = termAttribute.buffer(); int len = termAttribute.length(); if ((!keywordAtt.isKeyword()) && stemmer.stem(term, len)) { termAttribute.setEmpty().append(stemmer.asCharSequence()); } return true; } }
@Override public boolean incrementToken() throws IOException { if (input.incrementToken()) { final String term = termAtt.toString(); // Check the exclusion table. if (!keywordAttr.isKeyword() && (exclusions == null || !exclusions.contains(term))) { final String s = stemmer.stem(term); // If not stemmed, don't waste the time adjusting the token. if ((s != null) && !s.equals(term)) termAtt.setEmpty().append(s); } return true; } else { return false; } } }
@Override public boolean incrementToken() throws IOException { if (tailBuffer == null && tailBufferSaved == null && !input.incrementToken()) { return false; final char[] buffer = termAtt.buffer(); termAtt.setLength(tailBuffer.length); tailBuffer = null; } else if (tailBufferSaved != null) { System.arraycopy(tailBufferSaved, 0, buffer, 0, tailBufferSaved.length); termAtt.setLength(tailBufferSaved.length); tailBufferSaved = null; keywordAttr.setKeyword(true); return true; keywordAttr.setKeyword(true); return true; keywordAttr.setKeyword(true); for (int i = 0; i < termAtt.length(); i++) {
keywordAtt.setKeyword(false); posIncrAtt.setPositionIncrement(1); offsetAtt.setOffset( baseOffset + spans[spanOffset].getEnd() ); termAtt.setEmpty().append(tokens[spanOffset]); for (int i = 0; i < finders.length; i++) { if (types[i]) { keywordAtt.setKeyword(true); posIncrAtt.setPositionIncrement(0); tokenQueue.add(captureState()); termAtt.setEmpty().append(tokenTypeNames[i]);
termAtt.copyBuffer(word.toCharArray(), 0, word.length()); typeAtt.setType(HebrewTokenizer.tokenTypeSignature(HebrewTokenizer.TOKEN_TYPES.Hebrew)); stack.clear(); keywordAtt.setKeyword(true); keywordAtt.setKeyword(true); if ((tokenType & com.code972.hebmorph.Tokenizer.TokenType.Exact) == 0) { stack.add(new HebrewToken(word, (byte) 0, DescFlag.D_EMPTY, word, PrefixType.PS_EMPTY, 1.0f)); keywordAtt.setKeyword(true); termAtt.copyBuffer(word.toCharArray(), 0, word.length()); termAtt.copyBuffer(word.toCharArray(), 0, word.length()); keywordAtt.setKeyword(true); termAtt.copyBuffer(word.toCharArray(), 0, word.length()); keywordAtt.setKeyword(true); return true;
@Override public void copyTo(AttributeImpl target) { KeywordAttribute attr = (KeywordAttribute) target; attr.setKeyword(keyword); }
restoreState(savedState); posIncAtt.setPositionIncrement(0); termAtt.setEmpty().append(nextStem); return true; if (!input.incrementToken()) { return false; if (keywordAtt.isKeyword()) { return true; buffer = dedup ? stemmer.uniqueStems(termAtt.buffer(), termAtt.length()) : stemmer.stem(termAtt.buffer(), termAtt.length());
/** * @return Returns true for next token in the stream, or false at EOS */ @Override public boolean incrementToken() throws IOException { if (input.incrementToken()) { String term = termAtt.toString(); if (!keywordAttr.isKeyword()) { String s = stemmer.stem(term); // If not stemmed, don't waste the time adjusting the token. if ((s != null) && !s.equals(term)) termAtt.setEmpty().append(s); } return true; } else { return false; } }
@Override public boolean incrementToken() throws IOException { if (tailBuffer == null && tailBufferSaved == null && !input.incrementToken()) { return false; final char[] buffer = termAtt.buffer(); termAtt.setLength(tailBuffer.length); tailBuffer = null; } else if (tailBufferSaved != null) { System.arraycopy(tailBufferSaved, 0, buffer, 0, tailBufferSaved.length); termAtt.setLength(tailBufferSaved.length); tailBufferSaved = null; keywordAttr.setKeyword(true); return true; keywordAttr.setKeyword(true); return true; keywordAttr.setKeyword(true); for (int i = 0; i < termAtt.length(); i++) {
@Override public void copyTo(AttributeImpl target) { KeywordAttribute attr = (KeywordAttribute) target; attr.setKeyword(keyword); }
@Override public boolean incrementToken() throws IOException { if (input.incrementToken()) { if (!keywordAttribute.isKeyword()) termAttribute.setLength(bengaliStemmer.stem(termAttribute.buffer(), termAttribute.length())); return true; } else { return false; } } }
@Override public boolean incrementToken() throws IOException { if (input.incrementToken()) { if (!keywordAtt.isKeyword()) { // don't muck with already-keyworded terms String stem = dictionary.get(termAtt.buffer(), 0, termAtt.length()); if (stem != null) { termAtt.setEmpty().append(stem); keywordAtt.setKeyword(true); } } return true; } else { return false; } } }
@Override public boolean incrementToken() throws IOException { if (input.incrementToken()) { if (!keywordAtt.isKeyword()) { String basicForm = basicFormAtt.getBasicForm(); if (basicForm != null && !basicForm.equals("*")) termAtt.setEmpty().append(basicFormAtt.getBasicForm()); } return true; } else { return false; } } }
@Override public boolean incrementToken() throws IOException { if (state != null) { restoreState(state); posIncAttr.setPositionIncrement(0); keywordAttribute.setKeyword(false); state = null; return true; } if (input.incrementToken()) { state = captureState(); keywordAttribute.setKeyword(true); return true; } return false; }
@Override public void copyTo(AttributeImpl target) { KeywordAttribute attr = (KeywordAttribute) target; attr.setKeyword(keyword); }
@Override public boolean incrementToken() throws IOException { if (input.incrementToken()) { if (!keywordAtt.isKeyword()) termAtt.setLength(stemmer.stem(termAtt.buffer(), termAtt.length())); return true; } else { return false; } } }
@Override public final boolean incrementToken() throws IOException { if (input.incrementToken()) { if (isKeyword()) { keywordAttr.setKeyword(true); } return true; } else { return false; } }
@Override public void copyTo(AttributeImpl target) { KeywordAttribute attr = (KeywordAttribute) target; attr.setKeyword(keyword); }