@Override public int getEndOffset() { if(this.word instanceof OffsetAttribute) { return ((OffsetAttribute)this.word).endOffset(); } return -1; }
@Override public int getStartOffset() { if(this.word instanceof OffsetAttribute) { return ((OffsetAttribute)this.word).startOffset(); } return -1; }
@Override public void copyTo(AttributeImpl target) { CharTermAttribute t = (CharTermAttribute) target; t.copyBuffer(termBuffer, 0, termLength); }
private void setAttribs(PendingToken tok) { clearAttributes(); this.posIncrAtt.setPositionIncrement(tok.nonpos ? 0 : 1); this.termAtt.setEmpty(); this.termAtt.append(tok.str); this.offsetAtt.setOffset(tok.start, tok.end); }
@Override public final void end() throws IOException { super.end(); // set final offset int finalOffset = correctOffset(this.endPosition); offsetAtt.setOffset(finalOffset, finalOffset); posIncrAtt.setPositionIncrement(posIncrAtt.getPositionIncrement() + skippedPositions); } }
TokenStream tokenStream = analyzer.tokenStream(fieldName, reader); OffsetAttribute offsetAttribute = tokenStream.getAttribute(OffsetAttribute.class); TermAttribute termAttribute = tokenStream.getAttribute(TermAttribute.class); while (tokenStream.incrementToken()) { int startOffset = offsetAttribute.startOffset(); int endOffset = offsetAttribute.endOffset(); String term = termAttribute.term(); }
TokenStream tokenStream = analyzer.tokenStream(fieldName, reader); OffsetAttribute offsetAttribute = tokenStream.addAttribute(OffsetAttribute.class); CharTermAttribute charTermAttribute = tokenStream.addAttribute(CharTermAttribute.class); tokenStream.reset(); while (tokenStream.incrementToken()) { int startOffset = offsetAttribute.startOffset(); int endOffset = offsetAttribute.endOffset(); String term = charTermAttribute.toString(); }
@Override public void copyTo(AttributeImpl target) { OffsetAttribute t = (OffsetAttribute) target; t.setOffset(startOffset, endOffset); }
@Override public void copyTo(AttributeImpl target) { TypeAttribute t = (TypeAttribute) target; t.setType(type); }
@Override public void copyTo(AttributeImpl target) { PositionIncrementAttribute t = (PositionIncrementAttribute) target; t.setPositionIncrement(positionIncrement); }
@Override public void copyTo(AttributeImpl target) { KeywordAttribute attr = (KeywordAttribute) target; attr.setKeyword(keyword); }
@Override public void copyTo(AttributeImpl target) { FlagsAttribute t = (FlagsAttribute) target; t.setFlags(flags); }
/** Resets the attributes */ @Override public void clear() { super.clear(); positionIncrement = positionLength = 1; termFrequency = 1; startOffset = endOffset = 0; type = DEFAULT_TYPE; }
private CharTermAttribute appendNull() { resizeBuffer(termLength + 4); termBuffer[termLength++] = 'n'; termBuffer[termLength++] = 'u'; termBuffer[termLength++] = 'l'; termBuffer[termLength++] = 'l'; return this; }
/** Resets the attributes at end */ @Override public void end() { super.end(); // super.end already calls this.clear, so we only set values that are different from clear: positionIncrement = 0; }
@Override public void copyTo(AttributeImpl target) { TermFrequencyAttribute t = (TermFrequencyAttribute) target; t.setTermFrequency(termFrequency); }
@Override public int getStartOffset() { if(this.word instanceof OffsetAttribute) { return ((OffsetAttribute)this.word).startOffset(); } return -1; }
@Override public int getEndOffset() { if(this.word instanceof OffsetAttribute) { return ((OffsetAttribute)this.word).endOffset(); } return -1; }
/** * Fills CharTermAttribute with the current token text. */ public final void getText(CharTermAttribute t) { t.copyBuffer(zzBuffer, zzStartRead, zzMarkedPos-zzStartRead); }
@Override public final CharTermAttribute append(char c) { resizeBuffer(termLength + 1)[termLength++] = c; return this; }