@Override public final CharTermAttribute append(CharSequence csq) { if (csq == null) // needed for Appendable compliance return appendNull(); return append(csq, 0, csq.length()); }
@Override public void addCharTermAttribute(CharSequence sequence) { CharTermAttributeImpl attr = new CharTermAttributeImpl(); attr.append( sequence ); getAttributes().add( attr ); }
@Override public final CharTermAttribute append(CharSequence csq) { if (csq == null) // needed for Appendable compliance return appendNull(); return append(csq, 0, csq.length()); }
@Override public void addCharTermAttribute(CharSequence sequence) { CharTermAttributeImpl attr = new CharTermAttributeImpl(); attr.append( sequence ); getAttributes().add( attr ); }
@Override public final CharTermAttribute append(CharSequence csq) { if (csq == null) // needed for Appendable compliance return appendNull(); return append(csq, 0, csq.length()); }
public static List<List<AttributeImpl>> buildTokenStreamWithAttributes() { List<List<AttributeImpl>> tokens = new ArrayList<>(); tokens.add( new ArrayList<AttributeImpl>() ); CharTermAttributeImpl charAttr = new CharTermAttributeImpl(); charAttr.append( "Wazzza" ); tokens.get( 0 ).add( charAttr ); PayloadAttributeImpl payloadAttribute = new PayloadAttributeImpl(); payloadAttribute.setPayload( new BytesRef( new byte[] { 0, 1, 2, 3 } ) ); tokens.get( 0 ).add( payloadAttribute ); KeywordAttributeImpl keywordAttr = new KeywordAttributeImpl(); keywordAttr.setKeyword( true ); tokens.get( 0 ).add( keywordAttr ); PositionIncrementAttributeImpl posIncrAttr = new PositionIncrementAttributeImpl(); posIncrAttr.setPositionIncrement( 3 ); tokens.get( 0 ).add( posIncrAttr ); FlagsAttributeImpl flagsAttr = new FlagsAttributeImpl(); flagsAttr.setFlags( 435 ); tokens.get( 0 ).add( flagsAttr ); TypeAttributeImpl typeAttr = new TypeAttributeImpl(); typeAttr.setType( "acronym" ); tokens.get( 0 ).add( typeAttr ); OffsetAttributeImpl offsetAttr = new OffsetAttributeImpl(); offsetAttr.setOffset( 4, 7 ); tokens.get( 0 ).add( offsetAttr ); return tokens; }
@Override public final CharTermAttribute append(CharSequence csq) { if (csq == null) // needed for Appendable compliance return appendNull(); return append(csq, 0, csq.length()); }
@Override final public boolean incrementToken() throws IOException { IWord word = segmentor.next(); if ( word == null ) { fieldOffset = offsetAtt.endOffset(); /// System.out.println("set fieldOffset=" + fieldOffset); return false; } clearAttributes(); //char[] token = word.getValue().toCharArray(); //termAtt.copyBuffer(token, 0, token.length); termAtt.clear(); termAtt.append(word.getValue()); offsetAtt.setOffset( correctOffset(fieldOffset + word.getPosition()), correctOffset(fieldOffset + word.getPosition() + word.getLength()) ); typeAtt.setType("word"); return true; }