@Override public final CharTermAttribute append(CharSequence csq) { if (csq == null) // needed for Appendable compliance return appendNull(); return append(csq, 0, csq.length()); }
/** Resets the attributes */ @Override public void clear() { super.clear(); positionIncrement = positionLength = 1; termFrequency = 1; startOffset = endOffset = 0; type = DEFAULT_TYPE; }
@Override public PackedTokenAttributeImpl clone() { return (PackedTokenAttributeImpl) super.clone(); }
@Override public final CharTermAttribute append(StringBuilder s) { if (s == null) // needed for Appendable compliance return appendNull(); final int len = s.length(); s.getChars(0, len, resizeBuffer(termLength + len), termLength); termLength += len; return this; }
@Override public void addCharTermAttribute(CharSequence sequence) { CharTermAttributeImpl attr = new CharTermAttributeImpl(); attr.append( sequence ); getAttributes().add( attr ); }
@Override final public boolean incrementToken() throws IOException { IWord word = segmentor.next(); if ( word == null ) { fieldOffset = offsetAtt.endOffset(); /// System.out.println("set fieldOffset=" + fieldOffset); return false; } clearAttributes(); //char[] token = word.getValue().toCharArray(); //termAtt.copyBuffer(token, 0, token.length); termAtt.clear(); termAtt.append(word.getValue()); offsetAtt.setOffset( correctOffset(fieldOffset + word.getPosition()), correctOffset(fieldOffset + word.getPosition() + word.getLength()) ); typeAtt.setType("word"); return true; }
@Override public int hashCode() { int code = super.hashCode(); code = code * 31 + startOffset; code = code * 31 + endOffset; code = code * 31 + positionIncrement; code = code * 31 + positionLength; if (type != null) code = code * 31 + type.hashCode(); code = code * 31 + termFrequency;; return code; }
@Override public void reflectWith(AttributeReflector reflector) { super.reflectWith(reflector); reflector.reflect(OffsetAttribute.class, "startOffset", startOffset); reflector.reflect(OffsetAttribute.class, "endOffset", endOffset); reflector.reflect(PositionIncrementAttribute.class, "positionIncrement", positionIncrement); reflector.reflect(PositionLengthAttribute.class, "positionLength", positionLength); reflector.reflect(TypeAttribute.class, "type", type); reflector.reflect(TermFrequencyAttribute.class, "termFrequency", termFrequency); } }
@Override public final void copyBuffer(char[] buffer, int offset, int length) { growTermBuffer(length); System.arraycopy(buffer, offset, termBuffer, 0, length); termLength = length; }
@Override public boolean equals(Object obj) { if (obj == this) return true; if (obj instanceof PackedTokenAttributeImpl) { final PackedTokenAttributeImpl other = (PackedTokenAttributeImpl) obj; return (startOffset == other.startOffset && endOffset == other.endOffset && positionIncrement == other.positionIncrement && positionLength == other.positionLength && (type == null ? other.type == null : type.equals(other.type)) && termFrequency == other.termFrequency && super.equals(obj) ); } else return false; }
@Override public void copyTo(AttributeImpl target) { if (target instanceof PackedTokenAttributeImpl) { final PackedTokenAttributeImpl to = (PackedTokenAttributeImpl) target; to.copyBuffer(buffer(), 0, length()); to.positionIncrement = positionIncrement; to.positionLength = positionLength; to.startOffset = startOffset; to.endOffset = endOffset; to.type = type; to.termFrequency = termFrequency; } else { super.copyTo(target); ((OffsetAttribute) target).setOffset(startOffset, endOffset); ((PositionIncrementAttribute) target).setPositionIncrement(positionIncrement); ((PositionLengthAttribute) target).setPositionLength(positionLength); ((TypeAttribute) target).setType(type); ((TermFrequencyAttribute) target).setTermFrequency(termFrequency); } }
public StreamingElementTokens (TokenStream tokens) { super (tokens); // share the same attributes wrapped = tokens; termAtt = addAttribute(CharTermAttribute.class); posIncrAtt = addAttribute(PositionIncrementAttribute.class); term = new CharTermAttributeImpl(); qnames = new ArrayList<QName>(); }
@Override public final CharTermAttribute append(String s) { if (s == null) // needed for Appendable compliance return appendNull(); final int len = s.length(); s.getChars(0, len, resizeBuffer(termLength + len), termLength); termLength += len; return this; }
@Override public void addCharTermAttribute(CharSequence sequence) { CharTermAttributeImpl attr = new CharTermAttributeImpl(); attr.append( sequence ); getAttributes().add( attr ); }
@Override public int hashCode() { int code = super.hashCode(); code = code * 31 + startOffset; code = code * 31 + endOffset; code = code * 31 + positionIncrement; code = code * 31 + positionLength; if (type != null) code = code * 31 + type.hashCode(); code = code * 31 + termFrequency;; return code; }
@Override public void reflectWith(AttributeReflector reflector) { super.reflectWith(reflector); reflector.reflect(OffsetAttribute.class, "startOffset", startOffset); reflector.reflect(OffsetAttribute.class, "endOffset", endOffset); reflector.reflect(PositionIncrementAttribute.class, "positionIncrement", positionIncrement); reflector.reflect(PositionLengthAttribute.class, "positionLength", positionLength); reflector.reflect(TypeAttribute.class, "type", type); }
@Override public final void copyBuffer(char[] buffer, int offset, int length) { growTermBuffer(length); System.arraycopy(buffer, offset, termBuffer, 0, length); termLength = length; }
@Override public boolean equals(Object obj) { if (obj == this) return true; if (obj instanceof PackedTokenAttributeImpl) { final PackedTokenAttributeImpl other = (PackedTokenAttributeImpl) obj; return (startOffset == other.startOffset && endOffset == other.endOffset && positionIncrement == other.positionIncrement && positionLength == other.positionLength && (type == null ? other.type == null : type.equals(other.type)) && super.equals(obj) ); } else return false; }
@Override public void copyTo(AttributeImpl target) { if (target instanceof PackedTokenAttributeImpl) { final PackedTokenAttributeImpl to = (PackedTokenAttributeImpl) target; to.copyBuffer(buffer(), 0, length()); to.positionIncrement = positionIncrement; to.positionLength = positionLength; to.startOffset = startOffset; to.endOffset = endOffset; to.type = type; } else { super.copyTo(target); ((OffsetAttribute) target).setOffset(startOffset, endOffset); ((PositionIncrementAttribute) target).setPositionIncrement(positionIncrement); ((PositionLengthAttribute) target).setPositionLength(positionLength); ((TypeAttribute) target).setType(type); } }
@Override public final CharTermAttribute append(CharTermAttribute ta) { if (ta == null) // needed for Appendable compliance return appendNull(); final int len = ta.length(); System.arraycopy(ta.buffer(), 0, resizeBuffer(termLength + len), termLength, len); termLength += len; return this; }