@Override public boolean incrementToken() throws IOException { if (input.incrementToken()) { final char[] buffer = termAtt.buffer(); final int length = termAtt.length(); for (int i = 0; i < length; i++) { if (buffer[i] == delimiter) { payAtt.setPayload(encoder.encode(buffer, i + 1, (length - (i + 1)))); termAtt.setLength(i); // simply set a new length return true; } } // we have not seen the delimiter payAtt.setPayload(null); return true; } else return false; } }
void writeProx(TermVectorsPostingsArray postings, int termID) { if (doVectorOffsets) { int startOffset = fieldState.offset + offsetAttribute.startOffset(); int endOffset = fieldState.offset + offsetAttribute.endOffset(); writeVInt(1, startOffset - postings.lastOffsets[termID]); writeVInt(1, endOffset - startOffset); postings.lastOffsets[termID] = endOffset; } if (doVectorPositions) { final BytesRef payload; if (payloadAttribute == null) { payload = null; } else { payload = payloadAttribute.getPayload(); } final int pos = fieldState.position - postings.lastPositions[termID]; if (payload != null && payload.length > 0) { writeVInt(0, (pos<<1)|1); writeVInt(0, payload.length); writeBytes(0, payload.bytes, payload.offset, payload.length); hasPayloads = true; } else { writeVInt(0, pos<<1); } postings.lastPositions[termID] = fieldState.position; } }
private Token getNextSuffixInputToken(Token token) throws IOException { if (!suffix.incrementToken()) return null; token.copyBuffer(termAtt.buffer(), 0, termAtt.length()); token.setPositionIncrement(posIncrAtt.getPositionIncrement()); token.setFlags(flagsAtt.getFlags()); token.setOffset(offsetAtt.startOffset(), offsetAtt.endOffset()); token.setType(typeAtt.type()); token.setPayload(payloadAtt.getPayload()); return token; }
private Collection<Token> getTokens(String q, Analyzer analyzer) throws IOException { Collection<Token> result = new ArrayList<Token>(); assert analyzer != null; TokenStream ts = analyzer.tokenStream("", q); try { ts.reset(); // TODO: support custom attributes CharTermAttribute termAtt = ts.addAttribute(CharTermAttribute.class); OffsetAttribute offsetAtt = ts.addAttribute(OffsetAttribute.class); TypeAttribute typeAtt = ts.addAttribute(TypeAttribute.class); FlagsAttribute flagsAtt = ts.addAttribute(FlagsAttribute.class); PayloadAttribute payloadAtt = ts.addAttribute(PayloadAttribute.class); PositionIncrementAttribute posIncAtt = ts.addAttribute(PositionIncrementAttribute.class); while (ts.incrementToken()){ Token token = new Token(); token.copyBuffer(termAtt.buffer(), 0, termAtt.length()); token.setOffset(offsetAtt.startOffset(), offsetAtt.endOffset()); token.setType(typeAtt.type()); token.setFlags(flagsAtt.getFlags()); token.setPayload(payloadAtt.getPayload()); token.setPositionIncrement(posIncAtt.getPositionIncrement()); result.add(token); } ts.end(); return result; } finally { IOUtils.closeWhileHandlingException(ts); } }
TermToBytesRefAttribute termAtt = stream.getAttribute(TermToBytesRefAttribute.class); PositionIncrementAttribute posIncrAttribute = stream.addAttribute(PositionIncrementAttribute.class); OffsetAttribute offsetAtt = stream.addAttribute(OffsetAttribute.class); PayloadAttribute payloadAtt = storePayloads ? stream.addAttribute(PayloadAttribute.class) : null; stream.reset(); final int posIncr = posIncrAttribute.getPositionIncrement(); if (posIncr == 0) { info.numOverlapTokens++; postingsWriter.writeInt(pos); if (storeOffsets) { postingsWriter.writeInt(offsetAtt.startOffset() + offset); postingsWriter.writeInt(offsetAtt.endOffset() + offset); final BytesRef payload = payloadAtt.getPayload(); final int pIndex; if (payload == null || payload.length == 0) { if (info.numTokens > 0) { info.lastPosition = pos; info.lastOffset = offsetAtt.endOffset() + offset;
int tokenCount = 0; CharTermAttribute termAtt = ts.addAttribute(CharTermAttribute.class); PayloadAttribute payloadAttr = ts.addAttribute(PayloadAttribute.class); ts.reset(); while (ts.incrementToken()) { String word = termAtt.toString(); BytesRef payload = payloadAttr.getPayload(); float tokenWeight = 1.0f; // 1.0 or payload if set and a payload field if(isPayloadField(fieldName) && payload != null){
@Override public boolean incrementToken() throws IOException { if (!input.incrementToken()) { return false; } payloadAttribute.setPayload(payloadSpare); return true; } }
TermAttribute termAttribute = tokenStream.addAttribute(TermAttribute.class); PayloadAttribute payloadAttribute = tokenStream.addAttribute(PayloadAttribute.class); tokenStream.incrementToken(); tokenStream.end(); tokenStream.close(); PropertyMetaData pdm = PropertyMetaData .fromByteArray(payloadAttribute .getPayload().getData()); doc.add(new Field(field.name(), new SingletonTokenStream(value,
field.tokenStreamValue().incrementToken(); CharTermAttribute term = field.tokenStreamValue().getAttribute(CharTermAttribute.class); PayloadAttribute payload = field.tokenStreamValue().getAttribute(PayloadAttribute.class); String value = new String(term.buffer(), 0, term.length()); value = FieldNames.createNamedValue(path, value); term.setEmpty(); term.append(value); doc.add(new Field(field.name(), new SingletonTokenStream(term.toString(), payload .getPayload()))); doc.add(new Field(FieldNames.AGGREGATED_NODE_UUID, parent.getIdentifier(), Field.Store.NO, Field.Index.NOT_ANALYZED_NO_NORMS));
@Override public final boolean incrementToken() throws IOException { if (input.incrementToken()) { byte[] data = new byte[8]; PayloadHelper.encodeInt(offsetAtt.startOffset(), data, 0); PayloadHelper.encodeInt(offsetAtt.endOffset(), data, 4); BytesRef payload = new BytesRef(data); payAtt.setPayload(payload); return true; } else { return false; } } }
@Override public final boolean incrementToken() throws IOException { if (input.incrementToken()) { CharTermAttribute termAtt = this.getAttribute(CharTermAttribute.class); final String term = termAtt.toString(); termAtt.setEmpty(); final BytesRef payload = payloadAtt.getPayload(); if(payload == null) { return true; termAtt.append(boostedTerm); return true;
@Override public final boolean incrementToken() throws IOException { if (iter.hasNext()) { clearAttributes(); // This is the dummy term. termAttr.setEmpty(); termAttr.append(iter.next()); payloadAttr.setPayload(intToByteArray(Float.floatToIntBits(this.tokens.length),payload)); positionAttr.setPositionIncrement(1); offsetAttr.setOffset(0, count); count++; return true; } return false; }
assertThat( ( (PayloadAttribute) origAttr ).getPayload() ).isEqualTo( ( (PayloadAttribute) copyAttr ).getPayload() ); assertThat( ( (PositionIncrementAttribute) origAttr ).getPositionIncrement() ).isEqualTo( ( (PositionIncrementAttribute) copyAttr ).getPositionIncrement() ); OffsetAttribute orig = (OffsetAttribute) origAttr; OffsetAttribute cop = (OffsetAttribute) copyAttr; assertThat( orig.startOffset() ).isEqualTo( cop.startOffset() ); assertThat( orig.endOffset() ).isEqualTo( cop.endOffset() );
private void setCurrentToken(Token token) { if (token == null) return; clearAttributes(); termAtt.copyBuffer(token.buffer(), 0, token.length()); posIncrAtt.setPositionIncrement(token.getPositionIncrement()); flagsAtt.setFlags(token.getFlags()); offsetAtt.setOffset(token.startOffset(), token.endOffset()); typeAtt.setType(token.type()); payloadAtt.setPayload(token.getPayload()); }
posAttr.setPositionIncrement(posInc); charTermAttribute.setLength(0); charTermAttribute.append(bytesAtt.toUTF16()); payloadAttr.setPayload(this.payload);
protected void setAttributes(String token, float payload) { CharTermAttribute termAtt = this.getAttribute(CharTermAttribute.class); termAtt.setEmpty(); termAtt.append(token); termAtt.setLength(token.length()); PayloadAttribute payloadAtt = this.getAttribute(PayloadAttribute.class); byte[] bytes = PayloadHelper.encodeFloat(payload); payloadAtt.setPayload(new BytesRef(bytes)); }
final Map<String, Integer> tokenCount = new HashMap<String, Integer>(); while (input.incrementToken()) { CharTermAttribute termAtt = this.getAttribute(CharTermAttribute.class); final BytesRef payload = payloadAtt.getPayload(); if(payload == null) continue;
@Override public boolean incrementToken() throws IOException { if (consumed) { return false; } clearAttributes(); termAttribute.setEmpty(); termAttribute.append(value); payloadAttribute.setPayload(payload); consumed = true; return true; }
@Override public void copyTo(AttributeImpl target) { if (target instanceof Token) { final Token to = (Token) target; to.reinit(this); // reinit shares the payload, so clone it: if (payload !=null) { to.payload = payload.clone(); } } else { super.copyTo(target); ((OffsetAttribute) target).setOffset(startOffset, endOffset); ((PositionIncrementAttribute) target).setPositionIncrement(positionIncrement); ((PayloadAttribute) target).setPayload((payload == null) ? null : payload.clone()); ((FlagsAttribute) target).setFlags(flags); ((TypeAttribute) target).setType(type); } }
@Override final public boolean incrementToken() { // Capture token contents if (iterator.hasNext()) { String word = iterator.next(); termAttr.copyBuffer(word.toCharArray(), 0, word.length()); positionIncrementAttr.setPositionIncrement(incrementIt.next()); if (payloadAttr != null) { payloadAttr.setPayload(payloadIt.next()); } return true; } return false; }