void writeProx(int termID, int proxCode) { if (payloadAttribute == null) { writeVInt(1, proxCode<<1); } else { BytesRef payload = payloadAttribute.getPayload(); if (payload != null && payload.length > 0) { writeVInt(1, (proxCode<<1)|1); writeVInt(1, payload.length); writeBytes(1, payload.bytes, payload.offset, payload.length); sawPayloads = true; } else { writeVInt(1, proxCode<<1); } } assert postingsArray == freqProxPostingsArray; freqProxPostingsArray.lastPositions[termID] = fieldState.position; }
void writeProx(TermVectorsPostingsArray postings, int termID) { if (doVectorOffsets) { int startOffset = fieldState.offset + offsetAttribute.startOffset(); int endOffset = fieldState.offset + offsetAttribute.endOffset(); writeVInt(1, startOffset - postings.lastOffsets[termID]); writeVInt(1, endOffset - startOffset); postings.lastOffsets[termID] = endOffset; } if (doVectorPositions) { final BytesRef payload; if (payloadAttribute == null) { payload = null; } else { payload = payloadAttribute.getPayload(); } final int pos = fieldState.position - postings.lastPositions[termID]; if (payload != null && payload.length > 0) { writeVInt(0, (pos<<1)|1); writeVInt(0, payload.length); writeBytes(0, payload.bytes, payload.offset, payload.length); hasPayloads = true; } else { writeVInt(0, pos<<1); } postings.lastPositions[termID] = fieldState.position; } }
final BytesRef payload = payloadAtt.getPayload(); if(payload == null) continue;
final BytesRef payload = payloadAtt.getPayload(); if(payload == null) { return true;
BytesRef payload = payloadAttr.getPayload(); float tokenWeight = 1.0f; // 1.0 or payload if set and a payload field if(isPayloadField(fieldName) && payload != null){
void writeProx(int termID, int proxCode) { if (payloadAttribute == null) { writeVInt(1, proxCode<<1); } else { BytesRef payload = payloadAttribute.getPayload(); if (payload != null && payload.length > 0) { writeVInt(1, (proxCode<<1)|1); writeVInt(1, payload.length); writeBytes(1, payload.bytes, payload.offset, payload.length); sawPayloads = true; } else { writeVInt(1, proxCode<<1); } } assert postingsArray == freqProxPostingsArray; freqProxPostingsArray.lastPositions[termID] = fieldState.position; }
void writeProx(int termID, int proxCode) { if (payloadAttribute == null) { writeVInt(1, proxCode<<1); } else { BytesRef payload = payloadAttribute.getPayload(); if (payload != null && payload.length > 0) { writeVInt(1, (proxCode<<1)|1); writeVInt(1, payload.length); writeBytes(1, payload.bytes, payload.offset, payload.length); sawPayloads = true; } else { writeVInt(1, proxCode<<1); } } assert postingsArray == freqProxPostingsArray; freqProxPostingsArray.lastPositions[termID] = fieldState.position; }
GenericRecord record = new GenericData.Record( protocol.getType( "PayloadAttribute" ) ); PayloadAttribute payloadAttr = (PayloadAttribute) attr; BytesRef payload = payloadAttr.getPayload(); record.put( "payload", ByteBuffer.wrap( payload.bytes, payload.offset, payload.length ) ); return record;
assertThat( ( (PayloadAttribute) origAttr ).getPayload() ).isEqualTo( ( (PayloadAttribute) copyAttr ).getPayload() );
private Token getNextSuffixInputToken(Token token) throws IOException { if (!suffix.incrementToken()) return null; token.copyBuffer(termAtt.buffer(), 0, termAtt.length()); token.setPositionIncrement(posIncrAtt.getPositionIncrement()); token.setFlags(flagsAtt.getFlags()); token.setOffset(offsetAtt.startOffset(), offsetAtt.endOffset()); token.setType(typeAtt.type()); token.setPayload(payloadAtt.getPayload()); return token; }
private Token getNextInputToken(Token token) throws IOException { if (!input.incrementToken()) return null; token.copyBuffer(in_termAtt.buffer(), 0, in_termAtt.length()); token.setPositionIncrement(in_posIncrAtt.getPositionIncrement()); token.setFlags(in_flagsAtt.getFlags()); token.setOffset(in_offsetAtt.startOffset(), in_offsetAtt.endOffset()); token.setType(in_typeAtt.type()); token.setPayload(in_payloadAtt.getPayload()); return token; }
String value = new String(termAttribute.termBuffer(), 0, termAttribute.termLength()); tokenStream.reset(); field = new Field(f.name(), new SingletonTokenStream(value, (Payload) payloadAttribute.getPayload().clone()));
private Token getNextSuffixInputToken(Token token) throws IOException { if (!suffix.incrementToken()) return null; token.copyBuffer(termAtt.buffer(), 0, termAtt.length()); token.setPositionIncrement(posIncrAtt.getPositionIncrement()); token.setFlags(flagsAtt.getFlags()); token.setOffset(offsetAtt.startOffset(), offsetAtt.endOffset()); token.setType(typeAtt.type()); token.setPayload(payloadAtt.getPayload()); return token; }
private Token getNextPrefixInputToken(Token token) throws IOException { if (!prefix.incrementToken()) return null; token.copyBuffer(p_termAtt.buffer(), 0, p_termAtt.length()); token.setPositionIncrement(p_posIncrAtt.getPositionIncrement()); token.setFlags(p_flagsAtt.getFlags()); token.setOffset(p_offsetAtt.startOffset(), p_offsetAtt.endOffset()); token.setType(p_typeAtt.type()); token.setPayload(p_payloadAtt.getPayload()); return token; }
private Token getNextPrefixInputToken(Token token) throws IOException { if (!prefix.incrementToken()) return null; token.copyBuffer(p_termAtt.buffer(), 0, p_termAtt.length()); token.setPositionIncrement(p_posIncrAtt.getPositionIncrement()); token.setFlags(p_flagsAtt.getFlags()); token.setOffset(p_offsetAtt.startOffset(), p_offsetAtt.endOffset()); token.setType(p_typeAtt.type()); token.setPayload(p_payloadAtt.getPayload()); return token; }
void writeProx(TermVectorsPostingsArray postings, int termID) { if (doVectorOffsets) { int startOffset = fieldState.offset + offsetAttribute.startOffset(); int endOffset = fieldState.offset + offsetAttribute.endOffset(); writeVInt(1, startOffset - postings.lastOffsets[termID]); writeVInt(1, endOffset - startOffset); postings.lastOffsets[termID] = endOffset; } if (doVectorPositions) { final BytesRef payload; if (payloadAttribute == null) { payload = null; } else { payload = payloadAttribute.getPayload(); } final int pos = fieldState.position - postings.lastPositions[termID]; if (payload != null && payload.length > 0) { writeVInt(0, (pos<<1)|1); writeVInt(0, payload.length); writeBytes(0, payload.bytes, payload.offset, payload.length); hasPayloads = true; } else { writeVInt(0, pos<<1); } postings.lastPositions[termID] = fieldState.position; } }
void writeProx(TermVectorsPostingsArray postings, int termID) { if (doVectorOffsets) { int startOffset = fieldState.offset + offsetAttribute.startOffset(); int endOffset = fieldState.offset + offsetAttribute.endOffset(); writeVInt(1, startOffset - postings.lastOffsets[termID]); writeVInt(1, endOffset - startOffset); postings.lastOffsets[termID] = endOffset; } if (doVectorPositions) { final BytesRef payload; if (payloadAttribute == null) { payload = null; } else { payload = payloadAttribute.getPayload(); } final int pos = fieldState.position - postings.lastPositions[termID]; if (payload != null && payload.length > 0) { writeVInt(0, (pos<<1)|1); writeVInt(0, payload.length); writeBytes(0, payload.bytes, payload.offset, payload.length); hasPayloads = true; } else { writeVInt(0, pos<<1); } postings.lastPositions[termID] = fieldState.position; } }
private Token getNextToken(Token token) throws IOException { if (!this.incrementToken()) return null; token.copyBuffer(termAtt.buffer(), 0, termAtt.length()); token.setPositionIncrement(posIncrAtt.getPositionIncrement()); token.setFlags(flagsAtt.getFlags()); token.setOffset(offsetAtt.startOffset(), offsetAtt.endOffset()); token.setType(typeAtt.type()); token.setPayload(payloadAtt.getPayload()); return token; }
private Collection<Token> getTokens(String q, Analyzer analyzer) throws IOException { Collection<Token> result = new ArrayList<Token>(); assert analyzer != null; TokenStream ts = analyzer.tokenStream("", q); try { ts.reset(); // TODO: support custom attributes CharTermAttribute termAtt = ts.addAttribute(CharTermAttribute.class); OffsetAttribute offsetAtt = ts.addAttribute(OffsetAttribute.class); TypeAttribute typeAtt = ts.addAttribute(TypeAttribute.class); FlagsAttribute flagsAtt = ts.addAttribute(FlagsAttribute.class); PayloadAttribute payloadAtt = ts.addAttribute(PayloadAttribute.class); PositionIncrementAttribute posIncAtt = ts.addAttribute(PositionIncrementAttribute.class); while (ts.incrementToken()){ Token token = new Token(); token.copyBuffer(termAtt.buffer(), 0, termAtt.length()); token.setOffset(offsetAtt.startOffset(), offsetAtt.endOffset()); token.setType(typeAtt.type()); token.setFlags(flagsAtt.getFlags()); token.setPayload(payloadAtt.getPayload()); token.setPositionIncrement(posIncAtt.getPositionIncrement()); result.add(token); } ts.end(); return result; } finally { IOUtils.closeWhileHandlingException(ts); } }
GenericRecord record = new GenericData.Record( protocol.getType( "PayloadAttribute" ) ); PayloadAttribute payloadAttr = (PayloadAttribute) attr; BytesRef payload = payloadAttr.getPayload(); record.put( "payload", ByteBuffer.wrap( payload.bytes, payload.offset, payload.length ) ); return record;