Refine search
/** * Creates simple term query from the cached tokenstream contents */ protected Query analyzeTerm(String field, TokenStream stream) throws IOException { TermToBytesRefAttribute termAtt = stream.getAttribute(TermToBytesRefAttribute.class); stream.reset(); if (!stream.incrementToken()) { throw new AssertionError(); } return newTermQuery(new Term(field, termAtt.getBytesRef())); }
TermToBytesRefAttribute termAtt = ts.addAttribute(TermToBytesRefAttribute.class); while (ts.incrementToken()) { queryBuilder.addTerm(field, BytesRef.deepCopyOf(termAtt.getBytesRef())); ts.close();
while (tokenStream.incrementToken()) { if (log.isTraceEnabled()) { log.trace("Token: {}, posInc: {}, offset: [{},{}]", term = byteRefAtt.getBytesRef(); if (term.length == 0) { throw new IllegalArgumentException("term: " + term.utf8ToString() + " analyzed to a zero-length token"); tokenStream.end();
short priorMaxNumSightings[] = new short[windowSize]; int cursor = 0; while (input.incrementToken()) { BytesRef bytesRef = termBytesAtt.getBytesRef(); long tokenHash = MurmurHash3.hash128(bytesRef.bytes, bytesRef.offset, bytesRef.length, 0, seed).h1; byte tokenByte = (byte) (tokenHash & 0xFF);
/** * Creates a span query from the tokenstream. In the case of a single token, a simple <code>SpanTermQuery</code> is * returned. When multiple tokens, an ordered <code>SpanNearQuery</code> with slop 0 is returned. */ protected SpanQuery createSpanQuery(TokenStream in, String field) throws IOException { TermToBytesRefAttribute termAtt = in.getAttribute(TermToBytesRefAttribute.class); if (termAtt == null) { return null; } List<SpanTermQuery> terms = new ArrayList<>(); while (in.incrementToken()) { terms.add(new SpanTermQuery(new Term(field, termAtt.getBytesRef()))); } if (terms.isEmpty()) { return null; } else if (terms.size() == 1) { return terms.get(0); } else { return new SpanNearQuery(terms.toArray(new SpanTermQuery[0]), 0, true); } }
int termID = bytesHash.add(termAtt.getBytesRef());
while (tokenStream.incrementToken()) { if (log.isTraceEnabled()) { log.trace("Token: {}, posInc: {}, offset: [{},{}]", term = byteRefAtt.getBytesRef(); if (term.length == 0) { throw new IllegalArgumentException("term: " + term.utf8ToString() + " analyzed to a zero-length token"); tokenStream.end();
@Override @SuppressFBWarnings( value = "EXS_EXCEPTION_SOFTENING_NO_CHECKED", justification = "The contract of AbstractHitEnum makes sense without exposing IOException") public boolean next() { try { if (!tokenStream.incrementToken()) { return false; } BytesRef term = termRef.getBytesRef(); position += positionIncr.getPositionIncrement(); queryWeight = queryWeigher.weigh(term); corpusWeight = corpusWeigher.weigh(term); source = sourceFinder.source(term); return true; } catch (IOException e) { throw new WrappedExceptionFromLucene(e); } }
/** * Creates a span query from the tokenstream. In the case of a single token, a simple <code>SpanTermQuery</code> is * returned. When multiple tokens, an ordered <code>SpanNearQuery</code> with slop 0 is returned. */ protected SpanQuery createSpanQuery(TokenStream in, String field) throws IOException { TermToBytesRefAttribute termAtt = in.getAttribute(TermToBytesRefAttribute.class); if (termAtt == null) { return null; } List<SpanTermQuery> terms = new ArrayList<>(); while (in.incrementToken()) { terms.add(new SpanTermQuery(new Term(field, termAtt.getBytesRef()))); } if (terms.isEmpty()) { return null; } else if (terms.size() == 1) { return terms.get(0); } else { return new SpanNearQuery(terms.toArray(new SpanTermQuery[0]), 0, true); } }
@Override protected boolean accept() throws IOException { return termsHash.find(bytesAtt.getBytesRef()) >= 0; } }
/** * Creates simple boolean query from the cached tokenstream contents */ protected Query analyzeBoolean(String field, TokenStream stream) throws IOException { TermToBytesRefAttribute termAtt = stream.getAttribute(TermToBytesRefAttribute.class); stream.reset(); List<Term> terms = new ArrayList<>(); while (stream.incrementToken()) { terms.add(new Term(field, termAtt.getBytesRef())); } return newSynonymQuery(terms.toArray(new Term[terms.size()])); }
short priorMaxNumSightings[] = new short[windowSize]; int cursor = 0; while (input.incrementToken()) { BytesRef bytesRef = termBytesAtt.getBytesRef(); long tokenHash = MurmurHash3.hash128(bytesRef.bytes, bytesRef.offset, bytesRef.length, 0, seed).h1; byte tokenByte = (byte) (tokenHash & 0xFF);
@SuppressWarnings("unchecked") private List<String> customTokenize(String tokenizerClass, String value){ CachingTokenFilter stream = null; try { final List<String> terms = new ArrayList<>(); final Tokenizer tokenizer = ((Constructor<Tokenizer>) ClassLoader.getSystemClassLoader().loadClass(tokenizerClass) .getConstructor()).newInstance(); tokenizer.setReader(new StringReader(value)); stream = new CachingTokenFilter(tokenizer); final TermToBytesRefAttribute termAtt = stream.getAttribute(TermToBytesRefAttribute.class); stream.reset(); while (stream.incrementToken()) { terms.add(termAtt.getBytesRef().utf8ToString()); } return terms; } catch ( ReflectiveOperationException | IOException e) { throw new IllegalArgumentException(e.getMessage(),e); } finally { IOUtils.closeQuietly(stream); } }
/** * Creates complex boolean query from the cached tokenstream contents */ protected Query analyzeMultiBoolean(String field, TokenStream stream, BooleanClause.Occur operator) throws IOException { BooleanQuery.Builder q = newBooleanQuery(); List<Term> currentQuery = new ArrayList<>(); TermToBytesRefAttribute termAtt = stream.getAttribute(TermToBytesRefAttribute.class); PositionIncrementAttribute posIncrAtt = stream.getAttribute(PositionIncrementAttribute.class); stream.reset(); while (stream.incrementToken()) { if (posIncrAtt.getPositionIncrement() != 0) { add(q, currentQuery, operator); currentQuery.clear(); } currentQuery.add(new Term(field, termAtt.getBytesRef())); } add(q, currentQuery, operator); return q.build(); }
short priorMaxNumSightings[] = new short[windowSize]; int cursor = 0; while (input.incrementToken()) { BytesRef bytesRef = termBytesAtt.getBytesRef(); long tokenHash = MurmurHash3.hash128(bytesRef.bytes, bytesRef.offset, bytesRef.length, 0, seed).h1; byte tokenByte = (byte) (tokenHash & 0xFF);
protected Query doToQuery(QueryShardContext context) throws IOException { // Analyzer analyzer = context.getMapperService().searchAnalyzer(); Analyzer analyzer = new WhitespaceAnalyzer(); try (TokenStream source = analyzer.tokenStream(fieldName, value.toString())) { CachingTokenFilter stream = new CachingTokenFilter(new LowerCaseFilter(source)); TermToBytesRefAttribute termAtt = stream.getAttribute(TermToBytesRefAttribute.class); if (termAtt == null) { return null; } List<CustomSpanTermQuery> clauses = new ArrayList<>(); stream.reset(); while (stream.incrementToken()) { Term term = new Term(fieldName, termAtt.getBytesRef()); clauses.add(new CustomSpanTermQuery(term)); } return new PhraseCountQuery(clauses.toArray(new CustomSpanTermQuery[clauses.size()]), slop, inOrder, weightedCount); } catch (IOException e) { throw new RuntimeException("Error analyzing query text", e); } } private boolean isPosTag(String s) {
/** * Creates simple phrase query from the cached tokenstream contents */ protected Query analyzePhrase(String field, TokenStream stream, int slop) throws IOException { PhraseQuery.Builder builder = new PhraseQuery.Builder(); builder.setSlop(slop); TermToBytesRefAttribute termAtt = stream.getAttribute(TermToBytesRefAttribute.class); PositionIncrementAttribute posIncrAtt = stream.getAttribute(PositionIncrementAttribute.class); int position = -1; stream.reset(); while (stream.incrementToken()) { if (enablePositionIncrements) { position += posIncrAtt.getPositionIncrement(); } else { position += 1; } builder.add(new Term(field, termAtt.getBytesRef()), position); } return builder.build(); }
int termID = bytesHash.add(termAtt.getBytesRef());
mpqb.setSlop(slop); TermToBytesRefAttribute termAtt = stream.getAttribute(TermToBytesRefAttribute.class); PositionIncrementAttribute posIncrAtt = stream.getAttribute(PositionIncrementAttribute.class); int position = -1; stream.reset(); while (stream.incrementToken()) { int positionIncrement = posIncrAtt.getPositionIncrement(); multiTerms.add(new Term(field, termAtt.getBytesRef()));
int termID = bytesHash.add(termAtt.getBytesRef());