@Override public boolean incrementToken() throws IOException { if (finiteStrings == null) { if (wasReset == false) { throw new IllegalStateException("reset() missing before incrementToken"); } // lazy init/consume Automaton automaton = toAutomaton(); // calls reset(), incrementToken() repeatedly, and end() on inputTokenStream finiteStrings = new LimitedFiniteStringsIterator(automaton, maxGraphExpansions); //note: would be nice to know the startOffset but toAutomaton doesn't capture it. We'll assume 0 endOffset = inputTokenStream.getAttribute(OffsetAttribute.class).endOffset(); } IntsRef string = finiteStrings.next(); if (string == null) { return false; } clearAttributes(); if (finiteStrings.size() > 1) { // if number of iterated strings so far is more than one... posIncrAtt.setPositionIncrement(0); // stacked } offsetAtt.setOffset(0, endOffset); Util.toBytesRef(string, bytesAtt.builder()); // now we have UTF-8 if (charTermAttribute != null) { charTermAttribute.setLength(0); charTermAttribute.append(bytesAtt.toUTF16()); } return true; }
new LimitedFiniteStringsIterator(toAutomaton(surfaceForm, ts2a), maxGraphExpansions); for (IntsRef string; (string = finiteStrings.next()) != null; count++) { Util.toBytesRef(string, scratch);
@Override public String toString(String field) { StringBuilder buffer = new StringBuilder(); BytesRefBuilder scratch = new BytesRefBuilder(); for (IntsRef context : contexts.keySet()) { if (buffer.length() != 0) { buffer.append(","); } else { buffer.append("contexts"); buffer.append(":["); } buffer.append(Util.toBytesRef(context, scratch).utf8ToString()); ContextMetaData metaData = contexts.get(context); if (metaData.exact == false) { buffer.append("*"); } if (metaData.boost != 0) { buffer.append("^"); buffer.append(Float.toString(metaData.boost)); } } if (buffer.length() != 0) { buffer.append("]"); buffer.append(","); } return buffer.toString() + innerQuery.toString(field); }
@Override public BytesRef lookupOrd(long ord) { try { in.setPosition(0); fst.getFirstArc(firstArc); IntsRef output = Util.getByOutput(fst, ord, in, firstArc, scratchArc, scratchInts); return Util.toBytesRef(output, term); } catch (IOException bogus) { throw new RuntimeException(bogus); } }
@Override public BytesRef lookupOrd(int ord) { try { in.setPosition(0); fst.getFirstArc(firstArc); IntsRef output = Util.getByOutput(fst, ord, in, firstArc, scratchArc, scratchInts); return Util.toBytesRef(output, term); } catch (IOException bogus) { throw new RuntimeException(bogus); } }
public String getUrl(int id) { BytesRef scratchBytes = new BytesRef(); IntsRef key = null; try { key = Util.getByOutput(fst, id); } catch (IOException e) { LOG.error("Error id " + id); e.printStackTrace(); return null; } if (key == null) { return null; } return Util.toBytesRef(key, scratchBytes).utf8ToString(); }
@Override public BytesRef lookupOrd(long ord) { try { in.setPosition(0); fst.getFirstArc(firstArc); IntsRef output = Util.getByOutput(fst, ord, in, firstArc, scratchArc, scratchInts); term.grow(output.length); term.clear(); return Util.toBytesRef(output, term); } catch (IOException bogus) { throw new RuntimeException(bogus); } }
@Override public BytesRef lookupOrd(int ord) { try { in.setPosition(0); fst.getFirstArc(firstArc); IntsRef output = Util.getByOutput(fst, ord, in, firstArc, scratchArc, scratchInts); term.grow(output.length); term.clear(); return Util.toBytesRef(output, term); } catch (IOException bogus) { throw new RuntimeException(bogus); } }
Util.toBytesRef(finiteStrings.next(), bytesAtt.builder()); // now we have UTF-8 if (charTermAttribute != null) { charTermAttribute.setLength(0);
Util.toBytesRef(finiteStrings.next(), bytesAtt.builder()); // now we have UTF-8 if (charTermAttribute != null) { charTermAttribute.setLength(0);
Util.toBytesRef(finiteStrings.next(), bytesAtt.builder()); // now we have UTF-8 if (charTermAttribute != null) { charTermAttribute.setLength(0);
@Override public void seekExact(long ord) throws IOException { // TODO: would be better to make this simpler and faster. // but we dont want to introduce a bug that corrupts our enum state! bytesReader.setPosition(0); fst.getFirstArc(firstArc); IntsRef output = Util.getByOutput(fst, ord, bytesReader, firstArc, scratchArc, scratchInts); // TODO: we could do this lazily, better to try to push into FSTEnum though? in.seekExact(Util.toBytesRef(output, new BytesRefBuilder())); }
@Override public void seekExact(long ord) throws IOException { // TODO: would be better to make this simpler and faster. // but we dont want to introduce a bug that corrupts our enum state! bytesReader.setPosition(0); fst.getFirstArc(firstArc); IntsRef output = Util.getByOutput(fst, ord, bytesReader, firstArc, scratchArc, scratchInts); BytesRefBuilder scratchBytes = new BytesRefBuilder(); scratchBytes.clear(); Util.toBytesRef(output, scratchBytes); // TODO: we could do this lazily, better to try to push into FSTEnum though? in.seekExact(scratchBytes.get()); }
scratch.setLength(prefixLength); Util.toBytesRef(completion.input, suffix); scratch.append(suffix); spare.copyUTF8Bytes(scratch.get());
scratch.length = prefixLength; Util.toBytesRef(completion.input, suffix); scratch.append(suffix); spare.grow(scratch.length);
private void setInnerWeight(IntsRef ref, int offset) { IntsRefBuilder refBuilder = new IntsRefBuilder(); for (int i = offset; i < ref.length; i++) { if (ref.ints[ref.offset + i] == ContextSuggestField.CONTEXT_SEPARATOR) { if (i > 0) { refBuilder.copyInts(ref.ints, ref.offset, i); currentContext = Util.toBytesRef(refBuilder.get(), scratch).utf8ToString(); } else { currentContext = null; } ref.offset = ++i; assert ref.offset < ref.length : "input should not end with the context separator"; if (ref.ints[i] == CompletionAnalyzer.SEP_LABEL) { ref.offset++; assert ref.offset < ref.length : "input should not end with a context separator followed by SEP_LABEL"; } ref.length = ref.length - ref.offset; refBuilder.copyInts(ref.ints, ref.offset, ref.length); innerWeight.setNextMatch(refBuilder.get()); return; } } }
term.grow(io.input.length); Util.toBytesRef(io.input, term); if (io.input.length == 0) { currentFrame = staticFrame;
@Override public boolean incrementToken() throws IOException { clearAttributes(); if (finiteStrings == null) { Automaton automaton = toAutomaton(); finiteStrings = new LimitedFiniteStringsIterator(automaton, maxGraphExpansions); } IntsRef string = finiteStrings.next(); if (string == null) { return false; } Util.toBytesRef(string, bytesAtt.builder()); // now we have UTF-8 if (charTermAttribute != null) { charTermAttribute.setLength(0); charTermAttribute.append(bytesAtt.toUTF16()); } if (payload != null) { payloadAttr.setPayload(this.payload); } return true; }
new LimitedFiniteStringsIterator(toAutomaton(surfaceForm, ts2a), maxGraphExpansions); for (IntsRef string; (string = finiteStrings.next()) != null; count++) { Util.toBytesRef(string, scratch);
new LimitedFiniteStringsIterator(toAutomaton(surfaceForm, ts2a), maxGraphExpansions); for (IntsRef string; (string = finiteStrings.next()) != null; count++) { Util.toBytesRef(string, scratch);