@Override public boolean hasPayloads() { return wrapped.hasPayloads(); }
@Override public Set<BytesRef> contexts() { return wrapped.contexts(); }
@Override public boolean hasContexts() { return wrapped.hasContexts(); }
@Override public void build(InputIterator iterator) throws IOException { String prefix = getClass().getSimpleName(); Directory tempDir = getTempDir(); OfflineSorter sorter = new OfflineSorter(tempDir, prefix, new AnalyzingComparator(hasPayloads)); OfflineSorter.ByteSequencesReader reader = null; hasPayloads = iterator.hasPayloads(); TokenStreamToAutomaton ts2a = getTokenStreamToAutomaton(); String tempSortedFileName = null; ByteArrayDataOutput output = new ByteArrayDataOutput(buffer); for (BytesRef surfaceForm; (surfaceForm = iterator.next()) != null;) { LimitedFiniteStringsIterator finiteStrings = new LimitedFiniteStringsIterator(toAutomaton(surfaceForm, ts2a), maxGraphExpansions); for (IntsRef string; (string = finiteStrings.next()) != null; count++) { Util.toBytesRef(string, scratch); "cannot handle surface form > " + (Short.MAX_VALUE-2) + " in length (got " + surfaceForm.length + ")"); payload = iterator.payload(); output.writeInt(encodeWeight(iterator.weight()));
private LookupResult getLookupResult(Long output1, BytesRef output2, CharsRefBuilder spare) { LookupResult result; if (hasPayloads) { int sepIndex = -1; for(int i=0;i<output2.length;i++) { if (output2.bytes[output2.offset+i] == payloadSep) { sepIndex = i; break; } } assert sepIndex != -1; final int payloadLen = output2.length - sepIndex - 1; spare.copyUTF8Bytes(output2.bytes, output2.offset, sepIndex); BytesRef payload = new BytesRef(payloadLen); System.arraycopy(output2.bytes, sepIndex+1, payload.bytes, 0, payloadLen); payload.length = payloadLen; result = new LookupResult(spare.toString(), decodeWeight(output1), payload); } else { spare.copyUTF8Bytes(output2); result = new LookupResult(spare.toString(), decodeWeight(output1)); } return result; }
/** * Creates a new sorted wrapper, sorting by BytesRef * (ascending) then cost (ascending). */ public SortedInputIterator(InputIterator source, Comparator<BytesRef> comparator) throws IOException { this.hasPayloads = source.hasPayloads(); this.hasContexts = source.hasContexts(); this.source = source; this.comparator = comparator; this.reader = sort(); }
@Override public BytesRef payload() { return wrapped.payload(); }
/** * Look up a key and return possible completion for this key. * @param key lookup key. Depending on the implementation this may be * a prefix, misspelling, or even infix. * @param onlyMorePopular return only more popular results * @param num maximum number of results to return * @return a list of possible completions, with their relative weight (e.g. popularity) */ public List<LookupResult> lookup(CharSequence key, boolean onlyMorePopular, int num) throws IOException { return lookup(key, null, onlyMorePopular, num); }
public SortedTermFreqIteratorWrapper(TermFreqIterator source, Comparator<BytesRef> comparator, boolean compareRawBytes) throws IOException { this.source = source; this.comparator = comparator; this.reader = sort(compareRawBytes ? comparator : new BytesOnlyComparator(this.comparator)); }
@Override protected void setPivot(int i) { final int ord = orderdEntries[i]; get(pivot, ord); }
/** * sugar for {@link #iterator(Comparator)} with a <code>null</code> comparator */ public BytesRefIterator iterator() { return iterator(null); }
@Override public InputIterator getEntryIterator() throws IOException { return new DocumentInputIterator(payloadField!=null, contextsField!=null); }
@Override public InputIterator getEntryIterator() throws IOException { return new DocumentValueSourceInputIterator(payloadField!=null, contextsField!=null); }
WFSTInputIterator(InputIterator source) throws IOException { super(source); assert source.hasPayloads() == false; }
public BytesRef next() throws IOException { if (pos < size) { return get(spare, ords == null ? pos++ : ords[pos++]); } return null; }
public BytesRefIterator iterator() { closed = true; return buffer.iterator(comparator); }