ByteArrayDataOutput output = new ByteArrayDataOutput(buffer); for (BytesRef surfaceForm; (surfaceForm = iterator.next()) != null;) { LimitedFiniteStringsIterator finiteStrings = new LimitedFiniteStringsIterator(toAutomaton(surfaceForm, ts2a), maxGraphExpansions);
@Override public void build(InputIterator iterator) throws IOException { if (iterator.hasPayloads()) { throw new IllegalArgumentException("this suggester doesn't support payloads"); } if (iterator.hasContexts()) { throw new IllegalArgumentException("this suggester doesn't support contexts"); } root = new TernaryTreeNode(); // make sure it's sorted and the comparator uses UTF16 sort order iterator = new SortedInputIterator(iterator, BytesRef.getUTF8SortedAsUTF16Comparator()); count = 0; ArrayList<String> tokens = new ArrayList<>(); ArrayList<Number> vals = new ArrayList<>(); BytesRef spare; CharsRefBuilder charsSpare = new CharsRefBuilder(); while ((spare = iterator.next()) != null) { charsSpare.copyUTF8Bytes(spare); tokens.add(charsSpare.toString()); vals.add(Long.valueOf(iterator.weight())); count++; } autocomplete.balancedTree(tokens.toArray(), vals.toArray(), 0, tokens.size() - 1, root); }
@Override public void build(InputIterator iterator) throws IOException { if (iterator.hasPayloads()) { throw new IllegalArgumentException("this suggester doesn't support payloads"); } if (iterator.hasContexts()) { throw new IllegalArgumentException("this suggester doesn't support contexts"); } count = 0; trie = new JaspellTernarySearchTrie(); trie.setMatchAlmostDiff(editDistance); BytesRef spare; final CharsRefBuilder charsSpare = new CharsRefBuilder(); while ((spare = iterator.next()) != null) { final long weight = iterator.weight(); if (spare.length == 0) { continue; } charsSpare.copyUTF8Bytes(spare); trie.put(charsSpare.toString(), Long.valueOf(weight)); count++; } }
/** Creates a new iterator, buffering entries from the specified iterator */ public BufferedInputIterator(InputIterator source) throws IOException { BytesRef spare; int freqIndex = 0; hasPayloads = source.hasPayloads(); hasContexts = source.hasContexts(); while((spare = source.next()) != null) { entries.append(spare); if (hasPayloads) { payloads.append(source.payload()); } if (hasContexts) { contextSets.add(source.contexts()); } if (freqIndex >= freqs.length) { freqs = ArrayUtil.grow(freqs, freqs.length+1); } freqs[freqIndex++] = source.weight(); } }
ByteArrayDataOutput output = new ByteArrayDataOutput(buffer); while ((spare = source.next()) != null) { encode(writer, output, buffer, spare, source.payload(), source.contexts(), source.weight());
try { while (true) { BytesRef surfaceForm = iterator.next(); if (surfaceForm == null) { break;
while ((text = iter.next()) != null) { BytesRef payload; if (iter.hasPayloads()) {
PositiveIntOutputs outputs = PositiveIntOutputs.getSingleton(); Builder<Long> builder = new Builder<>(FST.INPUT_TYPE.BYTE1, outputs); while ((scratch = iter.next()) != null) { long cost = iter.weight();
ByteArrayDataOutput output = new ByteArrayDataOutput(buffer); BytesRef spare; while ((spare = iterator.next()) != null) { if (spare.length + 4 >= buffer.length) { buffer = ArrayUtil.grow(buffer, spare.length + 4);
ByteArrayDataOutput output = new ByteArrayDataOutput(buffer); for (BytesRef surfaceForm; (surfaceForm = iterator.next()) != null;) { LimitedFiniteStringsIterator finiteStrings = new LimitedFiniteStringsIterator(toAutomaton(surfaceForm, ts2a), maxGraphExpansions);
ByteArrayDataOutput output = new ByteArrayDataOutput(buffer); for (BytesRef surfaceForm; (surfaceForm = iterator.next()) != null;) { LimitedFiniteStringsIterator finiteStrings = new LimitedFiniteStringsIterator(toAutomaton(surfaceForm, ts2a), maxGraphExpansions);
ByteArrayDataOutput output = new ByteArrayDataOutput(buffer); for (BytesRef surfaceForm; (surfaceForm = iterator.next()) != null;) { LimitedFiniteStringsIterator finiteStrings = new LimitedFiniteStringsIterator(toAutomaton(surfaceForm, ts2a), maxGraphExpansions);
ByteArrayDataOutput output = new ByteArrayDataOutput(buffer); for (BytesRef surfaceForm; (surfaceForm = iterator.next()) != null;) { LimitedFiniteStringsIterator finiteStrings = new LimitedFiniteStringsIterator(toAutomaton(surfaceForm, ts2a), maxGraphExpansions);
ByteArrayDataOutput output = new ByteArrayDataOutput(buffer); for (BytesRef surfaceForm; (surfaceForm = iterator.next()) != null;) { LimitedFiniteStringsIterator finiteStrings = new LimitedFiniteStringsIterator(toAutomaton(surfaceForm, ts2a), maxGraphExpansions);