@Override public boolean hasPayloads() { return wrapped.hasPayloads(); }
@Override public BytesRef payload() { return wrapped.payload(); }
OfflineSorter.ByteSequencesReader reader = null; hasPayloads = iterator.hasPayloads(); ByteArrayDataOutput output = new ByteArrayDataOutput(buffer); for (BytesRef surfaceForm; (surfaceForm = iterator.next()) != null;) { LimitedFiniteStringsIterator finiteStrings = new LimitedFiniteStringsIterator(toAutomaton(surfaceForm, ts2a), maxGraphExpansions); "cannot handle surface form > " + (Short.MAX_VALUE-2) + " in length (got " + surfaceForm.length + ")"); payload = iterator.payload(); output.writeInt(encodeWeight(iterator.weight()));
if (iterator.hasPayloads()) { throw new IllegalArgumentException("this suggester doesn't support payloads"); if (iterator.hasContexts()) { throw new IllegalArgumentException("this suggester doesn't support contexts"); try { while (true) { BytesRef surfaceForm = iterator.next(); if (surfaceForm == null) { break;
/** * Creates a new sorted wrapper, sorting by BytesRef * (ascending) then cost (ascending). */ public SortedInputIterator(InputIterator source, Comparator<BytesRef> comparator) throws IOException { this.hasPayloads = source.hasPayloads(); this.hasContexts = source.hasContexts(); this.source = source; this.comparator = comparator; this.reader = sort(); }
@Override public Set<BytesRef> contexts() { return wrapped.contexts(); }
@Override public boolean hasContexts() { return wrapped.hasContexts(); }
/** Creates a new iterator, buffering entries from the specified iterator */ public BufferedInputIterator(InputIterator source) throws IOException { BytesRef spare; int freqIndex = 0; hasPayloads = source.hasPayloads(); hasContexts = source.hasContexts(); while((spare = source.next()) != null) { entries.append(spare); if (hasPayloads) { payloads.append(source.payload()); } if (hasContexts) { contextSets.add(source.contexts()); } if (freqIndex >= freqs.length) { freqs = ArrayUtil.grow(freqs, freqs.length+1); } freqs[freqIndex++] = source.weight(); } }
@Override public void build(InputIterator iterator) throws IOException { if (iterator.hasPayloads()) { throw new IllegalArgumentException("this suggester doesn't support payloads"); } if (iterator.hasContexts()) { throw new IllegalArgumentException("this suggester doesn't support contexts"); } count = 0; trie = new JaspellTernarySearchTrie(); trie.setMatchAlmostDiff(editDistance); BytesRef spare; final CharsRefBuilder charsSpare = new CharsRefBuilder(); while ((spare = iterator.next()) != null) { final long weight = iterator.weight(); if (spare.length == 0) { continue; } charsSpare.copyUTF8Bytes(spare); trie.put(charsSpare.toString(), Long.valueOf(weight)); count++; } }
WFSTInputIterator(InputIterator source) throws IOException { super(source); assert source.hasPayloads() == false; }
@Override public void build(InputIterator iterator) throws IOException { if (iterator.hasPayloads()) { throw new IllegalArgumentException("this suggester doesn't support payloads"); } if (iterator.hasContexts()) { throw new IllegalArgumentException("this suggester doesn't support contexts"); } root = new TernaryTreeNode(); // make sure it's sorted and the comparator uses UTF16 sort order iterator = new SortedInputIterator(iterator, BytesRef.getUTF8SortedAsUTF16Comparator()); count = 0; ArrayList<String> tokens = new ArrayList<>(); ArrayList<Number> vals = new ArrayList<>(); BytesRef spare; CharsRefBuilder charsSpare = new CharsRefBuilder(); while ((spare = iterator.next()) != null) { charsSpare.copyUTF8Bytes(spare); tokens.add(charsSpare.toString()); vals.add(Long.valueOf(iterator.weight())); count++; } autocomplete.balancedTree(tokens.toArray(), vals.toArray(), 0, tokens.size() - 1, root); }
while ((text = iter.next()) != null) { BytesRef payload; if (iter.hasPayloads()) { payload = iter.payload(); } else { payload = null; add(text, iter.contexts(), iter.weight(), payload);
@Override public void build(InputIterator iterator) throws IOException { if (iterator.hasPayloads()) { throw new IllegalArgumentException("this suggester doesn't support payloads"); if (iterator.hasContexts()) { throw new IllegalArgumentException("this suggester doesn't support contexts"); PositiveIntOutputs outputs = PositiveIntOutputs.getSingleton(); Builder<Long> builder = new Builder<>(FST.INPUT_TYPE.BYTE1, outputs); while ((scratch = iter.next()) != null) { long cost = iter.weight();
@Override public void build(InputIterator iterator) throws IOException { if (iterator.hasPayloads()) { throw new IllegalArgumentException("this suggester doesn't support payloads"); if (iterator.hasContexts()) { throw new IllegalArgumentException("this suggester doesn't support contexts"); ByteArrayDataOutput output = new ByteArrayDataOutput(buffer); BytesRef spare; while ((spare = iterator.next()) != null) { if (spare.length + 4 >= buffer.length) { buffer = ArrayUtil.grow(buffer, spare.length + 4); output.writeInt(encodeWeight(iterator.weight())); output.writeBytes(spare.bytes, spare.offset, spare.length); writer.write(buffer, 0, output.getPosition());
@Override public void build(InputIterator iterator) throws IOException { if (iterator.hasContexts()) { throw new IllegalArgumentException("this suggester doesn't support contexts"); Path tempSorted = Files.createTempFile(directory, prefix, ".sorted"); hasPayloads = iterator.hasPayloads(); ByteArrayDataOutput output = new ByteArrayDataOutput(buffer); for (BytesRef surfaceForm; (surfaceForm = iterator.next()) != null;) { LimitedFiniteStringsIterator finiteStrings = new LimitedFiniteStringsIterator(toAutomaton(surfaceForm, ts2a), maxGraphExpansions); throw new IllegalArgumentException("cannot handle surface form > " + (Short.MAX_VALUE-2) + " in length (got " + surfaceForm.length + ")"); payload = iterator.payload(); output.writeInt(encodeWeight(iterator.weight()));
Path tempSorted = Files.createTempFile(directory, prefix, ".sorted"); hasPayloads = iterator.hasPayloads(); ByteArrayDataOutput output = new ByteArrayDataOutput(buffer); for (BytesRef surfaceForm; (surfaceForm = iterator.next()) != null;) { LimitedFiniteStringsIterator finiteStrings = new LimitedFiniteStringsIterator(toAutomaton(surfaceForm, ts2a), maxGraphExpansions); throw new IllegalArgumentException("cannot handle surface form > " + (Short.MAX_VALUE-2) + " in length (got " + surfaceForm.length + ")"); payload = iterator.payload(); output.writeInt(encodeWeight(iterator.weight()));
OfflineSorter.ByteSequencesReader reader = null; hasPayloads = iterator.hasPayloads(); ByteArrayDataOutput output = new ByteArrayDataOutput(buffer); for (BytesRef surfaceForm; (surfaceForm = iterator.next()) != null;) { LimitedFiniteStringsIterator finiteStrings = new LimitedFiniteStringsIterator(toAutomaton(surfaceForm, ts2a), maxGraphExpansions); "cannot handle surface form > " + (Short.MAX_VALUE-2) + " in length (got " + surfaceForm.length + ")"); payload = iterator.payload(); output.writeInt(encodeWeight(iterator.weight()));
OfflineSorter.ByteSequencesReader reader = null; hasPayloads = iterator.hasPayloads(); ByteArrayDataOutput output = new ByteArrayDataOutput(buffer); for (BytesRef surfaceForm; (surfaceForm = iterator.next()) != null;) { LimitedFiniteStringsIterator finiteStrings = new LimitedFiniteStringsIterator(toAutomaton(surfaceForm, ts2a), maxGraphExpansions); "cannot handle surface form > " + (Short.MAX_VALUE-2) + " in length (got " + surfaceForm.length + ")"); payload = iterator.payload(); output.writeInt(encodeWeight(iterator.weight()));