index = indexBuilder.finish();
public FST<Pair<Long, BytesRef>> build() throws IOException { return builder.finish(); }
private FST<CharsRef> parseConversions(LineNumberReader reader, int num) throws IOException, ParseException { Map<String,String> mappings = new TreeMap<>(); for (int i = 0; i < num; i++) { String line = reader.readLine(); String parts[] = line.split("\\s+"); if (parts.length != 3) { throw new ParseException("invalid syntax: " + line, reader.getLineNumber()); } if (mappings.put(parts[1], parts[2]) != null) { throw new IllegalStateException("duplicate mapping specified for: " + parts[1]); } } Outputs<CharsRef> outputs = CharSequenceOutputs.getSingleton(); Builder<CharsRef> builder = new Builder<>(FST.INPUT_TYPE.BYTE2, outputs); IntsRefBuilder scratchInts = new IntsRefBuilder(); for (Map.Entry<String,String> entry : mappings.entrySet()) { Util.toUTF16(entry.getKey(), scratchInts); builder.add(scratchInts.get(), new CharsRef(entry.getValue())); } return builder.finish(); }
private FST<IntsRef> affixFST(TreeMap<String,List<Integer>> affixes) throws IOException { IntSequenceOutputs outputs = IntSequenceOutputs.getSingleton(); Builder<IntsRef> builder = new Builder<>(FST.INPUT_TYPE.BYTE4, outputs); IntsRefBuilder scratch = new IntsRefBuilder(); for (Map.Entry<String,List<Integer>> entry : affixes.entrySet()) { Util.toUTF32(entry.getKey(), scratch); List<Integer> entries = entry.getValue(); IntsRef output = new IntsRef(entries.size()); for (Integer c : entries) { output.ints[output.length++] = c; } builder.add(scratch.get(), output); } return builder.finish(); }
/** Builds the NormalizeCharMap; call this once you * are done calling {@link #add}. */ public NormalizeCharMap build() { final FST<CharsRef> map; try { final Outputs<CharsRef> outputs = CharSequenceOutputs.getSingleton(); final org.apache.lucene.util.fst.Builder<CharsRef> builder = new org.apache.lucene.util.fst.Builder<>(FST.INPUT_TYPE.BYTE2, outputs); final IntsRefBuilder scratch = new IntsRefBuilder(); for(Map.Entry<String,String> ent : pendingPairs.entrySet()) { builder.add(Util.toUTF16(ent.getKey(), scratch), new CharsRef(ent.getValue())); } map = builder.finish(); pendingPairs.clear(); } catch (IOException ioe) { // Bogus FST IOExceptions!! (will never happen) throw new RuntimeException(ioe); } return new NormalizeCharMap(map); } }
Builder<IntsRef> b = new Builder<>(FST.INPUT_TYPE.BYTE4, o); readDictionaryFiles(tempDir, tempFileNamePrefix, dictionaries, decoder, b); words = b.finish();
FST<BytesRef> fst = builder.finish(); return new SynonymMap(fst, words, maxHorizontalContext);
public FST<Pair<Long, BytesRef>> build() throws IOException { return builder.finish(); }
public FST<Pair<Long, BytesRef>> build() throws IOException { return builder.finish(); }
public FST<Pair<Long, BytesRef>> build() throws IOException { return builder.finish(); }
public FST<Pair<Long, BytesRef>> build() throws IOException { return builder.finish(); }
/** * Returns an {@link StemmerOverrideMap} to be used with the {@link StemmerOverrideFilter} * @return an {@link StemmerOverrideMap} to be used with the {@link StemmerOverrideFilter} * @throws IOException if an {@link IOException} occurs; */ public StemmerOverrideMap build() throws IOException { ByteSequenceOutputs outputs = ByteSequenceOutputs.getSingleton(); org.apache.lucene.util.fst.Builder<BytesRef> builder = new org.apache.lucene.util.fst.Builder<>( FST.INPUT_TYPE.BYTE4, outputs); final int[] sort = hash.sort(); IntsRefBuilder intsSpare = new IntsRefBuilder(); final int size = hash.size(); BytesRef spare = new BytesRef(); for (int i = 0; i < size; i++) { int id = sort[i]; BytesRef bytesRef = hash.get(id, spare); intsSpare.copyUTF8Bytes(bytesRef); builder.add(intsSpare.get(), new BytesRef(outputValues.get(id))); } return new StemmerOverrideMap(builder.finish(), ignoreCase); }
@Override public void finish(long termsFilePointer) throws IOException { fst = fstBuilder.finish(); if (fst != null) { fst.save(out); } } }
public void finish(long sumTotalTermFreq, long sumDocFreq, int docCount) throws IOException { // save FST dict if (numTerms > 0) { final FST<FSTTermOutputs.TermData> fst = builder.finish(); fields.add(new FieldMetaData(fieldInfo, numTerms, sumTotalTermFreq, sumDocFreq, docCount, longsSize, fst)); } } }
public void finish(long sumTotalTermFreq, long sumDocFreq, int docCount) throws IOException { if (numTerms > 0) { final FieldMetaData metadata = new FieldMetaData(); metadata.fieldInfo = fieldInfo; metadata.numTerms = numTerms; metadata.sumTotalTermFreq = sumTotalTermFreq; metadata.sumDocFreq = sumDocFreq; metadata.docCount = docCount; metadata.longsSize = longsSize; metadata.skipOut = skipOut; metadata.statsOut = statsOut; metadata.metaLongsOut = metaLongsOut; metadata.metaBytesOut = metaBytesOut; metadata.dict = builder.finish(); fields.add(metadata); } }
fst = builder.finish();
private void updateFST(SortedMap<String, Double> weights) throws IOException { PositiveIntOutputs outputs = PositiveIntOutputs.getSingleton(); Builder<Long> fstBuilder = new Builder<>(FST.INPUT_TYPE.BYTE1, outputs); BytesRefBuilder scratchBytes = new BytesRefBuilder(); IntsRefBuilder scratchInts = new IntsRefBuilder(); for (Map.Entry<String, Double> entry : weights.entrySet()) { scratchBytes.copyChars(entry.getKey()); fstBuilder.add(Util.toIntsRef(scratchBytes.get(), scratchInts), entry .getValue().longValue()); } fst = fstBuilder.finish(); }
private FST<IntsRef> affixFST(TreeMap<String,List<Integer>> affixes) throws IOException { IntSequenceOutputs outputs = IntSequenceOutputs.getSingleton(); Builder<IntsRef> builder = new Builder<>(FST.INPUT_TYPE.BYTE4, outputs); IntsRefBuilder scratch = new IntsRefBuilder(); for (Map.Entry<String,List<Integer>> entry : affixes.entrySet()) { Util.toUTF32(entry.getKey(), scratch); List<Integer> entries = entry.getValue(); IntsRef output = new IntsRef(entries.size()); for (Integer c : entries) { output.ints[output.length++] = c; } builder.add(scratch.get(), output); } return builder.finish(); }
private static void serialize(String file, BytesRef [] all) throws IOException { final Object nothing = NoOutputs.getSingleton().getNoOutput(); final Builder<Object> builder = new Builder<Object>(INPUT_TYPE.BYTE4, NoOutputs.getSingleton()); final IntsRefBuilder intsRef = new IntsRefBuilder(); for (BytesRef br : all) { intsRef.clear(); intsRef.copyUTF8Bytes(br); builder.add(intsRef.get(), nothing); } final FST<Object> fst = builder.finish(); final OutputStreamDataOutput out = new OutputStreamDataOutput(new FileOutputStream(file)); fst.save(out); out.close(); } }
public void finish(long sumTotalTermFreq, long sumDocFreq, int docCount) throws IOException { if (termCount > 0) { out.writeVInt(termCount); out.writeVInt(field.number); if (field.getIndexOptions() != IndexOptions.DOCS) { out.writeVLong(sumTotalTermFreq); } out.writeVLong(sumDocFreq); out.writeVInt(docCount); FST<BytesRef> fst = builder.finish(); fst.save(out); //System.out.println("finish field=" + field.name + " fp=" + out.getFilePointer()); } } }