nextLevelQueue.add(new FST.Arc<T>().copyFrom(arc)); sameLevelStates.add((int) arc.target);
arc.copyFrom(firstArc); CharsRef output = NO_OUTPUT; longestMatch = -1;
static<T> void walk(FST<T> fst) throws IOException { final ArrayList<FST.Arc<T>> queue = new ArrayList<>(); final BitSet seen = new BitSet(); final FST.BytesReader reader = fst.getBytesReader(); final FST.Arc<T> startArc = fst.getFirstArc(new FST.Arc<T>()); queue.add(startArc); while (!queue.isEmpty()) { final FST.Arc<T> arc = queue.remove(0); final long node = arc.target; //System.out.println(arc); if (FST.targetHasArcs(arc) && !seen.get((int) node)) { seen.set((int) node); fst.readFirstRealTargetArc(node, arc, reader); while (true) { queue.add(new FST.Arc<T>().copyFrom(arc)); if (arc.isLast()) { break; } else { fst.readNextRealArc(arc, reader); } } } } }
public void putTerm(Term term) throws IOException { appendToSequences(term); FST.Arc<BytesRef> scratchArc = new FST.Arc<>(); fst.getFirstArc(scratchArc); BytesRef pendingOutput = fst.outputs.getNoOutput(); FST.BytesReader fstReader = fst.getBytesReader(); boolean ok = true; CharSequence termValue = term.getValue(); for (int pos = 0, len = termValue.length(); ok && (pos < len);) { int codePoint = Character.codePointAt(termValue, pos); ok = null != fst.findTargetArc(codePoint, scratchArc, scratchArc, fstReader); pendingOutput = fst.outputs.add(pendingOutput, scratchArc.output); pos += Character.charCount(codePoint); } if (ok) { List<Term> terms = Collections.singletonList(term); FST.Arc<BytesRef> arc = new FST.Arc<>(); addSequences.add(new Sequence(arc.copyFrom(scratchArc), terms, pendingOutput)); if (scratchArc.isFinal()) { addOutput(fst.outputs.add(pendingOutput, scratchArc.nextFinalOutput)); } } }
static<T> void walk(FST<T> fst) throws IOException { final ArrayList<FST.Arc<T>> queue = new ArrayList<>(); final BitSet seen = new BitSet(); final FST.BytesReader reader = fst.getBytesReader(); final FST.Arc<T> startArc = fst.getFirstArc(new FST.Arc<T>()); queue.add(startArc); while (!queue.isEmpty()) { final FST.Arc<T> arc = queue.remove(0); final long node = arc.target; //System.out.println(arc); if (FST.targetHasArcs(arc) && !seen.get((int) node)) { seen.set((int) node); fst.readFirstRealTargetArc(node, arc, reader); while (true) { queue.add(new FST.Arc<T>().copyFrom(arc)); if (arc.isLast()) { break; } else { fst.readNextRealArc(arc, reader); } } } } }
/** * Cache the root node's output arcs starting with completions with the * highest weights. */ @SuppressWarnings({"all"}) private static Arc<Object>[] cacheRootArcs(FST<Object> automaton) { try { List<Arc<Object>> rootArcs = new ArrayList<Arc<Object>>(); Arc<Object> arc = automaton.getFirstArc(new Arc<Object>()); automaton.readFirstTargetArc(arc, arc); while (true) { rootArcs.add(new Arc<Object>().copyFrom(arc)); if (arc.isLast()) break; automaton.readNextArc(arc); } Collections.reverse(rootArcs); // we want highest weights first. return rootArcs.toArray(new Arc[rootArcs.size()]); } catch (IOException e) { throw new RuntimeException(e); } }
/** * Cache the root node's output arcs starting with completions with the highest weights. */ @SuppressWarnings("unchecked") private void cacheRootArcs() throws IOException { if (automaton != null) { List<Arc<Object>> rootArcs = new ArrayList<Arc<Object>>(); Arc<Object> arc = automaton.getFirstArc(new Arc<Object>()); automaton.readFirstTargetArc(arc, arc); while (true) { rootArcs.add(new Arc<Object>().copyFrom(arc)); if (arc.isLast()) break; automaton.readNextArc(arc); } Collections.reverse(rootArcs); // we want highest weights first. this.rootArcs = rootArcs.toArray(new Arc[rootArcs.size()]); } }
@SuppressWarnings({"rawtypes","unchecked"}) private FST.Arc<Long>[] cacheRootArcs() throws IOException { FST.Arc<Long> rootCache[] = new FST.Arc[1+(cacheCeiling-0xAC00)]; FST.Arc<Long> firstArc = new FST.Arc<>(); fst.getFirstArc(firstArc); FST.Arc<Long> arc = new FST.Arc<>(); final FST.BytesReader fstReader = fst.getBytesReader(); // TODO: jump to AC00, readNextRealArc to ceiling? (just be careful we don't add bugs) for (int i = 0; i < rootCache.length; i++) { if (fst.findTargetArc(0xAC00 + i, firstArc, arc, fstReader) != null) { rootCache[i] = new FST.Arc<Long>().copyFrom(arc); } } return rootCache; }
public FST.Arc<Long> findTargetArc(int ch, FST.Arc<Long> follow, FST.Arc<Long> arc, boolean useCache, FST.BytesReader fstReader) throws IOException { if (useCache && ch >= 0x3040 && ch <= cacheCeiling) { assert ch != FST.END_LABEL; final Arc<Long> result = rootCache[ch - 0x3040]; if (result == null) { return null; } else { arc.copyFrom(result); return arc; } } else { return fst.findTargetArc(ch, follow, arc, fstReader); } }
public FST.Arc<Long> findTargetArc(int ch, FST.Arc<Long> follow, FST.Arc<Long> arc, boolean useCache, FST.BytesReader fstReader) throws IOException { if (useCache && ch >= 0xAC00 && ch <= cacheCeiling) { assert ch != FST.END_LABEL; final Arc<Long> result = rootCache[ch - 0xAC00]; if (result == null) { return null; } else { arc.copyFrom(result); return arc; } } else { return fst.findTargetArc(ch, follow, arc, fstReader); } }
@SuppressWarnings({"rawtypes","unchecked"}) private FST.Arc<Long>[] cacheRootArcs() throws IOException { FST.Arc<Long> rootCache[] = new FST.Arc[1+(cacheCeiling-0x3040)]; FST.Arc<Long> firstArc = new FST.Arc<>(); fst.getFirstArc(firstArc); FST.Arc<Long> arc = new FST.Arc<>(); final FST.BytesReader fstReader = fst.getBytesReader(); // TODO: jump to 3040, readNextRealArc to ceiling? (just be careful we don't add bugs) for (int i = 0; i < rootCache.length; i++) { if (fst.findTargetArc(0x3040 + i, firstArc, arc, fstReader) != null) { rootCache[i] = new FST.Arc<Long>().copyFrom(arc); } } return rootCache; }
public FSTPath(T cost, FST.Arc<T> arc, IntsRefBuilder input, float boost, CharSequence context) { this.arc = new FST.Arc<T>().copyFrom(arc); this.cost = cost; this.input = input; this.boost = boost; this.context = context; }
public FSTPath(T cost, FST.Arc<T> arc, IntsRefBuilder input, float boost, CharSequence context) { this.arc = new FST.Arc<T>().copyFrom(arc); this.cost = cost; this.input = input; this.boost = boost; this.context = context; }
public FSTPath(T output, FST.Arc<T> arc, IntsRefBuilder input, float boost, CharSequence context, int payload) { this.arc = new FST.Arc<T>().copyFrom(arc); this.output = output; this.input = input; this.boost = boost; this.context = context; this.payload = payload; }
private NormalizeCharMap(FST<CharsRef> map) { this.map = map; if (map != null) { try { // Pre-cache root arcs: final FST.Arc<CharsRef> scratchArc = new FST.Arc<>(); final FST.BytesReader fstReader = map.getBytesReader(); map.getFirstArc(scratchArc); if (FST.targetHasArcs(scratchArc)) { map.readFirstRealTargetArc(scratchArc.target, scratchArc, fstReader); while(true) { assert scratchArc.label != FST.END_LABEL; cachedRootArcs.put(Character.valueOf((char) scratchArc.label), new FST.Arc<CharsRef>().copyFrom(scratchArc)); if (scratchArc.isLast()) { break; } map.readNextRealArc(scratchArc, fstReader); } } //System.out.println("cached " + cachedRootArcs.size() + " root arcs"); } catch (IOException ioe) { // Bogus FST IOExceptions!! (will never happen) throw new RuntimeException(ioe); } } }
return null; } else { arc.copyFrom(result); return arc;
assert arc.label != END_LABEL; if (arc.label < arcs.length) { arcs[arc.label] = new Arc<T>().copyFrom(arc); } else { break;
public FSTPath(T output, FST.Arc<T> arc, IntsRefBuilder input, float boost, CharSequence context, int payload) { this.arc = new FST.Arc<T>().copyFrom(arc); this.output = output; this.input = input; this.boost = boost; this.context = context; this.payload = payload; }