if (arc.isLast()) {
private long hash(long node) throws IOException { final int PRIME = 31; //System.out.println("hash frozen node=" + node); long h = 0; fst.readFirstRealTargetArc(node, scratchArc, in); while(true) { //System.out.println(" label=" + scratchArc.label + " target=" + scratchArc.target + " h=" + h + " output=" + fst.outputs.outputToString(scratchArc.output) + " next?=" + scratchArc.flag(4) + " final?=" + scratchArc.isFinal() + " pos=" + in.getPosition()); h = PRIME * h + scratchArc.label; h = PRIME * h + (int) (scratchArc.target^(scratchArc.target>>32)); h = PRIME * h + scratchArc.output.hashCode(); h = PRIME * h + scratchArc.nextFinalOutput.hashCode(); if (scratchArc.isFinal()) { h += 17; } if (scratchArc.isLast()) { break; } fst.readNextRealArc(scratchArc, in); } //System.out.println(" ret " + (h&Integer.MAX_VALUE)); return h & Long.MAX_VALUE; }
private boolean nodesEqual(Builder.UnCompiledNode<T> node, long address) throws IOException { fst.readFirstRealTargetArc(address, scratchArc, in); if (scratchArc.bytesPerArc != 0 && node.numArcs != scratchArc.numArcs) { return false; } for(int arcUpto=0;arcUpto<node.numArcs;arcUpto++) { final Builder.Arc<T> arc = node.arcs[arcUpto]; if (arc.label != scratchArc.label || !arc.output.equals(scratchArc.output) || ((Builder.CompiledNode) arc.target).node != scratchArc.target || !arc.nextFinalOutput.equals(scratchArc.nextFinalOutput) || arc.isFinal != scratchArc.isFinal()) { return false; } if (scratchArc.isLast()) { if (arcUpto == node.numArcs-1) { return true; } else { return false; } } fst.readNextRealArc(scratchArc, in); } return false; }
/** Load frame for sibling arc(node) on fst */ Frame loadNextFrame(Frame top, Frame frame) throws IOException { if (!canRewind(frame)) { return null; } while (!frame.arc.isLast()) { frame.arc = fst.readNextRealArc(frame.arc, fstReader); frame.state = fsa.step(top.state, frame.arc.label); if (frame.state != -1) { break; } } //if (TEST) System.out.println(" loadNext frame="+frame); if (frame.state == -1) { return null; } return frame; }
/** * Cache the root node's output arcs starting with completions with the highest weights. */ @SuppressWarnings("unchecked") private void cacheRootArcs() throws IOException { if (automaton != null) { List<Arc<Object>> rootArcs = new ArrayList<Arc<Object>>(); Arc<Object> arc = automaton.getFirstArc(new Arc<Object>()); automaton.readFirstTargetArc(arc, arc); while (true) { rootArcs.add(new Arc<Object>().copyFrom(arc)); if (arc.isLast()) break; automaton.readNextArc(arc); } Collections.reverse(rootArcs); // we want highest weights first. this.rootArcs = rootArcs.toArray(new Arc[rootArcs.size()]); } }
/** Load frame for sibling arc(node) on fst */ Frame loadNextFrame(Frame top, Frame frame) throws IOException { if (!canRewind(frame)) { return null; } while (!frame.fstArc.isLast()) { frame.fstArc = fst.readNextRealArc(frame.fstArc, fstReader); frame.fsaState = fsa.step(top.fsaState, frame.fstArc.label); if (frame.fsaState != -1) { break; } } //if (TEST) System.out.println(" loadNext frame="+frame); if (frame.fsaState == -1) { return null; } return frame; }
boolean canRewind(Frame frame) { // can jump to sibling return !frame.arc.isLast(); }
boolean canRewind(Frame frame) { // can jump to sibling return !frame.fstArc.isLast(); }
private NormalizeCharMap(FST<CharsRef> map) { this.map = map; if (map != null) { try { // Pre-cache root arcs: final FST.Arc<CharsRef> scratchArc = new FST.Arc<>(); final FST.BytesReader fstReader = map.getBytesReader(); map.getFirstArc(scratchArc); if (FST.targetHasArcs(scratchArc)) { map.readFirstRealTargetArc(scratchArc.target, scratchArc, fstReader); while(true) { assert scratchArc.label != FST.END_LABEL; cachedRootArcs.put(Character.valueOf((char) scratchArc.label), new FST.Arc<CharsRef>().copyFrom(scratchArc)); if (scratchArc.isLast()) { break; } map.readNextRealArc(scratchArc, fstReader); } } //System.out.println("cached " + cachedRootArcs.size() + " root arcs"); } catch (IOException ioe) { // Bogus FST IOExceptions!! (will never happen) throw new RuntimeException(ioe); } } }
while(!arc.isLast() && fst.readNextArcLabel(arc, in) < targetLabel) { fst.readNextArc(arc, fstReader); assert arc.isLast() || fst.readNextArcLabel(arc, in) > targetLabel; assert arc.label < targetLabel: "arc.label=" + arc.label + " vs targetLabel=" + targetLabel; pushLast(); while(!arc.isLast() && fst.readNextArcLabel(arc, fstReader) < targetLabel) { fst.readNextArc(arc, fstReader); arc = getArc(upto); } else if (!arc.isLast()) {
} else if (arc.isLast()) {
} else if (arc.label > labelToMatch) { return null; } else if (arc.isLast()) { return null; } else {
} else if (arc.isLast()) { return null; } else {
addIfCompetitive(path); if (path.arc.isLast()) { break;
assert !arc.isLast();
break; if (arc.isLast()) { break;
protected void doNext() throws IOException { //System.out.println("FE: next upto=" + upto); if (upto == 0) { //System.out.println(" init"); upto = 1; fst.readFirstTargetArc(getArc(0), getArc(1), fstReader); } else { // pop //System.out.println(" check pop curArc target=" + arcs[upto].target + " label=" + arcs[upto].label + " isLast?=" + arcs[upto].isLast()); while (arcs[upto].isLast()) { upto--; if (upto == 0) { //System.out.println(" eof"); return; } } fst.readNextArc(arcs[upto], fstReader); } pushFirst(); }
/** Adds all leaving arcs, including 'finished' arc, if * the node is final, from this node into the queue. */ public void addStartPaths(FST.Arc<T> node, T startOutput, boolean allowEmptyString, IntsRefBuilder input, float boost, CharSequence context, int payload) throws IOException { // De-dup NO_OUTPUT since it must be a singleton: if (startOutput.equals(fst.outputs.getNoOutput())) { startOutput = fst.outputs.getNoOutput(); } FSTPath<T> path = new FSTPath<>(startOutput, node, input, boost, context, payload); fst.readFirstTargetArc(node, path.arc, bytesReader); // Bootstrap: find the min starting arc while (true) { if (allowEmptyString || path.arc.label != FST.END_LABEL) { addIfCompetitive(path); } if (path.arc.isLast()) { break; } fst.readNextArc(path.arc, bytesReader); } }