/** In-place read; returns the arc. */ public Arc<T> readNextArc(Arc<T> arc, BytesReader in) throws IOException { if (arc.label == END_LABEL) { // This was a fake inserted "final" arc if (arc.nextArc <= 0) { throw new IllegalArgumentException("cannot readNextArc when arc.isLast()=true"); } return readFirstRealTargetArc(arc.nextArc, arc, in); } else { return readNextRealArc(arc, in); } }
/** * Follow the <code>follow</code> arc and read the first arc of its target; * this changes the provided <code>arc</code> (2nd arg) in-place and returns * it. * * @return Returns the second argument (<code>arc</code>). */ public Arc<T> readFirstTargetArc(Arc<T> follow, Arc<T> arc, BytesReader in) throws IOException { //int pos = address; //System.out.println(" readFirstTarget follow.target=" + follow.target + " isFinal=" + follow.isFinal()); if (follow.isFinal()) { // Insert "fake" final first arc: arc.label = END_LABEL; arc.output = follow.nextFinalOutput; arc.flags = BIT_FINAL_ARC; if (follow.target <= 0) { arc.flags |= BIT_LAST_ARC; } else { // NOTE: nextArc is a node (not an address!) in this case: arc.nextArc = follow.target; } arc.target = FINAL_END_NODE; //System.out.println(" insert isFinal; nextArc=" + follow.target + " isLast=" + arc.isLast() + " output=" + outputs.outputToString(arc.output)); return arc; } else { return readFirstRealTargetArc(follow.target, arc, in); } }
private long hash(long node) throws IOException { final int PRIME = 31; //System.out.println("hash frozen node=" + node); long h = 0; fst.readFirstRealTargetArc(node, scratchArc, in); while(true) { //System.out.println(" label=" + scratchArc.label + " target=" + scratchArc.target + " h=" + h + " output=" + fst.outputs.outputToString(scratchArc.output) + " next?=" + scratchArc.flag(4) + " final?=" + scratchArc.isFinal() + " pos=" + in.getPosition()); h = PRIME * h + scratchArc.label; h = PRIME * h + (int) (scratchArc.target^(scratchArc.target>>32)); h = PRIME * h + scratchArc.output.hashCode(); h = PRIME * h + scratchArc.nextFinalOutput.hashCode(); if (scratchArc.isFinal()) { h += 17; } if (scratchArc.isLast()) { break; } fst.readNextRealArc(scratchArc, in); } //System.out.println(" ret " + (h&Integer.MAX_VALUE)); return h & Long.MAX_VALUE; }
private boolean nodesEqual(Builder.UnCompiledNode<T> node, long address) throws IOException { fst.readFirstRealTargetArc(address, scratchArc, in); if (scratchArc.bytesPerArc != 0 && node.numArcs != scratchArc.numArcs) { return false; } for(int arcUpto=0;arcUpto<node.numArcs;arcUpto++) { final Builder.Arc<T> arc = node.arcs[arcUpto]; if (arc.label != scratchArc.label || !arc.output.equals(scratchArc.output) || ((Builder.CompiledNode) arc.target).node != scratchArc.target || !arc.nextFinalOutput.equals(scratchArc.nextFinalOutput) || arc.isFinal != scratchArc.isFinal()) { return false; } if (scratchArc.isLast()) { if (arcUpto == node.numArcs-1) { return true; } else { return false; } } fst.readNextRealArc(scratchArc, in); } return false; }
fst.readFirstRealTargetArc(arc.target, arc, r);
final BytesReader in = getBytesReader(); Arc<T>[] arcs = (Arc<T>[]) new Arc[0x80]; readFirstRealTargetArc(arc.target, arc, in); int count = 0; while(true) {
fst.readFirstRealTargetArc(follow.target, arc, in);
readFirstRealTargetArc(follow.target, arc, in);
fst.readFirstRealTargetArc(arc.target, arc, in);
private NormalizeCharMap(FST<CharsRef> map) { this.map = map; if (map != null) { try { // Pre-cache root arcs: final FST.Arc<CharsRef> scratchArc = new FST.Arc<>(); final FST.BytesReader fstReader = map.getBytesReader(); map.getFirstArc(scratchArc); if (FST.targetHasArcs(scratchArc)) { map.readFirstRealTargetArc(scratchArc.target, scratchArc, fstReader); while(true) { assert scratchArc.label != FST.END_LABEL; cachedRootArcs.put(Character.valueOf((char) scratchArc.label), new FST.Arc<CharsRef>().copyFrom(scratchArc)); if (scratchArc.isLast()) { break; } map.readNextRealArc(scratchArc, fstReader); } } //System.out.println("cached " + cachedRootArcs.size() + " root arcs"); } catch (IOException ioe) { // Bogus FST IOExceptions!! (will never happen) throw new RuntimeException(ioe); } } }
/** In-place read; returns the arc. */ public Arc<T> readNextArc(Arc<T> arc, BytesReader in) throws IOException { if (arc.label == END_LABEL) { // This was a fake inserted "final" arc if (arc.nextArc <= 0) { throw new IllegalArgumentException("cannot readNextArc when arc.isLast()=true"); } return readFirstRealTargetArc(arc.nextArc, arc, in); } else { return readNextRealArc(arc, in); } }
/** In-place read; returns the arc. */ public Arc<T> readNextArc(Arc<T> arc, BytesReader in) throws IOException { if (arc.label == END_LABEL) { // This was a fake inserted "final" arc if (arc.nextArc <= 0) { throw new IllegalArgumentException("cannot readNextArc when arc.isLast()=true"); } return readFirstRealTargetArc(arc.nextArc, arc, in); } else { return readNextRealArc(arc, in); } }
/** In-place read; returns the arc. */ public Arc<T> readNextArc(Arc<T> arc, BytesReader in) throws IOException { if (arc.label == END_LABEL) { // This was a fake inserted "final" arc if (arc.nextArc <= 0) { throw new IllegalArgumentException("cannot readNextArc when arc.isLast()=true"); } return readFirstRealTargetArc(arc.nextArc, arc, in); } else { return readNextRealArc(arc, in); } }
private boolean nodesEqual(Builder.UnCompiledNode<T> node, long address) throws IOException { fst.readFirstRealTargetArc(address, scratchArc, in); if (scratchArc.bytesPerArc != 0 && node.numArcs != scratchArc.numArcs) { return false; } for(int arcUpto=0;arcUpto<node.numArcs;arcUpto++) { final Builder.Arc<T> arc = node.arcs[arcUpto]; if (arc.label != scratchArc.label || !arc.output.equals(scratchArc.output) || ((Builder.CompiledNode) arc.target).node != scratchArc.target || !arc.nextFinalOutput.equals(scratchArc.nextFinalOutput) || arc.isFinal != scratchArc.isFinal()) { return false; } if (scratchArc.isLast()) { if (arcUpto == node.numArcs-1) { return true; } else { return false; } } fst.readNextRealArc(scratchArc, in); } return false; }
private long hash(long node) throws IOException { final int PRIME = 31; //System.out.println("hash frozen node=" + node); long h = 0; fst.readFirstRealTargetArc(node, scratchArc, in); while(true) { //System.out.println(" label=" + scratchArc.label + " target=" + scratchArc.target + " h=" + h + " output=" + fst.outputs.outputToString(scratchArc.output) + " next?=" + scratchArc.flag(4) + " final?=" + scratchArc.isFinal() + " pos=" + in.getPosition()); h = PRIME * h + scratchArc.label; h = PRIME * h + (int) (scratchArc.target^(scratchArc.target>>32)); h = PRIME * h + scratchArc.output.hashCode(); h = PRIME * h + scratchArc.nextFinalOutput.hashCode(); if (scratchArc.isFinal()) { h += 17; } if (scratchArc.isLast()) { break; } fst.readNextRealArc(scratchArc, in); } //System.out.println(" ret " + (h&Integer.MAX_VALUE)); return h & Long.MAX_VALUE; }
private long hash(long node) throws IOException { final int PRIME = 31; //System.out.println("hash frozen node=" + node); long h = 0; fst.readFirstRealTargetArc(node, scratchArc, in); while(true) { //System.out.println(" label=" + scratchArc.label + " target=" + scratchArc.target + " h=" + h + " output=" + fst.outputs.outputToString(scratchArc.output) + " next?=" + scratchArc.flag(4) + " final?=" + scratchArc.isFinal() + " pos=" + in.getPosition()); h = PRIME * h + scratchArc.label; h = PRIME * h + (int) (scratchArc.target^(scratchArc.target>>32)); h = PRIME * h + scratchArc.output.hashCode(); h = PRIME * h + scratchArc.nextFinalOutput.hashCode(); if (scratchArc.isFinal()) { h += 17; } if (scratchArc.isLast()) { break; } fst.readNextRealArc(scratchArc, in); } //System.out.println(" ret " + (h&Integer.MAX_VALUE)); return h & Long.MAX_VALUE; }
private long hash(long node) throws IOException { final int PRIME = 31; //System.out.println("hash frozen node=" + node); long h = 0; fst.readFirstRealTargetArc(node, scratchArc, in); while(true) { //System.out.println(" label=" + scratchArc.label + " target=" + scratchArc.target + " h=" + h + " output=" + fst.outputs.outputToString(scratchArc.output) + " next?=" + scratchArc.flag(4) + " final?=" + scratchArc.isFinal() + " pos=" + in.getPosition()); h = PRIME * h + scratchArc.label; h = PRIME * h + (int) (scratchArc.target^(scratchArc.target>>32)); h = PRIME * h + scratchArc.output.hashCode(); h = PRIME * h + scratchArc.nextFinalOutput.hashCode(); if (scratchArc.isFinal()) { h += 17; } if (scratchArc.isLast()) { break; } fst.readNextRealArc(scratchArc, in); } //System.out.println(" ret " + (h&Integer.MAX_VALUE)); return h & Long.MAX_VALUE; }
/** Load frame for target arc(node) on fst */ Frame loadExpandFrame(Frame top, Frame frame) throws IOException { if (!canGrow(top)) { return null; } frame.fstArc = fst.readFirstRealTargetArc(top.fstArc.target, frame.fstArc, fstReader); frame.fsaState = fsa.step(top.fsaState, frame.fstArc.label); //if (TEST) System.out.println(" loadExpand frame="+frame); if (frame.fsaState == -1) { return loadNextFrame(top, frame); } return frame; }
/** Load frame for target arc(node) on fst */ Frame loadExpandFrame(Frame top, Frame frame) throws IOException { if (!canGrow(top)) { return null; } frame.arc = fst.readFirstRealTargetArc(top.arc.target, frame.arc, fstReader); frame.state = fsa.step(top.state, frame.arc.label); //if (TEST) System.out.println(" loadExpand frame="+frame); if (frame.state == -1) { return loadNextFrame(top, frame); } return frame; }
static<T> void walk(FST<T> fst) throws IOException { final ArrayList<FST.Arc<T>> queue = new ArrayList<>(); final BitSet seen = new BitSet(); final FST.BytesReader reader = fst.getBytesReader(); final FST.Arc<T> startArc = fst.getFirstArc(new FST.Arc<T>()); queue.add(startArc); while (!queue.isEmpty()) { final FST.Arc<T> arc = queue.remove(0); final long node = arc.target; //System.out.println(arc); if (FST.targetHasArcs(arc) && !seen.get((int) node)) { seen.set((int) node); fst.readFirstRealTargetArc(node, arc, reader); while (true) { queue.add(new FST.Arc<T>().copyFrom(arc)); if (arc.isLast()) { break; } else { fst.readNextRealArc(arc, reader); } } } } }