Refine search
final Arc<T> arc = new Arc<>(); getFirstArc(arc); if (targetHasArcs(arc)) { final BytesReader in = getBytesReader(); Arc<T>[] arcs = (Arc<T>[]) new Arc[0x80]; readFirstRealTargetArc(arc.target, arc, in); assert arc.label != END_LABEL; if (arc.label < arcs.length) { arcs[arc.label] = new Arc<T>().copyFrom(arc); } else { break; if (arc.isLast()) { break;
/** Looks up the output for this input, or null if the * input is not accepted */ public static<T> T get(FST<T> fst, BytesRef input) throws IOException { assert fst.inputType == FST.INPUT_TYPE.BYTE1; final BytesReader fstReader = fst.getBytesReader(); // TODO: would be nice not to alloc this on every lookup final FST.Arc<T> arc = fst.getFirstArc(new FST.Arc<T>()); // Accumulate output as we go T output = fst.outputs.getNoOutput(); for(int i=0;i<input.length;i++) { if (fst.findTargetArc(input.bytes[i+input.offset] & 0xFF, arc, arc, fstReader) == null) { return null; } output = fst.outputs.add(output, arc.output); } if (arc.isFinal()) { return fst.outputs.add(output, arc.nextFinalOutput); } else { return null; } }
if (!targetHasArcs(follow)) { assert follow.isFinal(); arc.label = END_LABEL; arc.target = FINAL_END_NODE; while(!arc.isLast()) { readLabel(in); if (arc.flag(BIT_ARC_HAS_OUTPUT)) { outputs.skipOutput(in); if (arc.flag(BIT_ARC_HAS_FINAL_OUTPUT)) { outputs.skipFinalOutput(in); if (arc.flag(BIT_STOP_NODE)) { } else if (arc.flag(BIT_TARGET_NEXT)) { } else { readUnpackedNodeTarget(in); readNextRealArc(arc, in); assert arc.isLast(); return arc;
assert arc.isFinal(); output = arc.output; targetUpto = 0; output = BlockTreeTermsReader.FST_OUTPUTS.add(output, arc.output); if (arc.isFinal()) { lastFrame = stack[1+lastFrame.ord]; arc = fr.index.getFirstArc(arcs[0]); assert arc.isFinal(); assert arc.output != null; final FST.Arc<BytesRef> nextArc = fr.index.findTargetArc(targetLabel, arc, getArc(1+targetUpto), fstReader); if (arc.isFinal()) {
final FST.BytesReader in = fst.getBytesReader(); int low = arc.arcIdx; int high = arc.numArcs-1; fst.readNextRealArc(arc, in); assert arc.arcIdx == mid; assert arc.label == targetLabel: "arc.label=" + arc.label + " vs targetLabel=" + targetLabel + " mid=" + mid; while(!arc.isLast() && fst.readNextArcLabel(arc, in) < targetLabel) { fst.readNextArc(arc, fstReader); fst.readNextRealArc(arc, in); assert arc.isLast() || fst.readNextArcLabel(arc, in) > targetLabel; assert arc.label < targetLabel: "arc.label=" + arc.label + " vs targetLabel=" + targetLabel; pushLast(); while(!arc.isLast() && fst.readNextArcLabel(arc, fstReader) < targetLabel) { fst.readNextArc(arc, fstReader); arc = getArc(upto); } else if (!arc.isLast()) { if (fst.readNextArcLabel(arc, fstReader) > targetLabel) {
fstReader = null; } else { fstReader = fr.index.getBytesReader(); arcs[arcIdx] = new FST.Arc<>(); final FST.Arc<BytesRef> arc; if (fr.index != null) { arc = fr.index.getFirstArc(arcs[0]); assert arc.isFinal(); } else { arc = null;
if (follow.isFinal()) { if (follow.target <= 0) { arc.flags = FST.BIT_LAST_ARC; if (!FST.targetHasArcs(follow)) { return null; fst.readFirstTargetArc(follow, arc, in); if (arc.bytesPerArc != 0 && arc.label != FST.END_LABEL) { in.setPosition(arc.posArcsStart); in.skipBytes(arc.bytesPerArc * mid + 1); final int midLabel = fst.readLabel(in); final int cmp = midLabel - label; } else if (arc.isLast()) { return null; } else {
/** * Cache the root node's output arcs starting with completions with the * highest weights. */ @SuppressWarnings({"unchecked","rawtypes"}) private static Arc<Object>[] cacheRootArcs(FST<Object> automaton) { try { List<Arc<Object>> rootArcs = new ArrayList<>(); Arc<Object> arc = automaton.getFirstArc(new Arc<>()); FST.BytesReader fstReader = automaton.getBytesReader(); automaton.readFirstTargetArc(arc, arc, fstReader); while (true) { rootArcs.add(new Arc<>().copyFrom(arc)); if (arc.isLast()) break; automaton.readNextArc(arc, fstReader); } Collections.reverse(rootArcs); // we want highest weights first. return rootArcs.toArray(new Arc[rootArcs.size()]); } catch (IOException e) { throw new RuntimeException(e); } }
/** * Cache the root node's output arcs starting with completions with the highest weights. */ @SuppressWarnings("unchecked") private void cacheRootArcs() throws IOException { if (automaton != null) { List<Arc<Object>> rootArcs = new ArrayList<Arc<Object>>(); Arc<Object> arc = automaton.getFirstArc(new Arc<Object>()); automaton.readFirstTargetArc(arc, arc); while (true) { rootArcs.add(new Arc<Object>().copyFrom(arc)); if (arc.isLast()) break; automaton.readNextArc(arc); } Collections.reverse(rootArcs); // we want highest weights first. this.rootArcs = rootArcs.toArray(new Arc[rootArcs.size()]); } }
BytesReader bytesReader = fst.getBytesReader(); FST.Arc<Pair<Long,BytesRef>> scratchArc = new FST.Arc<>(); if (fst.findTargetArc(endByte, path.fstNode, scratchArc, bytesReader) != null) { if (fst.findTargetArc(endByte, path.fstNode, scratchArc, bytesReader) != null) {
final BytesReader fstReader = fst.getBytesReader(); final T NO_OUTPUT = fst.outputs.getNoOutput(); fst.readFirstTargetArc(path.arc, path.arc, fstReader); break; } else if (!foundZero) { scratchArc.copyFrom(path.arc); foundZero = true; } else { addIfCompetitive(path); if (path.arc.isLast()) { break; fst.readNextArc(path.arc, fstReader); path.arc.copyFrom(scratchArc);
/** Adds all leaving arcs, including 'finished' arc, if * the node is final, from this node into the queue. */ public void addStartPaths(FST.Arc<T> node, T startOutput, boolean allowEmptyString, IntsRefBuilder input, float boost, CharSequence context, int payload) throws IOException { // De-dup NO_OUTPUT since it must be a singleton: if (startOutput.equals(fst.outputs.getNoOutput())) { startOutput = fst.outputs.getNoOutput(); } FSTPath<T> path = new FSTPath<>(startOutput, node, input, boost, context, payload); fst.readFirstTargetArc(node, path.arc, bytesReader); // Bootstrap: find the min starting arc while (true) { if (allowEmptyString || path.arc.label != FST.END_LABEL) { addIfCompetitive(path); } if (path.arc.isLast()) { break; } fst.readNextArc(path.arc, bytesReader); } }
arc = fr.index.findTargetArc(target, arc, getArc(1+idx), fstReader); assert arc != null; output = fstOutputs.add(output, arc.output); assert arc.isFinal(); f.load(fstOutputs.add(output, arc.nextFinalOutput)); return f;
/** * Apply previously built synonyms to incoming tokens. * @param input input tokenstream * @param synonyms synonym map * @param ignoreCase case-folds input for matching with {@link Character#toLowerCase(int)}. * Note, if you set this to true, it's your responsibility to lowercase * the input entries when you create the {@link SynonymMap} */ public SynonymGraphFilter(TokenStream input, SynonymMap synonyms, boolean ignoreCase) { super(input); this.synonyms = synonyms; this.fst = synonyms.fst; if (fst == null) { throw new IllegalArgumentException("fst must be non-null"); } this.fstReader = fst.getBytesReader(); scratchArc = new FST.Arc<>(); this.ignoreCase = ignoreCase; }
stats.indexNumBytes = fr.index.ramBytesUsed(); FST.Arc<BytesRef> arc; if (fr.index != null) { arc = fr.index.getFirstArc(arcs[0]); assert arc.isFinal(); } else { arc = null; arc = fr.index.getFirstArc(arcs[0]); assert arc.isFinal(); } else { arc = null;
arc.label = readLabel(in); if (arc.flag(BIT_ARC_HAS_OUTPUT)) { arc.output = outputs.read(in); } else { if (arc.flag(BIT_ARC_HAS_FINAL_OUTPUT)) { arc.nextFinalOutput = outputs.readFinalOutput(in); } else { if (arc.flag(BIT_STOP_NODE)) { if (arc.flag(BIT_FINAL_ARC)) { arc.target = FINAL_END_NODE; } else { } else if (arc.flag(BIT_TARGET_NEXT)) { arc.nextArc = in.getPosition(); if (!arc.flag(BIT_LAST_ARC)) { if (arc.bytesPerArc == 0) { seekToNextNode(in); } else { in.setPosition(arc.posArcsStart); arc.target = readUnpackedNodeTarget(in); arc.nextArc = in.getPosition();
FST.Arc<CharsRef> arc = cachedRootArcs.get(Character.valueOf((char) firstCH)); if (arc != null) { if (!FST.targetHasArcs(arc)) { assert arc.isFinal(); lastMatchLen = 1; lastMatch = arc.output; lookahead++; if (arc.isFinal()) { if (!FST.targetHasArcs(arc)) { break; break; if ((arc = map.findTargetArc(ch, arc, scratchArc, fstReader)) == null) {
private long hash(long node) throws IOException { final int PRIME = 31; //System.out.println("hash frozen node=" + node); long h = 0; fst.readFirstRealTargetArc(node, scratchArc, in); while(true) { //System.out.println(" label=" + scratchArc.label + " target=" + scratchArc.target + " h=" + h + " output=" + fst.outputs.outputToString(scratchArc.output) + " next?=" + scratchArc.flag(4) + " final?=" + scratchArc.isFinal() + " pos=" + in.getPosition()); h = PRIME * h + scratchArc.label; h = PRIME * h + (int) (scratchArc.target^(scratchArc.target>>32)); h = PRIME * h + scratchArc.output.hashCode(); h = PRIME * h + scratchArc.nextFinalOutput.hashCode(); if (scratchArc.isFinal()) { h += 17; } if (scratchArc.isLast()) { break; } fst.readNextRealArc(scratchArc, in); } //System.out.println(" ret " + (h&Integer.MAX_VALUE)); return h & Long.MAX_VALUE; }
arc = fr.index.getFirstArc(arcs[0]); assert arc.isFinal(); } else { arc = null;
/** Load frame for sibling arc(node) on fst */ Frame loadNextFrame(Frame top, Frame frame) throws IOException { if (!canRewind(frame)) { return null; } while (!frame.fstArc.isLast()) { frame.fstArc = fst.readNextRealArc(frame.fstArc, fstReader); frame.fsaState = fsa.step(top.fsaState, frame.fstArc.label); if (frame.fsaState != -1) { break; } } //if (TEST) System.out.println(" loadNext frame="+frame); if (frame.fsaState == -1) { return null; } return frame; }