/** Looks up the output for this input, or null if the * input is not accepted */ public static<T> T get(FST<T> fst, BytesRef input) throws IOException { assert fst.inputType == FST.INPUT_TYPE.BYTE1; final BytesReader fstReader = fst.getBytesReader(); // TODO: would be nice not to alloc this on every lookup final FST.Arc<T> arc = fst.getFirstArc(new FST.Arc<T>()); // Accumulate output as we go T output = fst.outputs.getNoOutput(); for(int i=0;i<input.length;i++) { if (fst.findTargetArc(input.bytes[i+input.offset] & 0xFF, arc, arc, fstReader) == null) { return null; } output = fst.outputs.add(output, arc.output); } if (arc.isFinal()) { return fst.outputs.add(output, arc.nextFinalOutput); } else { return null; } }
@Override public String outputToString(Pair<A,B> output) { assert valid(output); return "<pair:" + outputs1.outputToString(output.output1) + "," + outputs2.outputToString(output.output2) + ">"; }
private long ramBytesUsed(Arc<T>[] arcs) { long size = 0; if (arcs != null) { size += RamUsageEstimator.shallowSizeOf(arcs); for (Arc<T> arc : arcs) { if (arc != null) { size += ARC_SHALLOW_RAM_BYTES_USED; if (arc.output != null && arc.output != outputs.getNoOutput()) { size += outputs.ramBytesUsed(arc.output); } if (arc.nextFinalOutput != null && arc.nextFinalOutput != outputs.getNoOutput()) { size += outputs.ramBytesUsed(arc.nextFinalOutput); } } } } return size; }
public void prependOutput(T outputPrefix) { assert owner.validOutput(outputPrefix); for(int arcIdx=0;arcIdx<numArcs;arcIdx++) { arcs[arcIdx].output = owner.fst.outputs.add(outputPrefix, arcs[arcIdx].output); assert owner.validOutput(arcs[arcIdx].output); } if (isFinal) { output = owner.fst.outputs.add(outputPrefix, output); assert owner.validOutput(output); } } }
final T NO_OUTPUT = fst.outputs.getNoOutput(); final BytesReader r = fst.getBytesReader(); emitDotState(out, Long.toString(startArc.target), isFinal ? finalStateShape : stateShape, stateColor, finalOutput == null ? "" : fst.outputs.outputToString(finalOutput)); finalOutput = fst.outputs.outputToString(arc.nextFinalOutput); } else { finalOutput = ""; outs = "/" + fst.outputs.outputToString(arc.output); } else { outs = ""; outs = outs + "/[" + fst.outputs.outputToString(arc.nextFinalOutput) + "]";
/** Fills virtual 'start' arc, ie, an empty incoming arc to * the FST's start node */ public Arc<T> getFirstArc(Arc<T> arc) { T NO_OUTPUT = outputs.getNoOutput(); if (emptyOutput != null) { arc.flags = BIT_FINAL_ARC | BIT_LAST_ARC; arc.nextFinalOutput = emptyOutput; if (emptyOutput != NO_OUTPUT) { arc.flags |= BIT_ARC_HAS_FINAL_OUTPUT; } } else { arc.flags = BIT_LAST_ARC; arc.nextFinalOutput = NO_OUTPUT; } arc.output = NO_OUTPUT; // If there are no nodes, ie, the FST only accepts the // empty string, then startNode is 0 arc.target = startNode; return arc; }
arc.output = outputs.read(in); } else { arc.output = outputs.getNoOutput(); arc.nextFinalOutput = outputs.readFinalOutput(in); } else { arc.nextFinalOutput = outputs.getNoOutput();
long addNode(Builder<T> builder, Builder.UnCompiledNode<T> nodeIn) throws IOException { T NO_OUTPUT = outputs.getNoOutput(); outputs.write(arc.output, builder.bytes); outputs.writeFinalOutput(arc.nextFinalOutput, builder.bytes);
commonOutputPrefix = fst.outputs.common(output, lastOutput); assert validOutput(commonOutputPrefix); wordSuffix = fst.outputs.subtract(lastOutput, commonOutputPrefix); assert validOutput(wordSuffix); parentNode.setLastOutput(input.ints[input.offset + idx - 1], commonOutputPrefix); output = fst.outputs.subtract(output, commonOutputPrefix); assert validOutput(output); lastNode.output = fst.outputs.merge(lastNode.output, output); } else {
/** Decode an output value previously written with {@link * #writeFinalOutput(Object, DataOutput)}. By default this * just calls {@link #read(DataInput)}. */ public T readFinalOutput(DataInput in) throws IOException { return read(in); }
/** Encode an final node output value into a {@link * DataOutput}. By default this just calls {@link #write(Object, * DataOutput)}. */ public void writeFinalOutput(T output, DataOutput out) throws IOException { write(output, out); }
@Override public void skipOutput(DataInput in) throws IOException { outputs1.skipOutput(in); outputs2.skipOutput(in); }
@Override public long ramBytesUsed(Pair<A,B> output) { long ramBytesUsed = BASE_NUM_BYTES; if (output.output1 != null) { ramBytesUsed += outputs1.ramBytesUsed(output.output1); } if (output.output2 != null) { ramBytesUsed += outputs2.ramBytesUsed(output.output2); } return ramBytesUsed; } }
private void seekToNextNode(BytesReader in) throws IOException { while(true) { final int flags = in.readByte(); readLabel(in); if (flag(flags, BIT_ARC_HAS_OUTPUT)) { outputs.skipOutput(in); } if (flag(flags, BIT_ARC_HAS_FINAL_OUTPUT)) { outputs.skipFinalOutput(in); } if (!flag(flags, BIT_STOP_NODE) && !flag(flags, BIT_TARGET_NEXT)) { readUnpackedNodeTarget(in); } if (flag(flags, BIT_LAST_ARC)) { return; } } }
private long ramBytesUsed(Arc<T>[] arcs) { long size = 0; if (arcs != null) { size += RamUsageEstimator.shallowSizeOf(arcs); for (Arc<T> arc : arcs) { if (arc != null) { size += ARC_SHALLOW_RAM_BYTES_USED; if (arc.output != null && arc.output != outputs.getNoOutput()) { size += outputs.ramBytesUsed(arc.output); } if (arc.nextFinalOutput != null && arc.nextFinalOutput != outputs.getNoOutput()) { size += outputs.ramBytesUsed(arc.nextFinalOutput); } } } } return size; }
final T NO_OUTPUT = fst.outputs.getNoOutput(); final BytesReader r = fst.getBytesReader(); emitDotState(out, Long.toString(startArc.target), isFinal ? finalStateShape : stateShape, stateColor, finalOutput == null ? "" : fst.outputs.outputToString(finalOutput)); finalOutput = fst.outputs.outputToString(arc.nextFinalOutput); } else { finalOutput = ""; outs = "/" + fst.outputs.outputToString(arc.output); } else { outs = ""; outs = outs + "/[" + fst.outputs.outputToString(arc.nextFinalOutput) + "]";
private boolean valid(Pair<A,B> pair) { final boolean noOutput1 = pair.output1.equals(outputs1.getNoOutput()); final boolean noOutput2 = pair.output2.equals(outputs2.getNoOutput()); if (noOutput1 && pair.output1 != outputs1.getNoOutput()) { return false; } if (noOutput2 && pair.output2 != outputs2.getNoOutput()) { return false; } if (noOutput1 && noOutput2) { if (pair != NO_OUTPUT) { return false; } else { return true; } } else { return true; } }