private void append(Builder<BytesRef> builder, FST<BytesRef> subIndex, IntsRefBuilder scratchIntsRef) throws IOException { final BytesRefFSTEnum<BytesRef> subIndexEnum = new BytesRefFSTEnum<>(subIndex); BytesRefFSTEnum.InputOutput<BytesRef> indexEnt; while((indexEnt = subIndexEnum.next()) != null) { //if (DEBUG) { // System.out.println(" add sub=" + indexEnt.input + " " + indexEnt.input + " output=" + indexEnt.output); //} builder.add(Util.toIntsRef(indexEnt.input, scratchIntsRef), indexEnt.output); } } }
/** doFloor controls the behavior of advance: if it's true * doFloor is true, advance positions to the biggest * term before target. */ protected FSTEnum(FST<T> fst) { this.fst = fst; fstReader = fst.getBytesReader(); NO_OUTPUT = fst.outputs.getNoOutput(); fst.getFirstArc(getArc(0)); output[0] = NO_OUTPUT; }
private void pushLast() throws IOException { FST.Arc<T> arc = arcs[upto]; assert arc != null; while (true) { setCurrentLabel(arc.label); output[upto] = fst.outputs.add(output[upto-1], arc.output); if (arc.label == FST.END_LABEL) { // Final node break; } incr(); arc = fst.readLastTargetArc(arc, getArc(upto), fstReader); } }
public PairOutputs(Outputs<A> outputs1, Outputs<B> outputs2) { this.outputs1 = outputs1; this.outputs2 = outputs2; NO_OUTPUT = new Pair<>(outputs1.getNoOutput(), outputs2.getNoOutput()); }
/** * Checks if <code>arc</code>'s target state is in expanded (or vector) format. * * @return Returns <code>true</code> if <code>arc</code> points to a state in an * expanded array format. */ boolean isExpandedTarget(Arc<T> follow, BytesReader in) throws IOException { if (!targetHasArcs(follow)) { return false; } else { in.setPosition(follow.target); return in.readByte() == ARCS_AS_FIXED_ARRAY; } }
@Override public String outputToString(Pair<A,B> output) { assert valid(output); return "<pair:" + outputs1.outputToString(output.output1) + "," + outputs2.outputToString(output.output2) + ">"; }
/** Seeks to biggest term that's <= target. */ public InputOutput<T> seekFloor(IntsRef target) throws IOException { this.target = target; targetLength = target.length; super.doSeekFloor(); return setResult(); }
/** Seeks to biggest term that's <= target. */ public InputOutput<T> seekFloor(BytesRef target) throws IOException { this.target = target; targetLength = target.length; super.doSeekFloor(); return setResult(); }
/** Seeks to smallest term that's >= target. */ public InputOutput<T> seekCeil(IntsRef target) throws IOException { this.target = target; targetLength = target.length; super.doSeekCeil(); return setResult(); }
/** Seeks to smallest term that's >= target. */ public InputOutput<T> seekCeil(BytesRef target) throws IOException { this.target = target; targetLength = target.length; super.doSeekCeil(); return setResult(); }
/** Returns a {@link BytesReader} for this FST, positioned at * position 0. */ public BytesReader getBytesReader() { if (bytesArray != null) { return new ReverseBytesReader(bytesArray); } else { return bytes.getReverseReader(); } }
FST(INPUT_TYPE inputType, Outputs<T> outputs, int bytesPageBits) { this.inputType = inputType; this.outputs = outputs; version = VERSION_CURRENT; bytesArray = null; bytes = new BytesStore(bytesPageBits); // pad: ensure no node gets address 0 which is reserved to mean // the stop state w/ no arcs bytes.writeByte((byte) 0); emptyOutput = null; }
@Override public long ramBytesUsed(Pair<A,B> output) { long ramBytesUsed = BASE_NUM_BYTES; if (output.output1 != null) { ramBytesUsed += outputs1.ramBytesUsed(output.output1); } if (output.output2 != null) { ramBytesUsed += outputs2.ramBytesUsed(output.output2); } return ramBytesUsed; } }
/** * Creates an unbounded TopNSearcher * @param fst the {@link org.apache.lucene.util.fst.FST} to search on * @param topN the number of top scoring entries to retrieve * @param maxQueueDepth the maximum size of the queue of possible top entries * @param comparator the comparator to select the top N */ public TopNSearcher(FST<T> fst, int topN, int maxQueueDepth, Comparator<T> comparator) { this(fst, topN, maxQueueDepth, comparator, new TieBreakByInputComparator<>(comparator)); }