/** * @param input input tokenstream * @param synonyms synonym map * @param ignoreCase case-folds input for matching with {@link Character#toLowerCase(int)}. * Note, if you set this to true, its your responsibility to lowercase * the input entries when you create the {@link SynonymMap} */ public SynonymFilter(TokenStream input, SynonymMap synonyms, boolean ignoreCase) { super(input); this.synonyms = synonyms; this.ignoreCase = ignoreCase; this.fst = synonyms.fst; this.fstReader = fst.getBytesReader(0); if (fst == null) { throw new IllegalArgumentException("fst must be non-null"); } // Must be 1+ so that when roll buffer is at full // lookahead we can distinguish this full buffer from // the empty buffer: rollBufferSize = 1+synonyms.maxHorizontalContext; futureInputs = new PendingInput[rollBufferSize]; futureOutputs = new PendingOutputs[rollBufferSize]; for(int pos=0;pos<rollBufferSize;pos++) { futureInputs[pos] = new PendingInput(); futureOutputs[pos] = new PendingOutputs(); } //System.out.println("FSTFilt maxH=" + synonyms.maxHorizontalContext); scratchArc = new FST.Arc<BytesRef>(); }
final CharsRef output = outputs.pullNext(); clearAttributes(); termAtt.copyBuffer(output.chars, output.offset, output.length); typeAtt.setType(TYPE_SYNONYM); int endOffset = outputs.getLastEndOffset(); if (endOffset == -1) { endOffset = input.endOffset; posLenAtt.setPositionLength(outputs.getLastPosLength()); if (outputs.count == 0) { if (outputs.upto < outputs.count) { final int posIncr = outputs.posIncr; final CharsRef output = outputs.pullNext(); futureInputs[nextRead].reset(); if (outputs.count == 0) {
final CharsRef output = outputs.pullNext(); clearAttributes(); termAtt.copyBuffer(output.chars, output.offset, output.length); typeAtt.setType(TYPE_SYNONYM); int endOffset = outputs.getLastEndOffset(); if (endOffset == -1) { endOffset = input.endOffset; posLenAtt.setPositionLength(outputs.getLastPosLength()); if (outputs.count == 0) { if (outputs.upto < outputs.count) { final int posIncr = outputs.posIncr; final CharsRef output = outputs.pullNext(); futureInputs[nextRead].reset(); if (outputs.count == 0) {
posLen = 1; futureOutputs[outputUpto].add(scratchChars.chars(), lastStart, outputLen, endOffset, posLen);
posLen = 1; futureOutputs[outputUpto].add(scratchChars.chars, lastStart, outputLen, endOffset, posLen);
/** * @param input input tokenstream * @param synonyms synonym map * @param ignoreCase case-folds input for matching with {@link Character#toLowerCase(int)}. * Note, if you set this to true, it's your responsibility to lowercase * the input entries when you create the {@link SynonymMap} */ public SynonymFilter(TokenStream input, SynonymMap synonyms, boolean ignoreCase) { super(input); this.synonyms = synonyms; this.ignoreCase = ignoreCase; this.fst = synonyms.fst; if (fst == null) { throw new IllegalArgumentException("fst must be non-null"); } this.fstReader = fst.getBytesReader(); // Must be 1+ so that when roll buffer is at full // lookahead we can distinguish this full buffer from // the empty buffer: rollBufferSize = 1+synonyms.maxHorizontalContext; futureInputs = new PendingInput[rollBufferSize]; futureOutputs = new PendingOutputs[rollBufferSize]; for(int pos=0;pos<rollBufferSize;pos++) { futureInputs[pos] = new PendingInput(); futureOutputs[pos] = new PendingOutputs(); } //System.out.println("FSTFilt maxH=" + synonyms.maxHorizontalContext); scratchArc = new FST.Arc<>(); }
@Override public void reset() throws IOException { super.reset(); captureCount = 0; finished = false; inputSkipCount = 0; nextRead = nextWrite = 0; // In normal usage these resets would not be needed, // since they reset-as-they-are-consumed, but the app // may not consume all input tokens (or we might hit an // exception), in which case we have leftover state // here: for (PendingInput input : futureInputs) { input.reset(); } for (PendingOutputs output : futureOutputs) { output.reset(); } } }
@Override public void reset() throws IOException { super.reset(); captureCount = 0; finished = false; inputSkipCount = 0; nextRead = nextWrite = 0; // In normal usage these resets would not be needed, // since they reset-as-they-are-consumed, but the app // may not consume all input tokens (or we might hit an // exception), in which case we have leftover state // here: for (PendingInput input : futureInputs) { input.reset(); } for (PendingOutputs output : futureOutputs) { output.reset(); } } }
public CharsRef pullNext() { assert upto < count; lastEndOffset = endOffsets[upto]; lastPosLength = posLengths[upto]; final CharsRef result = outputs[upto++]; posIncr = 0; if (upto == count) { reset(); } return result; }
public CharsRef pullNext() { assert upto < count; lastEndOffset = endOffsets[upto]; lastPosLength = posLengths[upto]; final CharsRefBuilder result = outputs[upto++]; posIncr = 0; if (upto == count) { reset(); } return result.get(); }
final CharsRef output = outputs.pullNext(); clearAttributes(); termAtt.copyBuffer(output.chars, output.offset, output.length); typeAtt.setType(TYPE_SYNONYM); int endOffset = outputs.getLastEndOffset(); if (endOffset == -1) { endOffset = input.endOffset; posLenAtt.setPositionLength(outputs.getLastPosLength()); if (outputs.count == 0) { if (outputs.upto < outputs.count) { final int posIncr = outputs.posIncr; final CharsRef output = outputs.pullNext(); futureInputs[nextRead].reset(); if (outputs.count == 0) {
posLen = 1; futureOutputs[outputUpto].add(scratchChars.chars(), lastStart, outputLen, endOffset, posLen);
/** * @param input input tokenstream * @param synonyms synonym map * @param ignoreCase case-folds input for matching with {@link Character#toLowerCase(int)}. * Note, if you set this to true, it's your responsibility to lowercase * the input entries when you create the {@link SynonymMap} */ public SynonymFilter(TokenStream input, SynonymMap synonyms, boolean ignoreCase) { super(input); this.synonyms = synonyms; this.ignoreCase = ignoreCase; this.fst = synonyms.fst; if (fst == null) { throw new IllegalArgumentException("fst must be non-null"); } this.fstReader = fst.getBytesReader(); // Must be 1+ so that when roll buffer is at full // lookahead we can distinguish this full buffer from // the empty buffer: rollBufferSize = 1+synonyms.maxHorizontalContext; futureInputs = new PendingInput[rollBufferSize]; futureOutputs = new PendingOutputs[rollBufferSize]; for(int pos=0;pos<rollBufferSize;pos++) { futureInputs[pos] = new PendingInput(); futureOutputs[pos] = new PendingOutputs(); } //System.out.println("FSTFilt maxH=" + synonyms.maxHorizontalContext); scratchArc = new FST.Arc<>(); }
@Override public void reset() throws IOException { super.reset(); captureCount = 0; finished = false; inputSkipCount = 0; nextRead = nextWrite = 0; // In normal usage these resets would not be needed, // since they reset-as-they-are-consumed, but the app // may not consume all input tokens (or we might hit an // exception), in which case we have leftover state // here: for (PendingInput input : futureInputs) { input.reset(); } for (PendingOutputs output : futureOutputs) { output.reset(); } } }
public CharsRef pullNext() { assert upto < count; lastEndOffset = endOffsets[upto]; lastPosLength = posLengths[upto]; final CharsRefBuilder result = outputs[upto++]; posIncr = 0; if (upto == count) { reset(); } return result.get(); }