/** * Convenience method to check if the current type is a gram type * * @return {@code true} if the current type is a gram type, {@code false} otherwise */ public boolean isGramType() { return GRAM_TYPE.equals(typeAttribute.type()); } }
/** * By default accept the token if its type is not a stop type. * When the useWhiteList parameter is set to true then accept the token if its type is contained in the stopTypes */ @Override protected boolean accept() { return useWhiteList == stopTypes.contains(typeAttribute.type()); } }
@Override public final boolean incrementToken() throws IOException { if (input.incrementToken()) { if (typeAtt.type().equals(typeMatch)) payloadAtt.setPayload(thePayload); return true; } else { return false; } } }
@Override public final boolean incrementToken() throws IOException { if (input.incrementToken()) { String type = typeAtt.type(); if (type != null && !type.isEmpty()) { payloadAtt.setPayload(new BytesRef(type)); } return true; } else { return false; } } }
@Override protected boolean accept() throws IOException { if ( termAtt.length() < min) { String type = typeAtt.type(); if (type == CJKBigramFilter.DOUBLE_TYPE || type == CJKBigramFilter.SINGLE_TYPE) { return true; } } return termAtt.length() >= min && termAtt.length() <= max; } }
final String type = typeAtt.type();
@Override public void nextToken() throws IOException { anyTokens = true; BytesRef term = fillBytesRef(termsRef); if (requireUnigram && typeAttribute.type() == ShingleFilter.DEFAULT_TOKEN_TYPE) { return; } anyUnigram = true; if (posIncAttr.getPositionIncrement() == 0 && typeAttribute.type() == SynonymFilter.TYPE_SYNONYM) { assert currentSet != null; long freq = 0; if ((freq = generator.frequency(term)) > 0) { currentSet.addOneCandidate(generator.createCandidate(BytesRef.deepCopyOf(term), freq, realWordLikelihood)); } } else { if (currentSet != null) { candidateSetsList.add(currentSet); } currentSet = new CandidateSet(Candidate.EMPTY, generator.createCandidate(BytesRef.deepCopyOf(term), true)); } }
@Override public boolean incrementToken() throws IOException { if (savedToken != null) { // Emit last token's type at the same position restoreState(savedToken); savedToken = null; termAtt.setEmpty(); if (prefix != null) { termAtt.append(prefix); } termAtt.append(typeAtt.type()); posIncrAtt.setPositionIncrement(0); return true; } else if (input.incrementToken()) { // Ho pending token type to emit savedToken = captureState(); return true; } return false; }
/** * Saves the existing attribute states */ private void saveState() { // otherwise, we have delimiters, save state savedStartOffset = offsetAttribute.startOffset(); savedEndOffset = offsetAttribute.endOffset(); // if length by start + end offsets doesn't match the term text then assume this is a synonym and don't adjust the offsets. hasIllegalOffsets = (savedEndOffset - savedStartOffset != termAttribute.length()); savedType = typeAttribute.type(); if (savedBuffer.length < termAttribute.length()) { savedBuffer = new char[ArrayUtil.oversize(termAttribute.length(), Character.BYTES)]; } System.arraycopy(termAttribute.buffer(), 0, savedBuffer, 0, termAttribute.length()); iterator.text = savedBuffer; hasSavedState = true; }
restoreState(previous); previous = current; previousType = typeAttribute.type();
lastOffset + offset.endOffset(), posLen.getPositionLength(), type.type(), null));
private void analyze(TokenStream stream, Analyzer analyzer, String field, Set<String> includeAttributes) { try { stream.reset(); CharTermAttribute term = stream.addAttribute(CharTermAttribute.class); PositionIncrementAttribute posIncr = stream.addAttribute(PositionIncrementAttribute.class); OffsetAttribute offset = stream.addAttribute(OffsetAttribute.class); TypeAttribute type = stream.addAttribute(TypeAttribute.class); PositionLengthAttribute posLen = stream.addAttribute(PositionLengthAttribute.class); while (stream.incrementToken()) { int increment = posIncr.getPositionIncrement(); if (increment > 0) { lastPosition = lastPosition + increment; } tokens.add(new AnalyzeResponse.AnalyzeToken(term.toString(), lastPosition, lastOffset + offset.startOffset(), lastOffset + offset.endOffset(), posLen.getPositionLength(), type.type(), extractExtendedAttributes(stream, includeAttributes))); } stream.end(); lastOffset += offset.endOffset(); lastPosition += posIncr.getPositionIncrement(); lastPosition += analyzer.getPositionIncrementGap(field); lastOffset += analyzer.getOffsetGap(field); } catch (IOException e) { throw new ElasticsearchException("failed to analyze", e); } finally { IOUtils.closeWhileHandlingException(stream); } }
/** * By default accept the token if its type is not a stop type. * When the useWhiteList parameter is set to true then accept the token if its type is contained in the stopTypes */ @Override protected boolean accept() { return useWhiteList == stopTypes.contains(typeAttribute.type()); } }
@Override public final boolean incrementToken() throws IOException { if (input.incrementToken()) { if (typeAtt.type().equals(typeMatch)) payloadAtt.setPayload(thePayload); return true; } else { return false; } } }
@Override public boolean accept(AttributeSource source) { if (typeAtt == null) { typeAtt = source.addAttribute(TypeAttribute.class); } //check to see if this is a Category return (typeToMatch.equals(typeAtt.type())); }
String type = typeAtt.type(); if (type == doHan || type == doHiragana || type == doKatakana || type == doHangul) {
private Token getNextSuffixInputToken(Token token) throws IOException { if (!suffix.incrementToken()) return null; token.copyBuffer(termAtt.buffer(), 0, termAtt.length()); token.setPositionIncrement(posIncrAtt.getPositionIncrement()); token.setFlags(flagsAtt.getFlags()); token.setOffset(offsetAtt.startOffset(), offsetAtt.endOffset()); token.setType(typeAtt.type()); token.setPayload(payloadAtt.getPayload()); return token; }
private Token getNextPrefixInputToken(Token token) throws IOException { if (!prefix.incrementToken()) return null; token.copyBuffer(p_termAtt.buffer(), 0, p_termAtt.length()); token.setPositionIncrement(p_posIncrAtt.getPositionIncrement()); token.setFlags(p_flagsAtt.getFlags()); token.setOffset(p_offsetAtt.startOffset(), p_offsetAtt.endOffset()); token.setType(p_typeAtt.type()); token.setPayload(p_payloadAtt.getPayload()); return token; }
private Token getNextInputToken(Token token) throws IOException { if (!input.incrementToken()) return null; token.copyBuffer(in_termAtt.buffer(), 0, in_termAtt.length()); token.setPositionIncrement(in_posIncrAtt.getPositionIncrement()); token.setFlags(in_flagsAtt.getFlags()); token.setOffset(in_offsetAtt.startOffset(), in_offsetAtt.endOffset()); token.setType(in_typeAtt.type()); token.setPayload(in_payloadAtt.getPayload()); return token; }
private Token getNextToken(Token token) throws IOException { if (!this.incrementToken()) return null; token.copyBuffer(termAtt.buffer(), 0, termAtt.length()); token.setPositionIncrement(posIncrAtt.getPositionIncrement()); token.setFlags(flagsAtt.getFlags()); token.setOffset(offsetAtt.startOffset(), offsetAtt.endOffset()); token.setType(typeAtt.type()); token.setPayload(payloadAtt.getPayload()); return token; }