Refine search
@Override public final boolean incrementToken() throws IOException { if (input.incrementToken()) { CharacterUtils.toLowerCase(termAtt.buffer(), 0, termAtt.length()); return true; } else return false; } }
private SToken[] getTokens(String text) throws IOException { //FIXME somehow integrate below cycle to getSummary to save the cloning and memory, //also creating Tokens is suboptimal with 3.0.0 , this whole class could be replaced by highlighter ArrayList<SToken> result = new ArrayList<>(); try (TokenStream ts = analyzer.tokenStream("full", text)) { CharTermAttribute term = ts.addAttribute(CharTermAttribute.class); OffsetAttribute offset = ts.addAttribute(OffsetAttribute.class); ts.reset(); while (ts.incrementToken()) { SToken t = new SToken(term.buffer(), 0, term.length(), offset.startOffset(), offset.endOffset()); result.add(t); } ts.end(); } return result.toArray(new SToken[result.size()]); }
final LinkedList<String> tokens = getTokens(); final CharTermAttribute termAtt = getTermAtt(); if (tokens.isEmpty() && input.incrementToken()) { final String text = new String(termAtt.buffer(), 0, termAtt.length()); if (UrlStringUtils.containsUrl(text)) { final String[] parts = text.split("\\s");
String[] parts; skipCounter = 0; while (input.incrementToken()) { final String text = new String(termAtt.buffer(), 0, termAtt.length()); if (text.isEmpty()) { return true;
return true; } else if (input.incrementToken()) { final String word = new String(termAtt.buffer(), 0, termAtt.length()); if (word.isEmpty()) { return true;
@Override public final boolean incrementToken() throws IOException { if (input.incrementToken()) { CharacterUtils.toUpperCase(termAtt.buffer(), 0, termAtt.length()); return true; } else return false; } }
@Override public boolean incrementToken() throws IOException { if (input.incrementToken()) { char[] chArray = termAtt.buffer(); int chLen = termAtt.length(); for (int i = 0; i < chLen;) { i += Character.toChars( lowerCase(Character.codePointAt(chArray, i, chLen)), chArray, i); } return true; } else { return false; } }
@Override public final boolean incrementToken() throws IOException { if (!input.incrementToken()) return false; final char[] buffer = termAtt.buffer(); final int length = termAtt.length(); for (int i = 0; i < length; i++) if (buffer[i] == '\'' || buffer[i] == '\u2019') { termAtt.setLength(i); return true; } return true; } }
@Override public boolean accept() { final int max32 = termAtt.length(); final int min32 = max32 >> 1; if (min32 >= min && max32 <= max) { // definitely within range return true; } else if (min32 > max || max32 < min) { // definitely not return false; } else { // we must count to be sure int len = Character.codePointCount(termAtt.buffer(), 0, termAtt.length()); return (len >= min && len <= max); } } }
@Override public boolean incrementToken() throws IOException { if (!input.incrementToken()) { return false; } final char[] buffer = termAtt.buffer(); final int bufferLength = termAtt.length(); if (bufferLength >= 2 && (buffer[bufferLength-2] == '\'' || buffer[bufferLength-2] == '\u2019' || buffer[bufferLength-2] == '\uFF07') && (buffer[bufferLength-1] == 's' || buffer[bufferLength-1] == 'S')) { termAtt.setLength(bufferLength - 2); // Strip last 2 characters off } return true; } }
@Override public boolean incrementToken() throws IOException { if (input.incrementToken()) { int newlen = normalizer.normalize(termAtt.buffer(), termAtt.length()); termAtt.setLength(newlen); return true; } return false; } }
private void capture() { captureCount++; //System.out.println(" capture slot=" + nextWrite); final PendingInput input = futureInputs[nextWrite]; input.state = captureState(); input.consumed = false; input.term.copyChars(termAtt.buffer(), 0, termAtt.length()); nextWrite = rollIncr(nextWrite); // Buffer head should never catch up to tail: assert nextWrite != nextRead; }