Code example for BreakIterator

Methods: currentnextsetText

0
  } 
   
  @Override 
  public boolean incrementToken() throws IOException {
    if (hasMoreTokensInClone) {
      int start = breaker.current();
      int end = breaker.next();
      if (end != BreakIterator.DONE) {
        clonedToken.copyTo(this);
        termAtt.copyBuffer(clonedTermAtt.buffer(), start, end - start);
        offsetAtt.setOffset(clonedOffsetAtt.startOffset() + start, clonedOffsetAtt.startOffset() + end);
        if (handlePosIncr) posAtt.setPositionIncrement(1);
        return true; 
      } 
      hasMoreTokensInClone = false;
    } 
 
    if (!input.incrementToken()) { 
      return false; 
    } 
     
    if (termAtt.length() == 0 || UnicodeBlock.of(termAtt.charAt(0)) != UnicodeBlock.THAI) {
      return true; 
    } 
     
    hasMoreTokensInClone = true;
 
    // we lazy init the cloned token, as in ctor not all attributes may be added 
    if (clonedToken == null) {
      clonedToken = cloneAttributes();
      clonedTermAtt = clonedToken.getAttribute(CharTermAttribute.class);
      clonedOffsetAtt = clonedToken.getAttribute(OffsetAttribute.class);
    } else { 
      this.copyTo(clonedToken);
    } 
     
    // reinit CharacterIterator 
    charIterator.setText(clonedTermAtt.buffer(), 0, clonedTermAtt.length());
    breaker.setText(charIterator);
    int end = breaker.next();
    if (end != BreakIterator.DONE) {
      termAtt.setLength(end);
      offsetAtt.setOffset(clonedOffsetAtt.startOffset(), clonedOffsetAtt.startOffset() + end);
      // position increment keeps as it is for first token 
      return true;