private static String toString(Token token, String indent) { StringBuilder builder = new StringBuilder(); builder.append(indent).append("components : {\n"); for (int i = 0, len = token.getNumComponents(); i < len; ++i) { Token comp = token.getComponent(i); builder.append(indent).append(" [").append(i).append("] : ").append(comp.getClass().getSimpleName()); builder.append(" {\n").append(toString(comp, indent + " ")); builder.append(indent).append(" }\n"); } builder.append(indent).append("}\n"); builder.append(indent).append("offset : ").append(token.getOffset()).append("\n"); builder.append(indent).append("orig : ").append(quoteString(token.getOrig())).append("\n"); builder.append(indent).append("script : ").append(token.getScript()).append("\n"); builder.append(indent).append("special : ").append(token.isSpecialToken()).append("\n"); builder.append(indent).append("token string : ").append(quoteString(token.getTokenString())).append("\n"); builder.append(indent).append("type : ").append(token.getType()).append("\n"); return builder.toString(); }
private void findSegments(Token token, List<String> out) { int len; if (token.isSpecialToken() || (len = token.getNumComponents()) == 0) { if (token.isIndexable()) { out.add(token.getOrig()); } } else { for (int i = 0; i < len; ++i) { findSegments(token.getComponent(i), out); } } }
private void findStems(Token token, List<StemList> out) { int len; if (token.isSpecialToken() || (len = token.getNumComponents()) == 0) { if (token.isIndexable()) { StemList word = new StemList(); word.add(token.getTokenString()); // takes care of getStem(0) for (int i = 1; i < token.getNumStems(); i++) { word.add(token.getStem(i)); } out.add(word); } } else { for (int i = 0; i < len; ++i) { findStems(token.getComponent(i), out); } } } }
if (token.getNumComponents() > 0) { for (int i = 0; i < token.getNumComponents(); ++i) { addAnnotationSpan(input, parent, tokenizer, token.getComponent(i), mode, termOccurrences);
if (!equalsOpt(getComponent(i), rhs.getComponent(i))) { return false;