public TextAnnotation preProcess(String text) throws AnnotatorException { String[] tokens = tokenizer.tokenizeSentence(text).getFirst(); return preProcess(Collections.singletonList(tokens)); }
public TextAnnotation createTextAnnotation(String corpusId, String textId, String text) throws IllegalArgumentException { Tokenizer.Tokenization tokenization = tokenizer.tokenizeTextSpan(text); TextAnnotation ta = new TextAnnotation(corpusId, textId, text, tokenization.getCharacterOffsets(), tokenization.getTokens(), tokenization.getSentenceEndTokenIndexes());
public TextAnnotation createTextAnnotation(String corpusId, String textId, String text) throws IllegalArgumentException { Tokenizer.Tokenization tokenization = tokenizer.tokenizeTextSpan(text); TextAnnotation ta = new TextAnnotation(corpusId, textId, text, tokenization.getCharacterOffsets(), tokenization.getTokens(), tokenization.getSentenceEndTokenIndexes());
public TextAnnotation preProcess(String text) throws AnnotatorException { String[] tokens = tokenizer.tokenizeSentence(text).getFirst(); return preProcess(Collections.singletonList(tokens)); }
Pair<String[], IntPair[]> toks = tokenizer.tokenizeSentence(s.text);
Pair<String[], IntPair[]> toks = tokenizer.tokenizeSentence(s.text);