/** * A utility method that tokenizes a range of text. */ protected void annotateRange(JCas jcas, int beginPos, int endPos) throws AnalysisEngineProcessException { String text = jcas.getDocumentText().substring(beginPos, endPos); List<Token> tokens = null; try { tokens = tokenizer.tokenizeAndSort(text); } catch (Exception e) { throw new AnalysisEngineProcessException(e); } Iterator<Token> tokenItr = tokens.iterator(); while (tokenItr.hasNext()) { Token token = tokenItr.next(); // convert token into JCas object BaseToken bta = TokenConverter.convert(token, jcas, beginPos); bta.setTokenNumber(tokenCount); // add JCas object to CAS index bta.addToIndexes(); tokenCount++; } } }
/** * A utility method that tokenizes a range of text. */ protected void annotateRange(JCas jcas, int beginPos, int endPos) throws AnalysisEngineProcessException { String text = jcas.getDocumentText().substring(beginPos, endPos); List<Token> tokens = null; try { tokens = tokenizer.tokenizeAndSort(text); } catch (Exception e) { throw new AnalysisEngineProcessException(e); } Iterator<Token> tokenItr = tokens.iterator(); while (tokenItr.hasNext()) { Token token = tokenItr.next(); // convert token into JCas object BaseToken bta = TokenConverter.convert(token, jcas, beginPos); bta.setTokenNumber(tokenCount); // add JCas object to CAS index bta.addToIndexes(); tokenCount++; } } }
baseToken.setPartOfSpeech(tag); baseToken.setTokenNumber(wordNumber++); baseToken.addToIndexes();
baseToken.setPartOfSpeech(tag); baseToken.setTokenNumber(wordNumber++); baseToken.addToIndexes();
wordEnd = wordStart + tokens[1].length(); BaseToken btoken = new BaseToken(jCas,wordStart,wordEnd); btoken.setTokenNumber(wordNumber++); btoken.addToIndexes(); documentText.append(tokens[1] + " "); depNodes.add( new ConllDependencyNode(jCas,wordStart,wordEnd) ); BaseToken btoken = new BaseToken(jCas,wordStart,wordEnd); btoken.setTokenNumber(wordNumber++); btoken.addToIndexes(); documentText.append(tokens[1] + " "); depNodes.add( new ConllDependencyNode(jCas,wordStart,wordEnd) ); BaseToken btoken = new BaseToken(jCas,wordStart,wordEnd); btoken.setTokenNumber(wordNumber++); btoken.setPartOfSpeech(tokens[2]); btoken.addToIndexes(); depNodes.add( new ConllDependencyNode(jCas,wordStart,wordEnd) ); BaseToken btoken = new BaseToken(jCas,wordStart,wordEnd); btoken.setTokenNumber(wordNumber++); btoken.setNormalizedForm(tokens[2]); btoken.addToIndexes(); depNodes.add( new ConllDependencyNode(jCas,wordStart,wordEnd) ); BaseToken btoken = new BaseToken(jCas,wordStart,wordEnd); btoken.setTokenNumber(wordNumber++); btoken.setNormalizedForm(tokens[2]); btoken.setPartOfSpeech(tokens[3]);
wordEnd = wordStart + tokens[1].length(); BaseToken btoken = new BaseToken(jCas,wordStart,wordEnd); btoken.setTokenNumber(wordNumber++); btoken.addToIndexes(); documentText.append(tokens[1] + " "); depNodes.add( new ConllDependencyNode(jCas,wordStart,wordEnd) ); BaseToken btoken = new BaseToken(jCas,wordStart,wordEnd); btoken.setTokenNumber(wordNumber++); btoken.addToIndexes(); documentText.append(tokens[1] + " "); depNodes.add( new ConllDependencyNode(jCas,wordStart,wordEnd) ); BaseToken btoken = new BaseToken(jCas,wordStart,wordEnd); btoken.setTokenNumber(wordNumber++); btoken.setPartOfSpeech(tokens[2]); btoken.addToIndexes(); depNodes.add( new ConllDependencyNode(jCas,wordStart,wordEnd) ); BaseToken btoken = new BaseToken(jCas,wordStart,wordEnd); btoken.setTokenNumber(wordNumber++); btoken.setNormalizedForm(tokens[2]); btoken.addToIndexes(); depNodes.add( new ConllDependencyNode(jCas,wordStart,wordEnd) ); BaseToken btoken = new BaseToken(jCas,wordStart,wordEnd); btoken.setTokenNumber(wordNumber++); btoken.setNormalizedForm(tokens[2]); btoken.setPartOfSpeech(tokens[3]);
for(BaseToken bta : tokens){ if (bta.getBegin()>=rangeBegin && bta.getBegin()<rangeEnd) { bta.setTokenNumber(tokenCount); tokenCount++;
for(BaseToken bta : tokens){ if (bta.getBegin()>=rangeBegin && bta.getBegin()<rangeEnd) { bta.setTokenNumber(tokenCount); tokenCount++;
newGoldToken.setTokenNumber(oldSystemToken.getTokenNumber());
newGoldToken.setTokenNumber(oldSystemToken.getTokenNumber());