/** * Getter for token text * @return the text of the token */ public String getTokenText(){ return token.getSpan(); } /**
/** * Getter for token text * @return the text of the token */ public String getTokenText(){ return token.getSpan(); } /**
/** * Creates a new Linked Entity including the first {@link Occurrence} * @param section the sentence (context) for the occurrence. * @param startToken the index of the start token * @param tokenSpan the number of token included in this span * @param suggestions the entity suggestions * @param types the types of the linked entity. */ protected LinkedEntity(Section section,Token startToken,Token endToken, List<Suggestion> suggestions, Set<IRI> types) { this(startToken.getSpan().substring(startToken.getStart(), endToken.getEnd()), suggestions,types); addOccurrence(section, startToken,endToken); } /**
/** * Creates a new Linked Entity including the first {@link Occurrence} * @param section the sentence (context) for the occurrence. * @param startToken the index of the start token * @param tokenSpan the number of token included in this span * @param suggestions the entity suggestions * @param types the types of the linked entity. */ protected LinkedEntity(Section section,Token startToken,Token endToken, List<Suggestion> suggestions, Set<IRI> types) { this(startToken.getSpan().substring(startToken.getStart(), endToken.getEnd()), suggestions,types); addOccurrence(section, startToken,endToken); } /**
/** * Checks if the parsed {@link Token} represents an negation * @param token the word * @param index the index of the token relative to the sentence | section * @param language the language * @return <code>true</code> if the {@link Token} represents a negation. * Otherwise <code>false</code> */ private boolean isNoun(Token token, boolean firstTokenInSentence, String language) { String word = token.getSpan(); if(!firstTokenInSentence && !word.isEmpty() && Character.isUpperCase(word.charAt(0))){ return true; //assume all upper case tokens are Nouns } Value<PosTag> posAnnotation = token.getAnnotation(NlpAnnotations.POS_ANNOTATION); if(posAnnotation != null && (posAnnotation.value().hasCategory(LexicalCategory.Noun) || posAnnotation.value().getPosHierarchy().contains(Pos.CardinalNumber))){ return true; } return false; } /**
String[] tokenTexts = new String[tokenList.size()]; for(int i=0;i<tokenList.size(); i++){ tokenTexts[i] = tokenList.get(i).getSpan();
Token t = it.next(); tokens.add(t); words.add(t.getSpan());
Token token = tokens.next(); tokenList.add(token); tokenTextList.add(token.getSpan()); Value<PosTag> posValue = token.getAnnotation(POS_ANNOTATION); if(posValue == null){ throw new EngineException("Missing POS value for Token '" + token.getSpan()+"' of ContentItem "+ci.getUri() + "(Sentence: '"+sentence.getSpan()+"'). This may " + "indicate that a POS tagging Engine is missing in "
log.debug(" [{}] '{}' pos: {}, sentiment {}", new Object[]{ addToList ? sentimentTokens.size() : "-", word.getSpan(),pos.value().getCategories(), sentiment == null ? "none" : sentiment.getValue()});
if(!ct.isLinkable) { //if not already processable log.debug(" > convert Token {}: {} (pos:{}) from matchable to processable", new Object[]{i,ct.token.getSpan(),ct.token.getAnnotations(POS_ANNOTATION)}); ct.isLinkable = true; if(!hasLinkableToken){
if(!ct.isLinkable) { //if not already processable log.debug(" > convert Token {}: {} (pos:{}) from matchable to processable", new Object[]{i,ct.token.getSpan(),ct.token.getAnnotations(POS_ANNOTATION)}); ct.isLinkable = true; if(!hasLinkableToken){
String word = token.getSpan(); double sentiment = 0.0; if(cats.isEmpty()){
List<String> list = new ArrayList<String>(searchStrings.size()); for(TokenData dt : searchStrings){ list.add(dt.token.getSpan());
List<String> list = new ArrayList<String>(searchStrings.size()); for(TokenData dt : searchStrings){ list.add(dt.token.getSpan());
this.index = index; this.inChunk = chunk; this.hasAlphaNumeric = Utils.hasAlphaNumericChar(token.getSpan()); this.hasSearchableLength = token.getSpan().length() >= tpc.getMinSearchTokenLength(); PosTag selectedPosTag = null; boolean matchedPosTag = false; //matched any of the POS annotations Character.isUpperCase(token.getSpan().codePointAt(0)); //and upper case boolean isLinkablePos = false; boolean isMatchablePos = false;
this.index = index; this.inChunk = chunk; this.hasAlphaNumeric = Utils.hasAlphaNumericChar(token.getSpan()); this.hasSearchableLength = token.getSpan().length() >= tpc.getMinSearchTokenLength(); PosTag selectedPosTag = null; boolean matchedPosTag = false; //matched any of the POS annotations Character.isUpperCase(token.getSpan().codePointAt(0)); //and upper case boolean isLinkablePos = false; boolean isMatchablePos = false;