protected SpanLabelView makeSpanLabeledView(List<String> chunkLabels, List<Integer> chunkStart, List<Integer> chunkEnd, TextAnnotation ta, String viewName) { SpanLabelView view = new SpanLabelView(viewName, "GoldStandard", ta, 1.0); for (int i = 0; i < chunkLabels.size(); i++) { view.addSpanLabel(chunkStart.get(i), chunkEnd.get(i), chunkLabels.get(i), 1.0); } return view; }
protected SpanLabelView makeSpanLabeledView(List<String> chunkLabels, List<Integer> chunkStart, List<Integer> chunkEnd, TextAnnotation ta, String viewName) { SpanLabelView view = new SpanLabelView(viewName, "GoldStandard", ta, 1.0); for (int i = 0; i < chunkLabels.size(); i++) { view.addSpanLabel(chunkStart.get(i), chunkEnd.get(i), chunkLabels.get(i), 1.0); } return view; }
protected void addGoldBIOView(TextAnnotation ta, List<String> labels) { SpanLabelView lightVerbView = new SpanLabelView(viewName, ta); int startSpan = -1; String prevLabel = null; for (int i = 0; i < labels.size(); i++) { String label = labels.get(i); if (label.startsWith("B")) { startSpan = i; prevLabel = label; } if (label.startsWith("O") && startSpan != -1) { lightVerbView.addSpanLabel(startSpan, i, prevLabel.substring(2), 1.0); startSpan = -1; } } ta.addView(viewName, lightVerbView); }
protected void addGoldBIOView(TextAnnotation ta, List<String> labels) { SpanLabelView lightVerbView = new SpanLabelView(viewName, ta); int startSpan = -1; String prevLabel = null; for (int i = 0; i < labels.size(); i++) { String label = labels.get(i); if (label.startsWith("B")) { startSpan = i; prevLabel = label; } if (label.startsWith("O") && startSpan != -1) { lightVerbView.addSpanLabel(startSpan, i, prevLabel.substring(2), 1.0); startSpan = -1; } } ta.addView(viewName, lightVerbView); }
protected void addGoldBIOView(TextAnnotation ta, List<String> labels) { SpanLabelView lightVerbView = new SpanLabelView(viewName, ta); int startSpan = -1; String prevLabel = null; for (int i = 0; i < labels.size(); i++) { String label = labels.get(i); if (label.startsWith("B")) { startSpan = i; prevLabel = label; } if (label.startsWith("O") && startSpan != -1) { lightVerbView.addSpanLabel(startSpan, i, prevLabel.substring(2), 1.0); startSpan = -1; } } ta.addView(viewName, lightVerbView); }
/** * The view will consist of potentially overlapping constituents representing those tokens that * matched entries in the gazetteers. Some tokens will match against several gazetteers. */ @Override public void addView(TextAnnotation ta) throws AnnotatorException { View view = ta.getView(ViewNames.TOKENS); List<Constituent> constituents = view.getConstituents(); SpanLabelView slv = new SpanLabelView(this.getViewName(), this.getClass().getName(), ta, 1d, true); for (int constindx = 0; constindx < constituents.size(); constindx++) { for (int dictindx = 0; dictindx < dictionaries.size(); dictindx++) { dictionaries.get(dictindx).match(constituents, constindx, slv); dictionariesIgnoreCase.get(dictindx).match(constituents, constindx, slv); } } ta.addView(slv.getViewName(), slv); } }
/** * The view will consist of potentially overlapping constituents representing those tokens that * matched entries in the gazetteers. Some tokens will match against several gazetteers. */ @Override public void addView(TextAnnotation ta) throws AnnotatorException { View view = ta.getView(ViewNames.TOKENS); List<Constituent> constituents = view.getConstituents(); SpanLabelView slv = new SpanLabelView(this.getViewName(), this.getClass().getName(), ta, 1d, true); for (int constindx = 0; constindx < constituents.size(); constindx++) { for (int dictindx = 0; dictindx < dictionaries.size(); dictindx++) { dictionaries.get(dictindx).match(constituents, constindx, slv); dictionariesIgnoreCase.get(dictindx).match(constituents, constindx, slv); } } ta.addView(slv.getViewName(), slv); } }
@Override protected void addView(TextAnnotation ta) throws AnnotatorException { SpanLabelView datalessView = new SpanLabelView(getViewName(), getClassName(), ta, 1d, true); List<Constituent> tokens = ta.getView(ViewNames.TOKENS).getConstituents(); int numTokens = tokens.size(); int textStart = tokens.get(0).getSpan().getFirst(); int textEnd = tokens.get(numTokens - 1).getSpan().getSecond(); StringBuilder sb = new StringBuilder(); for (String s : ta.getTokensInSpan(textStart, textEnd)) { sb.append(s); sb.append(" "); } SparseVector<Integer> docVector = embedding.getVector(sb.toString().trim()); Set<String> labelIDs = classifier.getFlatPredictions(docVector, topK); for (String labelID : labelIDs) { datalessView.addSpanLabel(textStart, textEnd, labelID, 1d); } ta.addView(getViewName(), datalessView); } }
@Override protected void addView(TextAnnotation ta) throws AnnotatorException { SpanLabelView datalessView = new SpanLabelView(getViewName(), getClassName(), ta, 1d, true); List<Constituent> tokens = ta.getView(ViewNames.TOKENS).getConstituents(); int numTokens = tokens.size(); int textStart = tokens.get(0).getSpan().getFirst(); int textEnd = tokens.get(numTokens - 1).getSpan().getSecond(); StringBuilder sb = new StringBuilder(); for (String s : ta.getTokensInSpan(textStart, textEnd)) { sb.append(s); sb.append(" "); } SparseVector<Integer> docVector = embedding.getVector(sb.toString().trim()); Set<String> labelIDs = classifier.getFlatPredictions(docVector, topK); for (String labelID : labelIDs) { datalessView.addSpanLabel(textStart, textEnd, labelID, 1d); } ta.addView(getViewName(), datalessView); } }
@Override public void addView(TextAnnotation ta) throws AnnotatorException { assert (ta.hasView(ViewNames.SENTENCE)); SpanLabelView quantifierView = new SpanLabelView(ViewNames.QUANTITIES, "illinois-quantifier", ta, 1d); List<QuantSpan> quantSpans = getSpans(ta.getTokenizedText(), true, ta); for (QuantSpan span : quantSpans) { int startToken = ta.getTokenIdFromCharacterOffset(span.start); int endToken = ta.getTokenIdFromCharacterOffset(span.end); quantifierView.addSpanLabel(startToken, endToken, span.object.toString(), 1d); } ta.addView(ViewNames.QUANTITIES, quantifierView); }
@Override public void addView(TextAnnotation ta) throws AnnotatorException { assert (ta.hasView(ViewNames.SENTENCE)); SpanLabelView quantifierView = new SpanLabelView(ViewNames.QUANTITIES, "illinois-quantifier", ta, 1d); List<QuantSpan> quantSpans = getSpans(ta.getTokenizedText(), true, ta); for (QuantSpan span : quantSpans) { int startToken = ta.getTokenIdFromCharacterOffset(span.start); int endToken = ta.getTokenIdFromCharacterOffset(span.end); quantifierView.addSpanLabel(startToken, endToken, span.object.toString(), 1d); } ta.addView(ViewNames.QUANTITIES, quantifierView); }
@Override public void addView(TextAnnotation ta) { lazyLoadClusters(); SpanLabelView view = new SpanLabelView(getViewName(), "BrownClusters", ta, 1.0, true); Map<String, List<IntPair>> m = getMatchingSpans(ta); for (Entry<String, List<IntPair>> entry : m.entrySet()) { String label = entry.getKey(); Set<IntPair> added = new LinkedHashSet<>(); for (IntPair p : entry.getValue()) { // don't add nested constituents of the same type boolean foundContainer = false; for (IntPair p1 : added) { if (p1 == p) continue; if (p1.getFirst() <= p.getFirst() && p1.getSecond() >= p.getSecond()) { foundContainer = true; break; } } if (!foundContainer) { view.addSpanLabel(p.getFirst(), p.getSecond(), label, 1.0); added.add(p); } } } ta.addView(getViewName(), view); }
/** * Helper for create a SpanLabelView from a stream of span labels */ private static int createSpanLabelView( Stream<Pair<IntPair, String>> spans, TextAnnotation ta, String viewName, boolean allowOverlapping) { SpanLabelView view = new SpanLabelView(viewName, "GoldStandard", ta, 1.0, allowOverlapping); spans.forEach(span -> view.addSpanLabel( span.getFirst().getFirst(), span.getFirst().getSecond(), span.getSecond(), 1.0)); ta.addView(viewName, view); return view.count(); }
/** * Helper for create a SpanLabelView from a stream of span labels */ private static int createSpanLabelView( Stream<Pair<IntPair, String>> spans, TextAnnotation ta, String viewName, boolean allowOverlapping) { SpanLabelView view = new SpanLabelView(viewName, "GoldStandard", ta, 1.0, allowOverlapping); spans.forEach(span -> view.addSpanLabel( span.getFirst().getFirst(), span.getFirst().getSecond(), span.getSecond(), 1.0)); ta.addView(viewName, view); return view.count(); }
/** * add constituent, creating view if needed (using constituent's viewname) * @param ta * @param c */ private static void addConstituentToTextAnnotation(TextAnnotation ta, Constituent c) { String viewName = c.getViewName(); View v = null; if ( !ta.hasView( viewName ) ) { v = new SpanLabelView( viewName, NAME, ta, 1.0 ); ta.addView( viewName, v ); } else v = ta.getView( viewName ); v.addConstituent( c ); return; }
protected TextAnnotation initializeDummySentenceVerb() { List<String[]> listOfTokens = new ArrayList<>(); listOfTokens.add(new String[] {"I", "do", "."}); TextAnnotation ta = BasicTextAnnotationBuilder.createTextAnnotationFromTokens("", "", listOfTokens); TokenLabelView tlv = new TokenLabelView(ViewNames.POS, "Test", ta, 1.0); tlv.addTokenLabel(0, "PRP", 1d); tlv.addTokenLabel(1, "VBP", 1d); tlv.addTokenLabel(2, ".", 1d); ta.addView(ViewNames.POS, tlv); ta.addView(ViewNames.NER, new SpanLabelView(ViewNames.NER, "test", ta, 1d)); SpanLabelView chunks = new SpanLabelView(ViewNames.SHALLOW_PARSE, "test", ta, 1d); chunks.addSpanLabel(0, 1, "NP", 1d); chunks.addSpanLabel(1, 2, "VP", 1d); ta.addView(ViewNames.SHALLOW_PARSE, chunks); TokenLabelView view = new TokenLabelView(ViewNames.LEMMA, "test", ta, 1d); view.addTokenLabel(0, "i", 1d); view.addTokenLabel(1, "do", 1d); view.addTokenLabel(2, ".", 1d); ta.addView(ViewNames.LEMMA, view); return ta; }
protected TextAnnotation initializeDummySentenceVerb() { List<String[]> listOfTokens = new ArrayList<>(); listOfTokens.add(new String[] {"I", "do", "."}); TextAnnotation ta = BasicTextAnnotationBuilder.createTextAnnotationFromTokens("", "", listOfTokens); TokenLabelView tlv = new TokenLabelView(ViewNames.POS, "Test", ta, 1.0); tlv.addTokenLabel(0, "PRP", 1d); tlv.addTokenLabel(1, "VBP", 1d); tlv.addTokenLabel(2, ".", 1d); ta.addView(ViewNames.POS, tlv); ta.addView(ViewNames.NER, new SpanLabelView(ViewNames.NER, "test", ta, 1d)); SpanLabelView chunks = new SpanLabelView(ViewNames.SHALLOW_PARSE, "test", ta, 1d); chunks.addSpanLabel(0, 1, "NP", 1d); chunks.addSpanLabel(1, 2, "VP", 1d); ta.addView(ViewNames.SHALLOW_PARSE, chunks); TokenLabelView view = new TokenLabelView(ViewNames.LEMMA, "test", ta, 1d); view.addTokenLabel(0, "i", 1d); view.addTokenLabel(1, "do", 1d); view.addTokenLabel(2, ".", 1d); ta.addView(ViewNames.LEMMA, view); return ta; }
@Override protected void addView(TextAnnotation ta) throws AnnotatorException { SpanLabelView view = new SpanLabelView(ViewNames.QUESTION_TYPE, ViewNames.QUESTION_TYPE, ta, 1.0); assert ta.getAvailableViews().contains(ViewNames.SHALLOW_PARSE) && ta.getAvailableViews().contains(ViewNames.NER_CONLL) && ta.getAvailableViews().contains(ViewNames.NER_ONTONOTES): "the annotator does not have the required views "; String fineLabel = fine.discreteValue(ta); Double fineLabelScore = fine.scores(ta).getScore(fineLabel).score; String coarseLabel = coarse.discreteValue(ta); Double coarseLabelScore = coarse.scores(ta).getScore(coarseLabel).score; Constituent cFine = new Constituent(fineLabel, fineLabelScore, ViewNames.QUESTION_TYPE, ta, 0, ta.getTokens().length); Constituent cCoarse = new Constituent(coarseLabel, coarseLabelScore, ViewNames.QUESTION_TYPE, ta, 0, ta.getTokens().length); view.addConstituent(cCoarse); view.addConstituent(cFine); ta.addView(ViewNames.QUESTION_TYPE, view); } }
@Override protected void addView(TextAnnotation ta) throws AnnotatorException { SpanLabelView view = new SpanLabelView(ViewNames.QUESTION_TYPE, ViewNames.QUESTION_TYPE, ta, 1.0); assert ta.getAvailableViews().contains(ViewNames.SHALLOW_PARSE) && ta.getAvailableViews().contains(ViewNames.NER_CONLL) && ta.getAvailableViews().contains(ViewNames.NER_ONTONOTES): "the annotator does not have the required views "; String fineLabel = fine.discreteValue(ta); Double fineLabelScore = fine.scores(ta).getScore(fineLabel).score; String coarseLabel = coarse.discreteValue(ta); Double coarseLabelScore = coarse.scores(ta).getScore(coarseLabel).score; Constituent cFine = new Constituent(fineLabel, fineLabelScore, ViewNames.QUESTION_TYPE, ta, 0, ta.getTokens().length); Constituent cCoarse = new Constituent(coarseLabel, coarseLabelScore, ViewNames.QUESTION_TYPE, ta, 0, ta.getTokens().length); view.addConstituent(cCoarse); view.addConstituent(cFine); ta.addView(ViewNames.QUESTION_TYPE, view); } }
@Override protected void addView(TextAnnotation ta) throws AnnotatorException { Annotation document = new Annotation(ta.text); pipeline.annotate(document); SpanLabelView vu = new SpanLabelView(viewName, ta); for (CoreMap sentence : document.get(CoreAnnotations.SentencesAnnotation.class)) { for(RelationMention rm : sentence.get(MachineReadingAnnotations.RelationMentionsAnnotation.class)) { if(rm.getType().equals("_NR")) continue; Map<String, Double> scores = new HashMap<>(); for(String label : rm.getTypeProbabilities().keySet()) scores.put(label, rm.getTypeProbabilities().getCount(label)); Constituent c1 = createConstituentGivenMention(rm.getEntityMentionArgs().get(0), ta); Constituent c2 = createConstituentGivenMention(rm.getEntityMentionArgs().get(1), ta); Relation r = new Relation(scores, c1, c2); vu.addRelation(r); if(!vu.containsConstituent(c1)) vu.addConstituent(c1); if(!vu.containsConstituent(c2)) vu.addConstituent(c2); } } for (CoreMap sentence : document.get(CoreAnnotations.SentencesAnnotation.class)) { for (EntityMention rm : sentence.get(MachineReadingAnnotations.EntityMentionsAnnotation.class)) { Constituent c = createConstituentGivenMention(rm, ta); if(!vu.containsConstituent(c)) vu.addConstituent(c); } } ta.addView(viewName, vu); }