protected double doCalculateItemAgreement(final ICodingAnnotationItem item) { Map<Object, Integer> annotationsPerCategory = CodingAnnotationStudy.countTotalAnnotationsPerCategory(item); double result = 0.0; for (Integer count : annotationsPerCategory.values()) result += count * (count - 1); int raterCount = item.getRaterCount(); if (raterCount <= 1) return 0.0; else return result / (double) (raterCount - 1.0); }
Map<Object, Integer> nk = CodingAnnotationStudy.countTotalAnnotationsPerCategory((ICodingAnnotationStudy) study); Integer v;
/** Calculates the expected inter-rater agreement that assumes the same * distribution for all raters and annotations. * @throws NullPointerException if the annotation study is null. * @throws ArithmeticException if there are no items in the * annotation study. */ @Override public double calculateExpectedAgreement() { Map<Object, Integer> annotationsPerCategory = CodingAnnotationStudy.countTotalAnnotationsPerCategory(study); BigDecimal result = new BigDecimal(0); for (Object category : study.getCategories()) result = result.add( new BigDecimal(annotationsPerCategory.get(category)).pow(2)); result = result.divide(new BigDecimal(4).multiply( new BigDecimal(study.getItemCount()).pow(2)), MathContext.DECIMAL128); return result.doubleValue(); }
/** Calculates the expected inter-rater agreement that assumes the same * distribution for all raters and annotations. * @throws NullPointerException if the annotation study is null. * @throws ArithmeticException if there are no items in the * annotation study. */ @Override public double calculateExpectedAgreement() { Map<Object, BigDecimal> categoryProbability = new HashMap<Object, BigDecimal>(); for (ICodingAnnotationItem item : study.getItems()) { Map<Object, Integer> annotationsPerCategory = CodingAnnotationStudy.countTotalAnnotationsPerCategory(item); for (Entry<Object, Integer> counts : annotationsPerCategory.entrySet()) { BigDecimal p = new BigDecimal(counts.getValue()).divide( new BigDecimal(item.getRaterCount()), MathContext.DECIMAL128); BigDecimal value = categoryProbability.get(counts.getKey()); if (value != null) p = p.add(value); categoryProbability.put(counts.getKey(), p); } } BigDecimal result = new BigDecimal(0); for (BigDecimal p : categoryProbability.values()) result = result.add(p.pow(2)); result = result.divide( new BigDecimal(study.getItemCount()).pow(2), MathContext.DECIMAL128); return result.doubleValue(); }
/** Calculates the observed inter-rater agreement for the annotation * study that was passed to the class constructor and the currently * assigned distance function. * @throws NullPointerException if the study is null. * @throws ArithmeticException if the study does not contain any item or * the number of raters is smaller than 2. */ public double calculateObservedDisagreement() { ensureDistanceFunction(); double result = 0.0; for (ICodingAnnotationItem item : study.getItems()) { Map<Object, Integer> annotationsPerCategory = CodingAnnotationStudy.countTotalAnnotationsPerCategory(item); for (Entry<Object, Integer> category1 : annotationsPerCategory.entrySet()) for (Entry<Object, Integer> category2 : annotationsPerCategory.entrySet()) { if (category1.getValue() == null) continue; if (category2.getValue() == null) continue; result += category1.getValue() * category2.getValue() * distanceFunction.measureDistance(study, category1.getKey(), category2.getKey()); } } result /= (double) (study.getItemCount() * study.getRaterCount() * (study.getRaterCount() - 1)); return result; }
/** Calculates the expected inter-rater agreement using the defined * distance function to infer the assumed probability distribution. * @throws NullPointerException if the annotation study is null. * @throws ArithmeticException if there are no items or raters in the * annotation study. */ public double calculateExpectedDisagreement() { ensureDistanceFunction(); double result = 0.0; Map<Object, Integer> annotationsPerCategory = CodingAnnotationStudy.countTotalAnnotationsPerCategory(study); for (Object category1 : study.getCategories()) for (Object category2 : study.getCategories()) result += annotationsPerCategory.get(category1) * annotationsPerCategory.get(category2) * distanceFunction.measureDistance(study, category1, category2); result /= (double) (study.getItemCount() * study.getRaterCount() * (study.getItemCount() * study.getRaterCount() - 1)); return result; }