protected double calculateMaximumObservedAgreement() { Map<Object, int[]> annotationsPerCategory = CodingAnnotationStudy.countAnnotationsPerCategory(study); BigDecimal result = new BigDecimal(0); for (Object category : study.getCategories()) { int[] annotations = annotationsPerCategory.get(category); int min = -1; for (int rater = 0; rater < study.getRaterCount(); rater++) if (annotations[rater] < min || min < 0) min = annotations[rater]; if (min > 0) result = result.add(new BigDecimal(min)); } result = result.divide(new BigDecimal(study.getItemCount()), MathContext.DECIMAL128); return result.doubleValue(); }
/** Calculates the expected inter-rater agreement that assumes a * different probability distribution for all raters. * @throws NullPointerException if the annotation study is null. * @throws ArithmeticException if there are no items in the * annotation study. */ @Override public double calculateExpectedAgreement() { Map<Object, int[]> annotationsPerCategory = CodingAnnotationStudy.countAnnotationsPerCategory(study); BigDecimal result = new BigDecimal(0); for (Object category : study.getCategories()) { int[] annotations = annotationsPerCategory.get(category); BigDecimal prod = new BigDecimal(1); for (int rater = 0; rater < study.getRaterCount(); rater++) prod = prod.multiply(new BigDecimal(annotations[rater])); result = result.add(prod); } result = result.divide(new BigDecimal(study.getItemCount()).pow(2), MathContext.DECIMAL128); return result.doubleValue(); }
/** Calculates the expected inter-rater agreement that assumes the same * distribution for all raters and annotations. * @throws NullPointerException if the annotation study is null. * @throws ArithmeticException if there are no items in the * annotation study. */ @Override public double calculateExpectedAgreement() { Map<Object, int[]> annotationsPerCategory = CodingAnnotationStudy.countAnnotationsPerCategory(study); BigDecimal result = new BigDecimal(0); for (Object category : study.getCategories()) { int[] annotationCounts = annotationsPerCategory.get(category); for (int m = 0; m < study.getRaterCount(); m++) for (int n = m + 1; n < study.getRaterCount(); n++) result = result.add(new BigDecimal(annotationCounts[m]) .multiply(new BigDecimal(annotationCounts[n]))); } result = result.multiply(new BigDecimal(2)); result = result.divide(new BigDecimal(study.getRaterCount()) .multiply(new BigDecimal(study.getRaterCount() - 1)) .multiply(new BigDecimal(study.getItemCount()).pow(2)), MathContext.DECIMAL128); return result.doubleValue(); }