public double calculateItemAgreement(final ICodingAnnotationItem item) { ensureDistanceFunction(); Map<Object, Map<Object, Double>> itemMatrix = CodingAnnotationStudy.countCategoryCoincidence(item); double D_E = calculateExpectedDisagreement();
@Override public double calculateObservedDisagreement() { ensureDistanceFunction(); if (coincidenceMatrix == null) coincidenceMatrix = CodingAnnotationStudy.countCategoryCoincidence(study); double n = 0.0; double result = 0.0; for (Entry<Object, Map<Object, Double>> cat1 : coincidenceMatrix.entrySet()) for (Entry<Object, Double> cat2 : cat1.getValue().entrySet()) { result += cat2.getValue() * distanceFunction.measureDistance(study, cat1.getKey(), cat2.getKey()); n += cat2.getValue(); } result /= n; return result; }
public IAgreementMeasure make(ICodingAnnotationStudy aStudy) { switch (this) { case COHEN_KAPPA_AGREEMENT: return new CohenKappaAgreement(aStudy); case FLEISS_KAPPA_AGREEMENT: return new FleissKappaAgreement(aStudy); case KRIPPENDORFF_ALPHA_NOMINAL_AGREEMENT: return new KrippendorffAlphaAgreement(aStudy, new NominalDistanceFunction()); default: throw new IllegalArgumentException(); } }
/** Calculates the expected inter-rater agreement using the defined * distance function to infer the assumed probability distribution. * @throws NullPointerException if the annotation study is null. * @throws ArithmeticException if there are no items or raters in the * annotation study. */ public double calculateExpectedDisagreement() { ensureDistanceFunction(); if (coincidenceMatrix == null) coincidenceMatrix = CodingAnnotationStudy.countCategoryCoincidence(study); double n = 0.0; Map<Object, Double> marginals = new HashMap<Object, Double>(); for (Entry<Object, Map<Object, Double>> cat1 : coincidenceMatrix.entrySet()) { double n_c = 0.0; for (Entry<Object, Double> cat2 : cat1.getValue().entrySet()) n_c += cat2.getValue(); marginals.put(cat1.getKey(), n_c); n += n_c; } double result = 0.0; for (Entry<Object, Double> cat1 : marginals.entrySet()) for (Entry<Object, Double> cat2 : marginals.entrySet()) result += cat1.getValue() * cat2.getValue() * distanceFunction.measureDistance(study, cat1.getKey(), cat2.getKey()); result /= n * (n - 1.0); return result; }
public double calculateCategoryAgreement(final Object category) { ensureDistanceFunction();