How to use

getRaterCount

method

in

de.tudarmstadt.ukp.dkpro.statistics.agreement.coding.ICodingAnnotationStudy

protected void ensureTwoRaters() {
  if (study.getRaterCount() != 2)
    throw new IllegalArgumentException("This agreement measure is only "
        + "applicable for annotation studies with two raters!");
}

/** Calculates the expected inter-rater agreement that assumes the same
 *  distribution for all raters and annotations. 
 *  @throws NullPointerException if the annotation study is null. 
 *  @throws ArithmeticException if there are no items in the 
 *      annotation study. */
@Override
public double calculateExpectedAgreement() {
  Map<Object, int[]> annotationsPerCategory 
      = CodingAnnotationStudy.countAnnotationsPerCategory(study);
  BigDecimal result = new BigDecimal(0);
  for (Object category : study.getCategories()) {
    int[] annotationCounts = annotationsPerCategory.get(category);
    for (int m = 0; m < study.getRaterCount(); m++)
      for (int n = m + 1; n < study.getRaterCount(); n++)
        result = result.add(new BigDecimal(annotationCounts[m])
            .multiply(new BigDecimal(annotationCounts[n])));
  }
  result = result.multiply(new BigDecimal(2));
  result = result.divide(new BigDecimal(study.getRaterCount())
      .multiply(new BigDecimal(study.getRaterCount() - 1))
      .multiply(new BigDecimal(study.getItemCount()).pow(2)), 
      MathContext.DECIMAL128);
  return result.doubleValue();
}

protected double calculateMaximumObservedAgreement() {
  Map<Object, int[]> annotationsPerCategory 
      = CodingAnnotationStudy.countAnnotationsPerCategory(study);
  BigDecimal result = new BigDecimal(0);
  for (Object category : study.getCategories()) {
    int[] annotations = annotationsPerCategory.get(category);
    int min = -1;
    for (int rater = 0; rater < study.getRaterCount(); rater++)
      if (annotations[rater] < min || min < 0)
        min = annotations[rater];
    if (min > 0)
      result = result.add(new BigDecimal(min));
  }
  result = result.divide(new BigDecimal(study.getItemCount()), MathContext.DECIMAL128);
  return result.doubleValue();
}

/** Calculates the expected inter-rater agreement that assumes a 
 *  different probability distribution for all raters. 
 *  @throws NullPointerException if the annotation study is null. 
 *  @throws ArithmeticException if there are no items in the 
 *      annotation study. */
@Override
public double calculateExpectedAgreement() {
  Map<Object, int[]> annotationsPerCategory 
      = CodingAnnotationStudy.countAnnotationsPerCategory(study);
  
  BigDecimal result = new BigDecimal(0);
  for (Object category : study.getCategories()) {
    int[] annotations = annotationsPerCategory.get(category);
    BigDecimal prod = new BigDecimal(1);
    for (int rater = 0; rater < study.getRaterCount(); rater++)
      prod = prod.multiply(new BigDecimal(annotations[rater]));
    result = result.add(prod);
  }
  result = result.divide(new BigDecimal(study.getItemCount()).pow(2), MathContext.DECIMAL128);
  return result.doubleValue();
}
 

/** Calculates the observed inter-rater agreement for the annotation 
 *  study that was passed to the class constructor and the currently
 *  assigned distance function.
 *  @throws NullPointerException if the study is null.
 *  @throws ArithmeticException if the study does not contain any item or
 *      the number of raters is smaller than 2. */
public double calculateObservedDisagreement() {
  ensureDistanceFunction();
  
  double result = 0.0;
  for (ICodingAnnotationItem item : study.getItems()) {
    Map<Object, Integer> annotationsPerCategory 
        = CodingAnnotationStudy.countTotalAnnotationsPerCategory(item);
          
    for (Entry<Object, Integer> category1 : annotationsPerCategory.entrySet())
      for (Entry<Object, Integer> category2 : annotationsPerCategory.entrySet()) {
        if (category1.getValue() == null)
          continue;
        if (category2.getValue() == null)
          continue;
        
        result += category1.getValue() * category2.getValue()
            * distanceFunction.measureDistance(study, 
                category1.getKey(), category2.getKey());
      }
  }
  result /= (double) (study.getItemCount() * study.getRaterCount() 
      * (study.getRaterCount() - 1));
  return result;
}

/** Returns a two-dimensional map of categories and raters and the
 *  corresponding usage frequencies in the given annotation study 
 *  (i.e., how often a certain rater used a certain category for
 *  coding an annotation unit). */
// Category x Rater -> #
public static Map<Object, int[]> countAnnotationsPerCategory(
    final ICodingAnnotationStudy study) {
  Map<Object, int[]> result = new HashMap<Object, int[]>();
  for (ICodingAnnotationItem item : study.getItems())
    for (IAnnotationUnit unit : item.getUnits()) {
      Object category = unit.getCategory();
      if (category == null)
        continue;
      
      int[] counts = result.get(category);
      if (counts == null)
        counts = new int[study.getRaterCount()];
      counts[unit.getRaterIdx()]++;
      result.put(category, counts);
    }
  return result;
}

if (study.getRaterCount() > 2)
  throw new IllegalArgumentException("Contingency tables are only applicable for two rater studies.");

/** Calculates the expected inter-rater agreement using the defined 
 *  distance function to infer the assumed probability distribution. 
 *  @throws NullPointerException if the annotation study is null. 
 *  @throws ArithmeticException if there are no items or raters in the 
 *      annotation study. */
public double calculateExpectedDisagreement() {
  ensureDistanceFunction();
  
  double result = 0.0;
  Map<Object, Integer> annotationsPerCategory 
      = CodingAnnotationStudy.countTotalAnnotationsPerCategory(study);
  
  for (Object category1 : study.getCategories())
    for (Object category2 : study.getCategories())
      result += annotationsPerCategory.get(category1)
          * annotationsPerCategory.get(category2)
          * distanceFunction.measureDistance(study, category1, category2);
    
  result /= (double) (study.getItemCount() * study.getRaterCount() 
      * (study.getItemCount() * study.getRaterCount() - 1));
  return result;
}

* study.getRaterCount() * (study.getRaterCount() - 1));

out.println();
for (int r = 0; r < study.getRaterCount(); r++) {
  out.print(r + 1);
  for (ICodingAnnotationItem item : study.getItems())

@Override
public double calculateCategoryAgreement(final Object category) {
  // N = # subjects = #items -> index i
  // n = # ratings/subject = #raters
  // k = # categories -> index j
  // n_ij = # raters that annotated item i as category j
  // 		
  // k_j = (P_j - p_j) / (1 - p_j)
  // P_j = (sum( n_ij^2 ) - N n p_j) / (N n (n-1) p_j )
  // p_j = 1/Nn sum n_ij  
  int N = study.getItemCount();
  int n = study.getRaterCount();
  int sum_nij = 0;
  int sum_nij_2 = 0;
  for (ICodingAnnotationItem item : study.getItems()) {
    int nij = 0;
    for (IAnnotationUnit unit : item.getUnits())
      if (unit.getCategory().equals(category))
        nij++;
    sum_nij += nij;
    sum_nij_2 += (nij * nij);
  }
    
  double pj = 1 / (double) (N * n) * sum_nij;
  double Pj = (sum_nij_2 - N * n * pj) / (double) (N * n * (n - 1) * pj);
  double kappaj = (Pj - pj) / (double) (1 - pj);
  return kappaj;
}
 

public double calculateCategoryAgreement(final Object category) {
  // N = # subjects = #items -> index i
  // n = # ratings/subject = #raters
  // k = # categories -> index j
  // n_ij = # raters that annotated item i as category j
  // 		
  // k_j = (P_j - p_j) / (1 - p_j)
  // P_j = (sum( n_ij^2 ) - N n p_j) / (N n (n-1) p_j )
  // p_j = 1/Nn sum n_ij  
  int N = study.getItemCount();
  int n = study.getRaterCount();
  int sum_nij = 0;
  int sum_nij_2 = 0;
  for (ICodingAnnotationItem item : study.getItems()) {
    int nij = 0;
    for (IAnnotationUnit annotation : item.getUnits())
      if (annotation.getCategory().equals(category))
        nij++;
    sum_nij += nij;
    sum_nij_2 += (nij * nij);
  }
    
  double pj = 1 / (double) (N * n) * sum_nij;
  double Pj = (sum_nij_2 - N * n * pj) / (double) (N * n * (n - 1) * pj);
  double kappaj = (Pj - pj) / (double) (1 - pj);
  return kappaj;
}
 

    * study.getRaterCount() * (study.getRaterCount() - 1);
    + nNullCategorySum * nKeepCategorySum * distanceFunction.measureDistance(study, NULL_CATEGORY, category)
    + nNullCategorySum * nNullCategorySum * distanceFunction.measureDistance(study, NULL_CATEGORY, NULL_CATEGORY);
expectedDisagreement /= (double) study.getItemCount() * study.getRaterCount() 
    * (study.getItemCount() * study.getRaterCount() - 1);