How to use

getItems

method

in

de.tudarmstadt.ukp.dkpro.statistics.agreement.coding.ICodingAnnotationStudy

public int getNonNullCount(String aCasGroupId)
{
  int i = 0;
  for (ICodingAnnotationItem item : study.getItems()) {
    if (item.getUnit(casGroupIds.indexOf(aCasGroupId)).getCategory() != null) {
      i++;
    }
  }
  return i;
}

public boolean isAllNull(String aCasGroupId)
{
  for (ICodingAnnotationItem item : study.getItems()) {
    if (item.getUnit(casGroupIds.indexOf(aCasGroupId)).getCategory() != null) {
      return false;
    }
  }
  return true;
}

/** Returns a two dimensional map of category pairs and their co-occurrence
 *  frequencies for the given annotation study. */	
// Category x Category -> #
public static Map<Object, Map<Object, Double>> 
    countCategoryCoincidence(final ICodingAnnotationStudy study) {
  Map<Object, Map<Object, Double>> result = 
      new HashMap<Object, Map<Object, Double>>();
  for (ICodingAnnotationItem item : study.getItems()) {
    Map<Object, Map<Object, Double>> itemMatrix = 
        countCategoryCoincidence(item);
  
    for (Entry<Object, Map<Object, Double>> itemCat : itemMatrix.entrySet()) {
      Map<Object, Double> resultCat = result.get(itemCat.getKey());
      if (resultCat == null) {
        resultCat = new HashMap<Object, Double>();
        result.put(itemCat.getKey(), resultCat);
      }
      for (Entry<Object, Double> itemEntry : itemCat.getValue().entrySet()) {
        Double resultEntry = resultCat.get(itemEntry.getKey());
        if (resultEntry == null)
          resultEntry = 0.0;
        resultCat.put(itemEntry.getKey(), resultEntry + itemEntry.getValue());
      }
    }
  }
  return result;
}

private static void dumpAgreementConfigurationSetsWithItems(PrintStream aOut,
    AgreementResult aAgreement, List<ConfigurationSet> aSets)
{
  int i = 0;
  for (ICodingAnnotationItem item : aAgreement.getStudy().getItems()) {
    StringBuilder sb = new StringBuilder();
    sb.append(aSets.get(i).getPosition());
    for (IAnnotationUnit unit : item.getUnits()) {
      if (sb.length() > 0) {
        sb.append(" \t");
      }
      sb.append(unit.getCategory());
    }
    aOut.println(sb);
    i++;
  }
}

@Override
public double calculateObservedAgreement() {
  double result = 0.0;
  double denominator = 0.0;
  for (ICodingAnnotationItem item : study.getItems()) {
    int raterCount = item.getRaterCount();
    if (raterCount > 1) {
      result += doCalculateItemAgreement(item);
      denominator += raterCount;
    }
  }
  return result / denominator;
}

/** Calculates the expected inter-rater agreement that assumes the same
 *  distribution for all raters and annotations. 
 *  @throws NullPointerException if the annotation study is null. 
 *  @throws ArithmeticException if there are no items in the 
 *      annotation study. */
@Override
public double calculateExpectedAgreement() {
  Map<Object, BigDecimal> categoryProbability = new HashMap<Object, BigDecimal>();
  for (ICodingAnnotationItem item : study.getItems()) {
    Map<Object, Integer> annotationsPerCategory 
        = CodingAnnotationStudy.countTotalAnnotationsPerCategory(item);
    for (Entry<Object, Integer> counts : annotationsPerCategory.entrySet()) {
      BigDecimal p = new BigDecimal(counts.getValue()).divide(
          new BigDecimal(item.getRaterCount()), 
          MathContext.DECIMAL128);
      BigDecimal value = categoryProbability.get(counts.getKey());
      if (value != null)
        p = p.add(value);
      categoryProbability.put(counts.getKey(), p);
    }
  }
  BigDecimal result = new BigDecimal(0);
  for (BigDecimal p : categoryProbability.values())
    result = result.add(p.pow(2));
  result = result.divide(
      new BigDecimal(study.getItemCount()).pow(2), 
      MathContext.DECIMAL128);
  return result.doubleValue();
}
 

/** Returns a map of categories and their usage frequencies (i.e., 
 *  how often they are used in annotation units) within the given
 *  annotation study. */
// Category -> #
public static Map<Object, Integer> countTotalAnnotationsPerCategory(
    final ICodingAnnotationStudy study) {
  Map<Object, Integer> result = new HashMap<Object, Integer>();
  for (ICodingAnnotationItem item : study.getItems()) {
    if (item.getRaterCount() <= 1)
      continue;
    
    for (IAnnotationUnit unit : item.getUnits()) {
      Object category = unit.getCategory();
      if (category == null)
        continue;
      
      Integer count = result.get(category);
      if (count == null)
        result.put(category, 1);
      else
        result.put(category, count + 1);
    }
  }
  return result;
}
 

public static void dumpStudy(PrintStream aOut, ICodingAnnotationStudy aStudy)
{
  try {
    aOut.printf("Category count: %d%n", aStudy.getCategoryCount());
  }
  catch (Throwable e) {
    aOut.printf("Category count: %s%n", ExceptionUtils.getRootCauseMessage(e));
  }
  try {
    aOut.printf("Item count: %d%n", aStudy.getItemCount());
  }
  catch (Throwable e) {
    aOut.printf("Item count: %s%n", ExceptionUtils.getRootCauseMessage(e));
  }
  
  for (ICodingAnnotationItem item : aStudy.getItems()) {
    StringBuilder sb = new StringBuilder();
    for (IAnnotationUnit unit : item.getUnits()) {
      if (sb.length() > 0) {
        sb.append(" \t");
      }
      sb.append(unit.getCategory());
    }
    aOut.println(sb);
  }
}

/** Returns a two-dimensional map of categories and raters and the
 *  corresponding usage frequencies in the given annotation study 
 *  (i.e., how often a certain rater used a certain category for
 *  coding an annotation unit). */
// Category x Rater -> #
public static Map<Object, int[]> countAnnotationsPerCategory(
    final ICodingAnnotationStudy study) {
  Map<Object, int[]> result = new HashMap<Object, int[]>();
  for (ICodingAnnotationItem item : study.getItems())
    for (IAnnotationUnit unit : item.getUnits()) {
      Object category = unit.getCategory();
      if (category == null)
        continue;
      
      int[] counts = result.get(category);
      if (counts == null)
        counts = new int[study.getRaterCount()];
      counts[unit.getRaterIdx()]++;
      result.put(category, counts);
    }
  return result;
}

/** Calculates the observed inter-rater agreement for the annotation 
 *  study that was passed to the class constructor and the currently
 *  assigned distance function.
 *  @throws NullPointerException if the study is null.
 *  @throws ArithmeticException if the study does not contain any item or
 *      the number of raters is smaller than 2. */
public double calculateObservedDisagreement() {
  ensureDistanceFunction();
  
  double result = 0.0;
  for (ICodingAnnotationItem item : study.getItems()) {
    Map<Object, Integer> annotationsPerCategory 
        = CodingAnnotationStudy.countTotalAnnotationsPerCategory(item);
          
    for (Entry<Object, Integer> category1 : annotationsPerCategory.entrySet())
      for (Entry<Object, Integer> category2 : annotationsPerCategory.entrySet()) {
        if (category1.getValue() == null)
          continue;
        if (category2.getValue() == null)
          continue;
        
        result += category1.getValue() * category2.getValue()
            * distanceFunction.measureDistance(study, 
                category1.getKey(), category2.getKey());
      }
  }
  result /= (double) (study.getItemCount() * study.getRaterCount() 
      * (study.getRaterCount() - 1));
  return result;
}

for (ICodingAnnotationItem item : study.getItems()) { 
  int cat1 = categories.get(item.getUnit(0).getCategory());
  int cat2 = categories.get(item.getUnit(1).getCategory());

private static void configurationSetsWithItemsToCsv(CSVPrinter aOut,
    AgreementResult aAgreement, List<ConfigurationSet> aSets)
  throws IOException
{
  List<String> headers = new ArrayList<>(
      asList("Type", "Collection", "Document", "Layer", "Feature", "Position"));
  headers.addAll(aAgreement.getCasGroupIds());
  aOut.printRecord(headers);
  
  int i = 0;
  for (ICodingAnnotationItem item : aAgreement.getStudy().getItems()) {
    Position pos = aSets.get(i).getPosition();
    List<String> values = new ArrayList<>();
    values.add(pos.getClass().getSimpleName());
    values.add(pos.getCollectionId());
    values.add(pos.getDocumentId());
    values.add(pos.getType());
    values.add(aAgreement.getFeature());
    values.add(aSets.get(i).getPosition().toMinimalString());
    for (IAnnotationUnit unit : item.getUnits()) {
      values.add(String.valueOf(unit.getCategory()));
    }
    aOut.printRecord(values);
    i++;
  }
}    

for (ICodingAnnotationItem item : study.getItems())
  out.print(DIVIDER + item.getUnit(r).getCategory());
out.println();
out.print(category);
int catSum = 0;
for (ICodingAnnotationItem item : study.getItems()) {
  int catCount = 0;
  for (IAnnotationUnit unit : item.getUnits())

@Override
public double calculateCategoryAgreement(final Object category) {
  // N = # subjects = #items -> index i
  // n = # ratings/subject = #raters
  // k = # categories -> index j
  // n_ij = # raters that annotated item i as category j
  // 		
  // k_j = (P_j - p_j) / (1 - p_j)
  // P_j = (sum( n_ij^2 ) - N n p_j) / (N n (n-1) p_j )
  // p_j = 1/Nn sum n_ij  
  int N = study.getItemCount();
  int n = study.getRaterCount();
  int sum_nij = 0;
  int sum_nij_2 = 0;
  for (ICodingAnnotationItem item : study.getItems()) {
    int nij = 0;
    for (IAnnotationUnit unit : item.getUnits())
      if (unit.getCategory().equals(category))
        nij++;
    sum_nij += nij;
    sum_nij_2 += (nij * nij);
  }
    
  double pj = 1 / (double) (N * n) * sum_nij;
  double Pj = (sum_nij_2 - N * n * pj) / (double) (N * n * (n - 1) * pj);
  double kappaj = (Pj - pj) / (double) (1 - pj);
  return kappaj;
}
 

public double calculateCategoryAgreement(final Object category) {
  // N = # subjects = #items -> index i
  // n = # ratings/subject = #raters
  // k = # categories -> index j
  // n_ij = # raters that annotated item i as category j
  // 		
  // k_j = (P_j - p_j) / (1 - p_j)
  // P_j = (sum( n_ij^2 ) - N n p_j) / (N n (n-1) p_j )
  // p_j = 1/Nn sum n_ij  
  int N = study.getItemCount();
  int n = study.getRaterCount();
  int sum_nij = 0;
  int sum_nij_2 = 0;
  for (ICodingAnnotationItem item : study.getItems()) {
    int nij = 0;
    for (IAnnotationUnit annotation : item.getUnits())
      if (annotation.getCategory().equals(category))
        nij++;
    sum_nij += nij;
    sum_nij_2 += (nij * nij);
  }
    
  double pj = 1 / (double) (N * n) * sum_nij;
  double Pj = (sum_nij_2 - N * n * pj) / (double) (N * n * (n - 1) * pj);
  double kappaj = (Pj - pj) / (double) (1 - pj);
  return kappaj;
}
 

for (ICodingAnnotationItem item : study.getItems()) {
  int catCount = 0;
  int otherCatCount = 0;

int nKeepCategorySum = 0;
int nNullCategorySum = 0;
for (ICodingAnnotationItem item : study.getItems()) {
  int nKeepCategory = 0;
  int nNullCategory = 0;