experiment.R

In neumanrq/fairreviewers: "Strictness correction mechanics: Re-order a set of documents that have been ranked by humans"

library(here)
library(fairreviewers)
library(magrittr)

N  <- 8 # Number of documents
NR <- 5 # Numer of reviewers

reviewerNames  <- c("Ricarda", "Ryan", "Rose", "Rahel", "Ruben") %>% t
applicantNames <- c("Anna", "Arthur", "Ashley", "Aisha", "Albert", "Ali", "Aljona", "Amanda") %>% t

# Choose "hidden" random expecation values between 2 and 8
# for our reviewers. This is a simulation of different
# strictness of the reviewers and shall "revealed" by our
# model afterwards
reviewerExpectedValues <- runif(NR, min=1, max=8) %>% ceiling

reviewerInformation <- rbind(reviewerExpectedValues)
rownames(reviewerInformation) <- c("Hidden expected value")
colnames(reviewerInformation) <- reviewerNames
reviewerInformation

# Returns a rating vector for a fixed
# reviewer for all documents
ratingVector <- function(name) {
  index <- reviewerNames == name
  selectedExpectation <- reviewerExpectedValues[index]
  ratings <- abs(rnorm(8 , selectedExpectation, 1.5))

  # Randomly delete some ratings to simulate that
  # not all reviewers reviewed every document
  indices <- sample(c(1:8), 3)
  ratings[indices] <- NA

  ratings
}

ratings <- reviewerNames %>% apply(2, ratingVector)

colnames(ratings) <- reviewerNames
rownames(ratings) <- applicantNames
ratings