getting_started.R

In neumanrq/fairreviewers: "Strictness correction mechanics: Re-order a set of documents that have been ranked by humans"

devtools::install_local(here())
# or: devtools::install_github("neumanrq/fairreviewers")
library(fairreviewers)
library(here)

# The rating dataset:
#
# Each column contains the ratings
# of a reviewer.
#
# Each row describes the ratings that
# a single applicant got.
data.env <- randomDataset(20, 5)

head(data.env$ratings)
# Output looks like this:
#
#       Roy Rose Robin Ricarda Ryan
# Arthur  10   10    NA       3    8
# Aisha   NA   10     3       7    7
# Anna    10   10     3      NA    9
# Aisha   10    9     4       2    8
# Ashley   9    9     4       4   NA
# Ally    NA   10     6       2   10
# …
# (Note that duplicated names can occur!)

# Let's start the analysis!
review <- fairreviewers::init(data.env$ratings)
strictness <- review$strictness # contains strictness factors for each reviewer

print(strictness)
# Output looks like this
#
#         Arithmetic Strictness Expected Strictness
# Roy     0.7541836             0.7540541
# Rose    0.7697874             0.7696552
# Robin   1.7717328             1.7714286
# Ricarda 1.5945595             1.5942857
# Ryan    0.8455997             0.8454545
#
# Interpreation: Robin and Ricarda are quite strict
#                reviewers, but Roy is a quite graceful one

result <- review$result
print(result)
#        Roy Rose Robin Ricarda Ryan Mean Rating after AS correction Rating after ES correction
# Arthur  10   10    NA       3    8 7.75                       6.70                       6.70
# Aisha   NA   10     3       7    7 6.75                       7.52                       7.52
# Anna    10   10     3      NA    9 8.00                       7.04                       7.04
# Aisha   10    9     4       2    8 6.60                       6.30                       6.30
# Ashley   9    9     4       4   NA 6.50                       6.80                       6.79
# Ally    NA   10     6       2   10 7.00                       7.49                       7.49
# Ali      9    9     2       4    6 6.00                       5.74                       5.74
# Ali      9   10     1       6    8 6.80                       6.52                       6.52
# Albert   8   NA     3       5    4 5.00                       5.68                       5.67
# Albert  10    5     4      NA    9 7.00                       6.52                       6.52
# Ally     9    6     6       1   NA 5.50                       5.91                       5.91
# Ali     10    9    NA       5    7 7.75                       7.09                       7.09
# Arthur   6   NA     5       6    8 6.25                       7.43                       7.43
# Amanda  10    9     6       3    7 7.00                       7.16                       7.16
# Anna     8    7     2      NA   NA 5.67                       4.99                       4.99
# Aisha   NA    9     5       5    8 6.75                       7.63                       7.63
# Ally     7   NA     4       3    8 5.50                       5.98                       5.98
# Aljona   8    7     4       5    7 6.20                       6.48                       6.48
# Amanda   8    8     1       4    8 5.80                       5.42                       5.42
# Amanda   7    8    NA       5   10 7.50                       6.97                       6.97